From 5e5245db04a5f5fe2c3b3bc854363df882ad8e82 Mon Sep 17 00:00:00 2001 From: h-munakata Date: Thu, 17 Sep 2026 15:32:30 +0900 Subject: [PATCH 1/3] Fix bug of timestamps --- data/castella/castella_test_release.jsonl | 2 +- data/castella/castella_train_release.jsonl | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/data/castella/castella_test_release.jsonl b/data/castella/castella_test_release.jsonl index 0de43c9..9e4208a 100644 --- a/data/castella/castella_test_release.jsonl +++ b/data/castella/castella_test_release.jsonl @@ -666,7 +666,7 @@ {"qid": "RfGapDlAYoQ_3", "query": "A whistle blows as cheers erupt", "duration": 150, "vid": "RfGapDlAYoQ", "relevant_windows": [[113, 117]]} {"qid": "Rk-ujWKzPuc_1", "query": "A man speaks catching his breath", "duration": 300, "vid": "Rk-ujWKzPuc", "relevant_windows": [[84, 98], [137, 143], [181, 199], [203, 206], [237, 245], [247, 258], [290, 300]]} {"qid": "Rk-ujWKzPuc_2", "query": "Wind blows as water flows", "duration": 300, "vid": "Rk-ujWKzPuc", "relevant_windows": [[261, 290], [57, 83], [103, 113]]} -{"qid": "Rp4Ct_TQvAM_1", "query": "A man talks with background noise", "duration": 299, "vid": "Rp4Ct_TQvAM", "relevant_windows": [[0, 10], [15, 34], [49, 56], [65, 78], [89, 96], [104, 128], [141, 173], [184, 191], [233, 238], [265, 291], [298, 301]]} +{"qid": "Rp4Ct_TQvAM_1", "query": "A man talks with background noise", "duration": 299, "vid": "Rp4Ct_TQvAM", "relevant_windows": [[0, 10], [15, 34], [49, 56], [65, 78], [89, 96], [104, 128], [141, 173], [184, 191], [233, 238], [265, 291], [298, 300]]} {"qid": "Rp4Ct_TQvAM_2", "query": "A man talks and a door closes with background noise", "duration": 299, "vid": "Rp4Ct_TQvAM", "relevant_windows": [[236, 238]]} {"qid": "Rp4Ct_TQvAM_3", "query": "A train passes by with background noise", "duration": 299, "vid": "Rp4Ct_TQvAM", "relevant_windows": [[245, 260]]} {"qid": "RrmBGjJqlEo_1", "query": "Someone sniffles", "duration": 300, "vid": "RrmBGjJqlEo", "relevant_windows": [[77, 79], [109, 111], [121, 122], [186, 187], [245, 246], [297, 298]]} diff --git a/data/castella/castella_train_release.jsonl b/data/castella/castella_train_release.jsonl index b48506f..65b5c88 100644 --- a/data/castella/castella_train_release.jsonl +++ b/data/castella/castella_train_release.jsonl @@ -653,7 +653,7 @@ {"qid": "66Sl9ixB_5U_2", "query": "Many people are talking and birds are chirping", "duration": 300, "vid": "66Sl9ixB_5U", "relevant_windows": [[134, 180]]} {"qid": "674mDF2k6Jw_1", "query": "While two men are having a conversation, someone keeps banging on something", "duration": 299, "vid": "674mDF2k6Jw", "relevant_windows": [[19, 39]]} {"qid": "674mDF2k6Jw_2", "query": "Someone fires a gun", "duration": 299, "vid": "674mDF2k6Jw", "relevant_windows": [[79, 81], [103, 104], [88, 89]]} -{"qid": "67AE9ZKvECs_1", "query": "Birds chirp while something makes a scraping sound", "duration": 299, "vid": "67AE9ZKvECs", "relevant_windows": [[206, 209], [221, 223], [283, 285], [299, 4800], [255, 258], [266, 270]]} +{"qid": "67AE9ZKvECs_1", "query": "Birds chirp while something makes a scraping sound", "duration": 299, "vid": "67AE9ZKvECs", "relevant_windows": [[206, 209], [221, 223], [283, 285], [299, 300], [255, 258], [266, 270]]} {"qid": "67AE9ZKvECs_2", "query": "A bird is chirping while electronic sounds are happening", "duration": 299, "vid": "67AE9ZKvECs", "relevant_windows": [[232, 235], [268, 270], [291, 294]]} {"qid": "67AE9ZKvECs_3", "query": "A bird is chirping while an insect is flying", "duration": 299, "vid": "67AE9ZKvECs", "relevant_windows": [[248, 252], [292, 294]]} {"qid": "67qTV1MDXhc_1", "query": "A dog barks as the wind blows and the crowd murmurs", "duration": 241, "vid": "67qTV1MDXhc", "relevant_windows": [[160, 161], [166, 169]]} From 6359b66d450f2352db07145c732c90e5e3f27729 Mon Sep 17 00:00:00 2001 From: h-munakata Date: Thu, 17 Sep 2026 15:57:06 +0900 Subject: [PATCH 2/3] Apply ruff --- lighthouse/common/CIM.py | 14 +++++++++++--- lighthouse/common/cg_detr_transformer.py | 6 +++++- lighthouse/common/qd_detr_transformer.py | 7 +++++-- lighthouse/common/taskweave_transformer.py | 8 ++++++-- lighthouse/common/tr_detr_transformer.py | 12 ++++++++++-- 5 files changed, 37 insertions(+), 10 deletions(-) diff --git a/lighthouse/common/CIM.py b/lighthouse/common/CIM.py index 71530d7..0fde210 100644 --- a/lighthouse/common/CIM.py +++ b/lighthouse/common/CIM.py @@ -700,7 +700,11 @@ def forward_post(self, # print(src_key_padding_mask[:, 76:].shape) # torch.Size([32, 26]) qmask, kmask = src_key_padding_mask[:, 1:video_length + 1].unsqueeze(2), src_key_padding_mask[:, video_length + 1:].unsqueeze(1) - attn_mask = torch.matmul(qmask.float(), kmask.float()).bool().repeat_interleave(self.nhead, dim=0) # batch-major layout for nn.MultiheadAttention 3D attn_mask + attn_mask = ( + torch.matmul(qmask.float(), kmask.float()) + .bool() + .repeat_interleave(self.nhead, dim=0) + ) # batch-major layout for nn.MultiheadAttention 3D attn_mask # print(attn_mask.shape) # print(attn_mask[0][0]) # print(q.shape) 75 32 256 @@ -789,7 +793,11 @@ def forward_post(self, global_token, q, k, v = src[0].unsqueeze(0), pos_src[video_length + 1: ], pos_src[1:video_length + 1], src[1:video_length + 1] qmask, kmask = src_key_padding_mask[:, video_length + 1:].unsqueeze(2), src_key_padding_mask[:, 1:video_length + 1].unsqueeze(1) - attn_mask = torch.matmul(qmask.float(), kmask.float()).bool().repeat_interleave(self.nhead, dim=0) # batch-major layout for nn.MultiheadAttention 3D attn_mask + attn_mask = ( + torch.matmul(qmask.float(), kmask.float()) + .bool() + .repeat_interleave(self.nhead, dim=0) + ) # batch-major layout for nn.MultiheadAttention 3D attn_mask # print(attn_mask.shape) # print(attn_mask[0][0]) # print(q.shape) 23 32 256 @@ -1214,4 +1222,4 @@ def _get_activation_fn(activation): return nn.PReLU() if activation == "selu": return F.selu - raise RuntimeError(F"activation should be relu/gelu, not {activation}.") \ No newline at end of file + raise RuntimeError(F"activation should be relu/gelu, not {activation}.") diff --git a/lighthouse/common/cg_detr_transformer.py b/lighthouse/common/cg_detr_transformer.py index 1a9bc2b..343e2e8 100644 --- a/lighthouse/common/cg_detr_transformer.py +++ b/lighthouse/common/cg_detr_transformer.py @@ -576,7 +576,11 @@ def forward_post(self, qmask, kmask = src_key_padding_mask[:, :video_length].unsqueeze(2), src_key_padding_mask[:, video_length:].unsqueeze(1) - attn_mask = torch.matmul(qmask.float(), kmask.float()).bool().repeat_interleave(self.nhead, dim=0) # batch-major layout for nn.MultiheadAttention 3D attn_mask + attn_mask = ( + torch.matmul(qmask.float(), kmask.float()) + .bool() + .repeat_interleave(self.nhead, dim=0) + ) # batch-major layout for nn.MultiheadAttention 3D attn_mask # - key_padding_mask: :math:`(S)` or :math:`(N, S)` where N is the batch size, S is the source sequence length. # If a FloatTensor is provided, it will be directly added to the value. diff --git a/lighthouse/common/qd_detr_transformer.py b/lighthouse/common/qd_detr_transformer.py index 3697ad1..08ef3fd 100755 --- a/lighthouse/common/qd_detr_transformer.py +++ b/lighthouse/common/qd_detr_transformer.py @@ -476,8 +476,11 @@ def forward_post(self, # print(src_key_padding_mask[:, 76:].shape) # torch.Size([32, 26]) qmask, kmask = src_key_padding_mask[:, 1:video_length + 1].unsqueeze(2), src_key_padding_mask[:, video_length + 1:].unsqueeze(1) - attn_mask = torch.matmul(qmask.float(), kmask.float()).bool().repeat_interleave(self.nhead, dim=0) # batch-major layout for nn.MultiheadAttention 3D attn_mask - # print(attn_mask.shape) + attn_mask = ( + torch.matmul(qmask.float(), kmask.float()) + .bool() + .repeat_interleave(self.nhead, dim=0) + ) # batch-major layout for nn.MultiheadAttention 3D attn_mask # print(attn_mask[0][0]) # print(q.shape) 75 32 256 # print(k.shape) 26 32 256 diff --git a/lighthouse/common/taskweave_transformer.py b/lighthouse/common/taskweave_transformer.py index 87369b2..3dfd3ea 100644 --- a/lighthouse/common/taskweave_transformer.py +++ b/lighthouse/common/taskweave_transformer.py @@ -591,7 +591,11 @@ def forward_post(self, q, k, v = pos_src[:video_length], pos_src[video_length:], src[video_length:] qmask, kmask = src_key_padding_mask[:, :video_length].unsqueeze(2), src_key_padding_mask[:, video_length :].unsqueeze(1) - attn_mask = torch.matmul(qmask.float(), kmask.float()).bool().repeat_interleave(self.nhead, dim=0) # batch-major layout for nn.MultiheadAttention 3D attn_mask + attn_mask = ( + torch.matmul(qmask.float(), kmask.float()) + .bool() + .repeat_interleave(self.nhead, dim=0) + ) # batch-major layout for nn.MultiheadAttention 3D attn_mask src2 = self.self_attn(q, k, value=v, attn_mask=attn_mask, key_padding_mask=src_key_padding_mask[:, video_length:])[0] @@ -844,4 +848,4 @@ def _get_activation_fn(activation): return nn.PReLU() if activation == "selu": return F.selu - raise RuntimeError(F"activation should be relu/gelu, not {activation}.") \ No newline at end of file + raise RuntimeError(F"activation should be relu/gelu, not {activation}.") diff --git a/lighthouse/common/tr_detr_transformer.py b/lighthouse/common/tr_detr_transformer.py index d3d8b58..985502b 100644 --- a/lighthouse/common/tr_detr_transformer.py +++ b/lighthouse/common/tr_detr_transformer.py @@ -434,7 +434,11 @@ def forward_post(self, # print(src_key_padding_mask[:, 76:].shape) # torch.Size([32, 26]) # print(src_key_padding_mask[:, 1:video_length+1].shape) # torch.Size([32, 97]) 切片会保留维度torch.Size([32, 75])，只有坐标便不会 qmask, kmask = src_key_padding_mask[:, :video_length].unsqueeze(2), src_key_padding_mask[:, video_length:].unsqueeze(1) - attn_mask = torch.matmul(qmask.float(), kmask.float()).bool().repeat_interleave(self.nhead, dim=0) # batch-major layout for nn.MultiheadAttention 3D attn_mask + attn_mask = ( + torch.matmul(qmask.float(), kmask.float()) + .bool() + .repeat_interleave(self.nhead, dim=0) + ) # batch-major layout for nn.MultiheadAttention 3D attn_mask # print(attn_mask.shape) # print(attn_mask[0][0]) # print(q.shape) 75 32 256 @@ -525,7 +529,11 @@ def forward_post(self, # print(src_key_padding_mask[:, 76:].shape) # torch.Size([32, 26]) qmask, kmask = src_key_padding_mask[:, 1:video_length + 1].unsqueeze(2), src_key_padding_mask[:, video_length + 1:].unsqueeze(1) - attn_mask = torch.matmul(qmask.float(), kmask.float()).bool().repeat_interleave(self.nhead, dim=0) # batch-major layout for nn.MultiheadAttention 3D attn_mask + attn_mask = ( + torch.matmul(qmask.float(), kmask.float()) + .bool() + .repeat_interleave(self.nhead, dim=0) + ) # batch-major layout for nn.MultiheadAttention 3D attn_mask # print(attn_mask.shape) # print(attn_mask[0][0]) # print(q.shape) 75 32 256 From 6516b89d939bd6a04b0898e7e24e3445658f5eda Mon Sep 17 00:00:00 2001 From: h-munakata Date: Thu, 17 Sep 2026 16:15:02 +0900 Subject: [PATCH 3/3] Fix version of ruff --- .github/workflows/mypy_ruff.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/mypy_ruff.yml b/.github/workflows/mypy_ruff.yml index f7e423f..2c8dee4 100644 --- a/.github/workflows/mypy_ruff.yml +++ b/.github/workflows/mypy_ruff.yml @@ -23,7 +23,7 @@ jobs: run: | pip install -e . pip install torch==2.1.0 torchvision==0.16.0 torchaudio==2.1.0 - pip install mypy ruff + pip install mypy ruff==0.14.14 - name: Run mypy run: find lighthouse -type f -name "*.py" -not -path 'lighthouse/common/*' | xargs mypy