Bug/castella

.github/workflows/mypy_ruff.yml

@@ -23,7 +23,7 @@ jobs:
         run: |
           pip install -e .
           pip install torch==2.1.0 torchvision==0.16.0 torchaudio==2.1.0
-          pip install mypy ruff
+          pip install mypy ruff==0.14.14
 
       - name: Run mypy
         run: find lighthouse -type f -name "*.py" -not -path 'lighthouse/common/*' | xargs mypy

data/castella/castella_test_release.jsonl

@@ -666,7 +666,7 @@
 {"qid": "RfGapDlAYoQ_3", "query": "A whistle blows as cheers erupt", "duration": 150, "vid": "RfGapDlAYoQ", "relevant_windows": [[113, 117]]}
 {"qid": "Rk-ujWKzPuc_1", "query": "A man speaks catching his breath", "duration": 300, "vid": "Rk-ujWKzPuc", "relevant_windows": [[84, 98], [137, 143], [181, 199], [203, 206], [237, 245], [247, 258], [290, 300]]}
 {"qid": "Rk-ujWKzPuc_2", "query": "Wind blows as water flows", "duration": 300, "vid": "Rk-ujWKzPuc", "relevant_windows": [[261, 290], [57, 83], [103, 113]]}
-{"qid": "Rp4Ct_TQvAM_1", "query": "A man talks with background noise", "duration": 299, "vid": "Rp4Ct_TQvAM", "relevant_windows": [[0, 10], [15, 34], [49, 56], [65, 78], [89, 96], [104, 128], [141, 173], [184, 191], [233, 238], [265, 291], [298, 301]]}
+{"qid": "Rp4Ct_TQvAM_1", "query": "A man talks with background noise", "duration": 299, "vid": "Rp4Ct_TQvAM", "relevant_windows": [[0, 10], [15, 34], [49, 56], [65, 78], [89, 96], [104, 128], [141, 173], [184, 191], [233, 238], [265, 291], [298, 300]]}
 {"qid": "Rp4Ct_TQvAM_2", "query": "A man talks and a door closes with background noise", "duration": 299, "vid": "Rp4Ct_TQvAM", "relevant_windows": [[236, 238]]}
 {"qid": "Rp4Ct_TQvAM_3", "query": "A train passes by with background noise", "duration": 299, "vid": "Rp4Ct_TQvAM", "relevant_windows": [[245, 260]]}
 {"qid": "RrmBGjJqlEo_1", "query": "Someone sniffles", "duration": 300, "vid": "RrmBGjJqlEo", "relevant_windows": [[77, 79], [109, 111], [121, 122], [186, 187], [245, 246], [297, 298]]}

data/castella/castella_train_release.jsonl

@@ -653,7 +653,7 @@
 {"qid": "66Sl9ixB_5U_2", "query": "Many people are talking and birds are chirping", "duration": 300, "vid": "66Sl9ixB_5U", "relevant_windows": [[134, 180]]}
 {"qid": "674mDF2k6Jw_1", "query": "While two men are having a conversation, someone keeps banging on something", "duration": 299, "vid": "674mDF2k6Jw", "relevant_windows": [[19, 39]]}
 {"qid": "674mDF2k6Jw_2", "query": "Someone fires a gun", "duration": 299, "vid": "674mDF2k6Jw", "relevant_windows": [[79, 81], [103, 104], [88, 89]]}
-{"qid": "67AE9ZKvECs_1", "query": "Birds chirp while something makes a scraping sound", "duration": 299, "vid": "67AE9ZKvECs", "relevant_windows": [[206, 209], [221, 223], [283, 285], [299, 4800], [255, 258], [266, 270]]}
+{"qid": "67AE9ZKvECs_1", "query": "Birds chirp while something makes a scraping sound", "duration": 299, "vid": "67AE9ZKvECs", "relevant_windows": [[206, 209], [221, 223], [283, 285], [299, 300], [255, 258], [266, 270]]}
 {"qid": "67AE9ZKvECs_2", "query": "A bird is chirping while electronic sounds are happening", "duration": 299, "vid": "67AE9ZKvECs", "relevant_windows": [[232, 235], [268, 270], [291, 294]]}
 {"qid": "67AE9ZKvECs_3", "query": "A bird is chirping while an insect is flying", "duration": 299, "vid": "67AE9ZKvECs", "relevant_windows": [[248, 252], [292, 294]]}
 {"qid": "67qTV1MDXhc_1", "query": "A dog barks as the wind blows and the crowd murmurs", "duration": 241, "vid": "67qTV1MDXhc", "relevant_windows": [[160, 161], [166, 169]]}

lighthouse/common/CIM.py

@@ -700,7 +700,11 @@ def forward_post(self,
         # print(src_key_padding_mask[:, 76:].shape) # torch.Size([32, 26])
 
         qmask, kmask = src_key_padding_mask[:, 1:video_length + 1].unsqueeze(2), src_key_padding_mask[:, video_length + 1:].unsqueeze(1)
-        attn_mask = torch.matmul(qmask.float(), kmask.float()).bool().repeat_interleave(self.nhead, dim=0)  # batch-major layout for nn.MultiheadAttention 3D attn_mask
+        attn_mask = (
+                    torch.matmul(qmask.float(), kmask.float())
+                    .bool()
+                    .repeat_interleave(self.nhead, dim=0)
+                )  # batch-major layout for nn.MultiheadAttention 3D attn_mask
         # print(attn_mask.shape)
         # print(attn_mask[0][0])
         # print(q.shape) 75 32 256
@@ -789,7 +793,11 @@ def forward_post(self,
         global_token, q, k, v = src[0].unsqueeze(0), pos_src[video_length + 1: ], pos_src[1:video_length + 1], src[1:video_length + 1]
 
         qmask, kmask = src_key_padding_mask[:, video_length + 1:].unsqueeze(2), src_key_padding_mask[:, 1:video_length + 1].unsqueeze(1)
-        attn_mask = torch.matmul(qmask.float(), kmask.float()).bool().repeat_interleave(self.nhead, dim=0)  # batch-major layout for nn.MultiheadAttention 3D attn_mask
+        attn_mask = (
+                    torch.matmul(qmask.float(), kmask.float())
+                    .bool()
+                    .repeat_interleave(self.nhead, dim=0)
+                )  # batch-major layout for nn.MultiheadAttention 3D attn_mask
         # print(attn_mask.shape)
         # print(attn_mask[0][0])
         # print(q.shape) 23 32 256
@@ -1214,4 +1222,4 @@ def _get_activation_fn(activation):
         return nn.PReLU()
     if activation == "selu":
         return F.selu
-    raise RuntimeError(F"activation should be relu/gelu, not {activation}.")
+    raise RuntimeError(F"activation should be relu/gelu, not {activation}.")

lighthouse/common/cg_detr_transformer.py

@@ -576,7 +576,11 @@ def forward_post(self,
 
         qmask, kmask = src_key_padding_mask[:, :video_length].unsqueeze(2), src_key_padding_mask[:,
                                                                                  video_length:].unsqueeze(1)
-        attn_mask = torch.matmul(qmask.float(), kmask.float()).bool().repeat_interleave(self.nhead, dim=0)  # batch-major layout for nn.MultiheadAttention 3D attn_mask
+        attn_mask = (
+                    torch.matmul(qmask.float(), kmask.float())
+                    .bool()
+                    .repeat_interleave(self.nhead, dim=0)
+                )  # batch-major layout for nn.MultiheadAttention 3D attn_mask
 
         # - key_padding_mask: :math:`(S)` or :math:`(N, S)` where N is the batch size, S is the source sequence length.
         #   If a FloatTensor is provided, it will be directly added to the value.

lighthouse/common/qd_detr_transformer.py

@@ -476,8 +476,11 @@ def forward_post(self,
         # print(src_key_padding_mask[:, 76:].shape) # torch.Size([32, 26])
 
         qmask, kmask = src_key_padding_mask[:, 1:video_length + 1].unsqueeze(2), src_key_padding_mask[:, video_length + 1:].unsqueeze(1)
-        attn_mask = torch.matmul(qmask.float(), kmask.float()).bool().repeat_interleave(self.nhead, dim=0)  # batch-major layout for nn.MultiheadAttention 3D attn_mask
-        # print(attn_mask.shape)
+        attn_mask = (
+                    torch.matmul(qmask.float(), kmask.float())
+                    .bool()
+                    .repeat_interleave(self.nhead, dim=0)
+                )  # batch-major layout for nn.MultiheadAttention 3D attn_mask
         # print(attn_mask[0][0])
         # print(q.shape) 75 32 256
         # print(k.shape) 26 32 256

lighthouse/common/taskweave_transformer.py

@@ -591,7 +591,11 @@ def forward_post(self,
         q, k, v = pos_src[:video_length], pos_src[video_length:], src[video_length:]
 
         qmask, kmask = src_key_padding_mask[:, :video_length].unsqueeze(2), src_key_padding_mask[:, video_length :].unsqueeze(1)
-        attn_mask = torch.matmul(qmask.float(), kmask.float()).bool().repeat_interleave(self.nhead, dim=0)  # batch-major layout for nn.MultiheadAttention 3D attn_mask
+        attn_mask = (
+                    torch.matmul(qmask.float(), kmask.float())
+                    .bool()
+                    .repeat_interleave(self.nhead, dim=0)
+                )  # batch-major layout for nn.MultiheadAttention 3D attn_mask
 
         src2 = self.self_attn(q, k, value=v, attn_mask=attn_mask,
                               key_padding_mask=src_key_padding_mask[:, video_length:])[0]
@@ -844,4 +848,4 @@ def _get_activation_fn(activation):
         return nn.PReLU()
     if activation == "selu":
         return F.selu
-    raise RuntimeError(F"activation should be relu/gelu, not {activation}.")
+    raise RuntimeError(F"activation should be relu/gelu, not {activation}.")

lighthouse/common/tr_detr_transformer.py

@@ -434,7 +434,11 @@ def forward_post(self,
         # print(src_key_padding_mask[:, 76:].shape) # torch.Size([32, 26])
         # print(src_key_padding_mask[:, 1:video_length+1].shape) # torch.Size([32, 97])  切片会保留维度torch.Size([32, 75])，只有坐标便不会
         qmask, kmask = src_key_padding_mask[:, :video_length].unsqueeze(2), src_key_padding_mask[:, video_length:].unsqueeze(1)
-        attn_mask = torch.matmul(qmask.float(), kmask.float()).bool().repeat_interleave(self.nhead, dim=0)  # batch-major layout for nn.MultiheadAttention 3D attn_mask
+        attn_mask = (
+                    torch.matmul(qmask.float(), kmask.float())
+                    .bool()
+                    .repeat_interleave(self.nhead, dim=0)
+                )  # batch-major layout for nn.MultiheadAttention 3D attn_mask
         # print(attn_mask.shape)
         # print(attn_mask[0][0])
         # print(q.shape) 75 32 256
@@ -525,7 +529,11 @@ def forward_post(self,
         # print(src_key_padding_mask[:, 76:].shape) # torch.Size([32, 26])
 
         qmask, kmask = src_key_padding_mask[:, 1:video_length + 1].unsqueeze(2), src_key_padding_mask[:, video_length + 1:].unsqueeze(1)
-        attn_mask = torch.matmul(qmask.float(), kmask.float()).bool().repeat_interleave(self.nhead, dim=0)  # batch-major layout for nn.MultiheadAttention 3D attn_mask
+        attn_mask = (
+                    torch.matmul(qmask.float(), kmask.float())
+                    .bool()
+                    .repeat_interleave(self.nhead, dim=0)
+                )  # batch-major layout for nn.MultiheadAttention 3D attn_mask
         # print(attn_mask.shape)
         # print(attn_mask[0][0])
         # print(q.shape) 75 32 256