Release GasBench 0.11.2

pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "gasbench"
-version = "0.11.1"
+version = "0.11.2"
 description = "GASBench - ML model benchmark evaluation package"
 readme = "README.md"
 requires-python = ">=3.10"

src/gasbench/dataset/config.py

@@ -74,9 +74,11 @@ class BenchmarkDatasetConfig:
     # When set, gasbench downloads parquet files one at a time (sequentially)
     # and stops as soon as enough filtered samples have been collected,
     # avoiding unnecessary downloads from large datasets.
-    # Example: filter_column: "label", filter_value: "fake"
+    # Values retain their YAML scalar type for comparison with parquet columns.
+    # Examples: filter_column: "label", filter_value: "fake" or 0
     filter_column: Optional[str] = None
-    filter_value: Optional[str] = None
+    filter_value: Optional[Union[str, int, float, bool]] = None
+    filter_values: Optional[Dict[str, Union[str, int, float, bool]]] = None
 
     # For holdout datasets: stores the original name before obfuscation
     original_name: Optional[str] = None
@@ -356,6 +358,14 @@ def validate_dataset_config(
                 or not prefixes or any(not isinstance(p, str) or not p for p in prefixes)):
             errors.append(f"Dataset '{dataset_name}': s3_prefixes requires S3 and nonempty key prefixes")
 
+    filter_values = config_dict.get("filter_values")
+    if filter_values is not None and (
+        not isinstance(filter_values, dict)
+        or not filter_values
+        or any(not isinstance(key, str) or not key or value is None for key, value in filter_values.items())
+    ):
+        errors.append(f"Dataset '{dataset_name}': filter_values must map nonempty column names to values")
+
     numeric_fields = [
         "media_per_archive",
         "archives_per_dataset",
@@ -405,6 +415,7 @@ def _dataset_dict_to_config(d: dict, **overrides) -> BenchmarkDatasetConfig:
         "notes": d.get("notes"),
         "filter_column": d.get("filter_column"),
         "filter_value": d.get("filter_value"),
+        "filter_values": d.get("filter_values"),
         "generator_family": d.get("generator_family"),
         "content_category": d.get("content_category"),
     }

src/gasbench/dataset/configs/real_audio.yaml

@@ -4,6 +4,19 @@ datasets:
 
   # -- diverse --
 
+  - name: arabic-deepfake-real
+    path: DeepFake-Audio-Rangers/Arabic_Audio_Deepfake
+    modality: audio
+    media_type: real
+    source_format: parquet
+    media_per_archive: 250
+    archives_per_dataset: 4
+    filter_column: label
+    filter_value: 1
+    notes: Arabic language deepfake detection — real rows only (label 1)
+    generator_family: real
+    content_category: diverse
+
   - name: cfad-real
     path: 34data/cfad-real
     modality: audio
@@ -883,3 +896,60 @@ datasets:
     archives_per_dataset: -1
     notes: 'Released from v24 bmcore holdouts; exact local subset. '
     hf_revision: 0699bb1d85463a1cb5928db83bc61bdd127b4590
+
+  # Released v26 holdouts — pinned upstream HF sources.
+  - name: galgame-speech-asr-16khz
+    path: litagin/Galgame_Speech_ASR_16kHz
+    hf_revision: 3fb86654222b3f0af0f7c332ae6a0ef9752a9451
+    media_per_archive: 1000
+    archives_per_dataset: 1
+    source_format: tar
+    include_paths:
+    - data/galgame-speech-asr-16kHz-train-
+    modality: audio
+    media_type: real
+  - name: sada2022-arabic-tts
+    path: mohamedalgmaal/sada2022-arabic-tts
+    hf_revision: 10870c1a64d17a24a791ee08e8297f216f96a447
+    media_per_archive: 1000
+    archives_per_dataset: 1
+    source_format: wav
+    include_paths:
+    - batch_
+    modality: audio
+    media_type: real
+  - name: starrail-voice
+    path: simon3000/starrail-voice
+    hf_revision: 1e5f1e8658ab2e618c4815cf64ddf905c361067d
+    media_per_archive: 1000
+    archives_per_dataset: 1
+    source_format: parquet
+    include_paths:
+    - data/train-
+    data_columns:
+    - audio
+    modality: audio
+    media_type: real
+  - name: worldspeech
+    path: disco-eth/WorldSpeech
+    hf_revision: b76b85395039f50332fbc421f7c098ee2f2653d7
+    media_per_archive: 1000
+    archives_per_dataset: 1
+    source_format: parquet
+    include_paths:
+    - data/af_za/test-
+    - data/am_et/test-
+    - data/ar_bh/test-
+    - data/ar_dz/test-
+    - data/ar_eg/test-
+    - data/ar_iq/test-
+    - data/ar_kw/test-
+    - data/ar_ma/test-
+    - data/ar_sa/test-
+    - data/ar_tn/test-
+    - data/ar_un/test-
+    - data/az_az/test-
+    data_columns:
+    - audio
+    modality: audio
+    media_type: real

src/gasbench/dataset/configs/real_images.yaml

@@ -1288,3 +1288,86 @@ datasets:
     archives_per_dataset: -1
     notes: 'Released from v24 bmcore holdouts; exact local subset. '
     hf_revision: a3cf2557e18a832c4b2921c8d20948202eab54f5
+
+  # Released v26 holdouts — pinned upstream HF sources.
+  - name: beans-test
+    path: AI-Lab-Makerere/beans
+    hf_revision: 27aa014ce09b193e1a6f58112d4a66e0eddb69c5
+    media_per_archive: 1000
+    archives_per_dataset: 1
+    source_format: parquet
+    include_paths:
+    - data/test-
+    data_columns:
+    - image
+    modality: image
+    media_type: real
+  - name: beans-train
+    path: AI-Lab-Makerere/beans
+    hf_revision: 27aa014ce09b193e1a6f58112d4a66e0eddb69c5
+    media_per_archive: 1000
+    archives_per_dataset: 1
+    source_format: parquet
+    include_paths:
+    - data/train-
+    data_columns:
+    - image
+    modality: image
+    media_type: real
+  - name: beans-validation
+    path: AI-Lab-Makerere/beans
+    hf_revision: 27aa014ce09b193e1a6f58112d4a66e0eddb69c5
+    media_per_archive: 1000
+    archives_per_dataset: 1
+    source_format: parquet
+    include_paths:
+    - data/validation-
+    data_columns:
+    - image
+    modality: image
+    media_type: real
+  - name: oxford-iiit-pet-test
+    path: timm/oxford-iiit-pet
+    hf_revision: 089695c834a7deb60505b7cc506672db1c31a6aa
+    media_per_archive: 1000
+    archives_per_dataset: 1
+    source_format: parquet
+    include_paths:
+    - data/test-
+    data_columns:
+    - image
+    modality: image
+    media_type: real
+  - name: oxford-iiit-pet-train
+    path: timm/oxford-iiit-pet
+    hf_revision: 089695c834a7deb60505b7cc506672db1c31a6aa
+    media_per_archive: 1000
+    archives_per_dataset: 1
+    source_format: parquet
+    include_paths:
+    - data/train-
+    data_columns:
+    - image
+    modality: image
+    media_type: real
+  - name: dcvlm-pool-large-dvqa
+    path: mlfoundations/dcvlm_pool_large
+    hf_revision: 479162ae38203029e06807038cd2e5e545bfd2e0
+    media_per_archive: 1000
+    archives_per_dataset: 1
+    source_format: tar
+    include_paths:
+    - dvqa/
+    modality: image
+    media_type: real
+  - name: puffin-16m
+    path: ACERobotics/Puffin-16M
+    hf_revision: 2112b3502f9ea0cc1d31d2c5a2b7f2259b52cffd
+    media_per_archive: 1000
+    archives_per_dataset: 1
+    source_format: zip
+    include_paths:
+    - Puffin-Cam-15M/
+    - Puffin-Traj-1M/
+    modality: image
+    media_type: real

src/gasbench/dataset/configs/real_videos.yaml

@@ -1339,3 +1339,155 @@ datasets:
     archives_per_dataset: -1
     notes: 'Released from v24 bmcore holdouts; exact local subset. '
     hf_revision: f0af08aece50ff99a9c399f063d71e455d610bbf
+
+  # Released v26 holdouts — pinned upstream HF sources.
+  - name: physical-interaction-bowling-00000
+    path: nvidia/PhysicalAI-WorldModel-Synthetic-Physical-Interaction-Scenes
+    hf_revision: bdafca2a9c575629e47509f547e27ab7f3680369
+    media_per_archive: 250
+    archives_per_dataset: 1
+    source_format: tar
+    include_paths:
+    - videos/bowling/videos-bowling-00000.tar
+    archive_include_paths:
+    - Bowler.mp4
+    - Overhead.mp4
+    - Pins.mp4
+    - Side.mp4
+    modality: video
+    media_type: real
+  - name: physical-interaction-bowling-00042
+    path: nvidia/PhysicalAI-WorldModel-Synthetic-Physical-Interaction-Scenes
+    hf_revision: bdafca2a9c575629e47509f547e27ab7f3680369
+    media_per_archive: 250
+    archives_per_dataset: 1
+    source_format: tar
+    include_paths:
+    - videos/bowling/videos-bowling-00042.tar
+    archive_include_paths:
+    - Bowler.mp4
+    - Overhead.mp4
+    - Pins.mp4
+    - Side.mp4
+    modality: video
+    media_type: real
+  - name: physical-interaction-bowling-00063
+    path: nvidia/PhysicalAI-WorldModel-Synthetic-Physical-Interaction-Scenes
+    hf_revision: bdafca2a9c575629e47509f547e27ab7f3680369
+    media_per_archive: 250
+    archives_per_dataset: 1
+    source_format: tar
+    include_paths:
+    - videos/bowling/videos-bowling-00063.tar
+    archive_include_paths:
+    - Bowler.mp4
+    - Overhead.mp4
+    - Pins.mp4
+    - Side.mp4
+    modality: video
+    media_type: real
+  - name: physical-interaction-bowling-00084
+    path: nvidia/PhysicalAI-WorldModel-Synthetic-Physical-Interaction-Scenes
+    hf_revision: bdafca2a9c575629e47509f547e27ab7f3680369
+    media_per_archive: 250
+    archives_per_dataset: 1
+    source_format: tar
+    include_paths:
+    - videos/bowling/videos-bowling-00084.tar
+    archive_include_paths:
+    - Bowler.mp4
+    - Overhead.mp4
+    - Pins.mp4
+    - Side.mp4
+    modality: video
+    media_type: real
+  - name: physicalai-warehouse-forklift-nearmiss-00005
+    path: nvidia/PhysicalAI-WorldModel-Synthetic-Warehouse-Operations-Scenes
+    hf_revision: d5b88d3abcf659f304a107f4336b71b4e2159133
+    media_per_archive: 250
+    archives_per_dataset: 1
+    source_format: tar
+    include_paths:
+    - artifacts/forklift_human_nearmiss/nearmiss-artifacts-00005.tar
+    archive_include_paths:
+    - .rgb.mp4
+    modality: video
+    media_type: real
+  - name: physicalai-warehouse-forklift-nearmiss-00016
+    path: nvidia/PhysicalAI-WorldModel-Synthetic-Warehouse-Operations-Scenes
+    hf_revision: d5b88d3abcf659f304a107f4336b71b4e2159133
+    media_per_archive: 250
+    archives_per_dataset: 1
+    source_format: tar
+    include_paths:
+    - artifacts/forklift_human_nearmiss/nearmiss-artifacts-00016.tar
+    archive_include_paths:
+    - .rgb.mp4
+    modality: video
+    media_type: real
+  - name: physicalai-warehouse-forklift-nearmiss-00028
+    path: nvidia/PhysicalAI-WorldModel-Synthetic-Warehouse-Operations-Scenes
+    hf_revision: d5b88d3abcf659f304a107f4336b71b4e2159133
+    media_per_archive: 250
+    archives_per_dataset: 1
+    source_format: tar
+    include_paths:
+    - artifacts/forklift_human_nearmiss/nearmiss-artifacts-00028.tar
+    archive_include_paths:
+    - .rgb.mp4
+    modality: video
+    media_type: real
+  - name: physicalai-warehouse-forklift-nearmiss-00039
+    path: nvidia/PhysicalAI-WorldModel-Synthetic-Warehouse-Operations-Scenes
+    hf_revision: d5b88d3abcf659f304a107f4336b71b4e2159133
+    media_per_archive: 250
+    archives_per_dataset: 1
+    source_format: tar
+    include_paths:
+    - artifacts/forklift_human_nearmiss/nearmiss-artifacts-00039.tar
+    archive_include_paths:
+    - .rgb.mp4
+    modality: video
+    media_type: real
+  - name: physicalai-warehouse-forklift-nearmiss-00050
+    path: nvidia/PhysicalAI-WorldModel-Synthetic-Warehouse-Operations-Scenes
+    hf_revision: d5b88d3abcf659f304a107f4336b71b4e2159133
+    media_per_archive: 250
+    archives_per_dataset: 1
+    source_format: tar
+    include_paths:
+    - artifacts/forklift_human_nearmiss/nearmiss-artifacts-00050.tar
+    archive_include_paths:
+    - .rgb.mp4
+    modality: video
+    media_type: real
+  - name: physicalai-robotics-manipulation-objects
+    path: nvidia/PhysicalAI-Robotics-Manipulation-Objects
+    hf_revision: 538b6833148b4731e4ea30fe181cc4eecfc13440
+    media_per_archive: 250
+    archives_per_dataset: 1
+    source_format: mp4
+    include_paths:
+    - .rgb/
+    modality: video
+    media_type: real
+  - name: pick-tblock-mp-centered-0701-rendered
+    path: younghyopark/pick_tblock_mp_centered_0701_rendered
+    hf_revision: b353fb987b6cc55524fb144ea61632168bc2ca5f
+    media_per_archive: 250
+    archives_per_dataset: 1
+    source_format: mp4
+    include_paths:
+    - videos/chunk-000/
+    modality: video
+    media_type: real
+  - name: vidgen-1m
+    path: Fudan-FUXI/VIDGEN-1M
+    hf_revision: 9158f218b6b81694a7809b6ee74e30003d58a70e
+    media_per_archive: 250
+    archives_per_dataset: 1
+    source_format: zip
+    include_paths:
+    - VidGen_video_
+    modality: video
+    media_type: real