From 8e3b2e6168900bffac17f20f517d22143e1d024a Mon Sep 17 00:00:00 2001 From: dylan Date: Mon, 5 Oct 2026 10:37:07 -0500 Subject: [PATCH 1/5] Fix Arabic audio and Senorita dataset label contamination --- src/gasbench/dataset/config.py | 5 +- src/gasbench/dataset/configs/real_audio.yaml | 13 ++++ .../dataset/configs/synthetic_audio.yaml | 4 +- .../dataset/configs/synthetic_videos.yaml | 6 ++ src/gasbench/dataset/download/core.py | 2 +- src/gasbench/dataset/download/extract.py | 2 +- tests/integration/test_parquet_processing.py | 64 +++++++++++++++++-- 7 files changed, 87 insertions(+), 9 deletions(-) diff --git a/src/gasbench/dataset/config.py b/src/gasbench/dataset/config.py index 52be1ca..84c197e 100644 --- a/src/gasbench/dataset/config.py +++ b/src/gasbench/dataset/config.py @@ -74,9 +74,10 @@ class BenchmarkDatasetConfig: # When set, gasbench downloads parquet files one at a time (sequentially) # and stops as soon as enough filtered samples have been collected, # avoiding unnecessary downloads from large datasets. - # Example: filter_column: "label", filter_value: "fake" + # Values retain their YAML scalar type for comparison with parquet columns. + # Examples: filter_column: "label", filter_value: "fake" or 0 filter_column: Optional[str] = None - filter_value: Optional[str] = None + filter_value: Optional[Union[str, int, float, bool]] = None # For holdout datasets: stores the original name before obfuscation original_name: Optional[str] = None diff --git a/src/gasbench/dataset/configs/real_audio.yaml b/src/gasbench/dataset/configs/real_audio.yaml index ad0ea4b..4d6f44f 100644 --- a/src/gasbench/dataset/configs/real_audio.yaml +++ b/src/gasbench/dataset/configs/real_audio.yaml @@ -4,6 +4,19 @@ datasets: # -- diverse -- + - name: arabic-deepfake-real + path: DeepFake-Audio-Rangers/Arabic_Audio_Deepfake + modality: audio + media_type: real + source_format: parquet + media_per_archive: 250 + archives_per_dataset: 4 + filter_column: label + filter_value: 1 + notes: Arabic language deepfake detection — real rows only (label 1) + generator_family: real + content_category: diverse + - name: cfad-real path: 34data/cfad-real modality: audio diff --git a/src/gasbench/dataset/configs/synthetic_audio.yaml b/src/gasbench/dataset/configs/synthetic_audio.yaml index 12c2950..ae8c069 100644 --- a/src/gasbench/dataset/configs/synthetic_audio.yaml +++ b/src/gasbench/dataset/configs/synthetic_audio.yaml @@ -22,7 +22,9 @@ datasets: source_format: parquet media_per_archive: 250 archives_per_dataset: 4 - notes: Arabic language deepfake audio + filter_column: label + filter_value: 0 + notes: Arabic language deepfake audio — fake rows only (label 0) generator_family: mixed-generation content_category: diverse diff --git a/src/gasbench/dataset/configs/synthetic_videos.yaml b/src/gasbench/dataset/configs/synthetic_videos.yaml index c2ee23e..964a887 100644 --- a/src/gasbench/dataset/configs/synthetic_videos.yaml +++ b/src/gasbench/dataset/configs/synthetic_videos.yaml @@ -230,6 +230,7 @@ datasets: media_per_archive: 250 archives_per_dataset: 2 include_paths: [controlable_videos_upload] + archive_exclude_paths: ["_org"] generator_family: mixed-generation content_category: diverse @@ -241,6 +242,7 @@ datasets: media_per_archive: 250 archives_per_dataset: 2 include_paths: [inpainting_upload] + archive_exclude_paths: ["_org"] generator_family: mixed-generation content_category: diverse @@ -252,6 +254,7 @@ datasets: media_per_archive: 250 archives_per_dataset: 2 include_paths: [local_style_transfer_upload] + archive_exclude_paths: ["_org"] generator_family: mixed-generation content_category: diverse @@ -263,6 +266,7 @@ datasets: media_per_archive: 250 archives_per_dataset: 2 include_paths: [obj_removal_videos_upload, obj_removal_videos_upload2] + archive_exclude_paths: ["_org"] generator_family: mixed-generation content_category: diverse @@ -274,6 +278,7 @@ datasets: media_per_archive: 250 archives_per_dataset: 2 include_paths: [obj_swap_upload] + archive_exclude_paths: ["_org"] generator_family: mixed-generation content_category: diverse @@ -285,6 +290,7 @@ datasets: media_per_archive: 250 archives_per_dataset: 2 include_paths: [style_transfer_upload] + archive_exclude_paths: ["_org"] generator_family: mixed-generation content_category: diverse diff --git a/src/gasbench/dataset/download/core.py b/src/gasbench/dataset/download/core.py index 07eb9bd..9bc3115 100644 --- a/src/gasbench/dataset/download/core.py +++ b/src/gasbench/dataset/download/core.py @@ -145,7 +145,7 @@ def download_and_extract( # one at a time and stop as soon as the sample target is hit. filter_column = getattr(dataset, "filter_column", None) filter_value = getattr(dataset, "filter_value", None) - if filter_column and filter_value: + if filter_column and filter_value is not None: try: all_filenames = _list_remote_dataset_files( dataset.path, diff --git a/src/gasbench/dataset/download/extract.py b/src/gasbench/dataset/download/extract.py index 8f81cf0..318000e 100644 --- a/src/gasbench/dataset/download/extract.py +++ b/src/gasbench/dataset/download/extract.py @@ -426,7 +426,7 @@ def _process_parquet( filter_column = getattr(dataset, "filter_column", None) filter_value = getattr(dataset, "filter_value", None) - if filter_column and filter_value: + if filter_column and filter_value is not None: if filter_column not in df.columns: logger.warning(f"filter_column '{filter_column}' not in {source_path.name}; skipping") return diff --git a/tests/integration/test_parquet_processing.py b/tests/integration/test_parquet_processing.py index 18131f9..6054863 100644 --- a/tests/integration/test_parquet_processing.py +++ b/tests/integration/test_parquet_processing.py @@ -7,9 +7,11 @@ from PIL import Image import pytest import soundfile as sf +import yaml -from gasbench.dataset.config import BenchmarkDatasetConfig +from gasbench.dataset.config import BenchmarkDatasetConfig, load_datasets_from_yaml from gasbench.dataset.download import _process_parquet +from gasbench.dataset.download import core def png(red): @@ -88,16 +90,70 @@ def test_corrupt_media_does_not_discard_other_rows(parquet): assert [s["image"].getpixel((0, 0))[0] for s in samples] == [10, 20] -def test_row_filter_applies_before_sampling(parquet): - path = parquet([{"image": png(i), "split": "keep" if i == 5 else "drop"} for i in range(8)]) +@pytest.mark.parametrize("keep,drop", [("keep", "drop"), (0, 1), (False, True), ("", "drop")]) +def test_row_filter_applies_before_sampling(parquet, keep, drop): + path = parquet([{"image": png(i), "split": keep if i == 5 else drop} for i in range(8)]) config = BenchmarkDatasetConfig( - "filtered", "test/images", "image", "real", filter_column="split", filter_value="keep", + "filtered", "test/images", "image", "real", filter_column="split", filter_value=keep, ) samples = list(_process_parquet(path, config, num_items=1, seed=1)) assert len(samples) == 1 assert samples[0]["image"].getpixel((0, 0)) == (5, 0, 0) +@pytest.mark.parametrize("label,media_type", [(0, "synthetic"), (1, "real")]) +def test_numeric_yaml_filter_collects_only_matching_audio_until_target( + tmp_path, monkeypatch, label, media_type, +): + config_path = tmp_path / "datasets.yaml" + config_path.write_text(yaml.safe_dump({"datasets": [{ + "name": "filtered-audio", "path": "test/audio", "modality": "audio", + "media_type": media_type, "source_format": "parquet", + "filter_column": "label", "filter_value": label, + }]})) + (config,) = load_datasets_from_yaml(str(config_path))["audio"] + filenames = [f"shard-{i}.parquet" for i in range(4)] + monkeypatch.setattr(core, "_list_remote_dataset_files", lambda *a, **k: filenames) + monkeypatch.setattr(core, "_get_download_urls", lambda path, files, *a: list(files)) + downloaded = [] + + def download(paths, root, **kwargs): + for filename in paths: + path = root / filename + pd.DataFrame([ + {"audio": {"bytes": f"{filename}:{value}".encode()}, "label": value} + for value in (0, 1) + ]).to_parquet(path) + downloaded.append(filename) + yield path + + monkeypatch.setattr(core, "_stream_downloads", download) + monkeypatch.setattr(core, "download_files", lambda *a, **k: list(download(*a, **k))) + samples = list(core.download_and_extract( + config, media_per_archive=3, archives_per_dataset=1, + temp_dir=str(tmp_path), force_download=True, seed=7, + )) + + # One matching row per shard requires continuing past the archive count, + # then stopping before the fourth shard once the filtered target is met. + assert len(samples) == len(downloaded) == 3 + assert len(set(downloaded)) == 3 + assert [sample["audio_bytes"] for sample in samples] == [ + f"{filename}:{label}".encode() for filename in downloaded + ] + assert all(sample["label"] == label and sample["media_type"] == media_type for sample in samples) + + +@pytest.mark.parametrize("metadata", [{}, {"label": 1}]) +def test_zero_filter_yields_nothing_when_label_is_missing_or_unmatched(parquet, metadata): + config = BenchmarkDatasetConfig( + "filtered", "test/audio", "audio", "synthetic", + filter_column="label", filter_value=0, + ) + path = parquet([{"audio": b"excluded", **metadata}]) + assert list(_process_parquet(path, config, num_items=-1)) == [] + + def test_audio_array_preserves_sample_rate_and_waveform(parquet): waveform = np.array([0.0, 0.25, -0.5, 0.75], dtype=np.float32) path = parquet([{"audio": {"array": waveform, "sampling_rate": 22050}}]) From 4c5fc206f3d87dab962d5d4ae2972d90ef0b6a83 Mon Sep 17 00:00:00 2001 From: dylan Date: Tue, 6 Oct 2026 11:00:14 -0500 Subject: [PATCH 2/5] Exclude local style archives from global Senorita style transfer --- src/gasbench/dataset/configs/synthetic_videos.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/src/gasbench/dataset/configs/synthetic_videos.yaml b/src/gasbench/dataset/configs/synthetic_videos.yaml index 964a887..e6c5b6d 100644 --- a/src/gasbench/dataset/configs/synthetic_videos.yaml +++ b/src/gasbench/dataset/configs/synthetic_videos.yaml @@ -290,6 +290,7 @@ datasets: media_per_archive: 250 archives_per_dataset: 2 include_paths: [style_transfer_upload] + exclude_paths: [local_style_transfer_upload] archive_exclude_paths: ["_org"] generator_family: mixed-generation content_category: diverse From ed0693ca3cad7acd2b2f638b86c6934c163eecf5 Mon Sep 17 00:00:00 2001 From: dylan Date: Tue, 6 Oct 2026 11:57:10 -0500 Subject: [PATCH 3/5] Release selected v26 holdouts from upstream HF sources --- src/gasbench/dataset/config.py | 10 ++ src/gasbench/dataset/configs/real_audio.yaml | 57 +++++++ src/gasbench/dataset/configs/real_images.yaml | 83 ++++++++++ src/gasbench/dataset/configs/real_videos.yaml | 152 ++++++++++++++++++ .../dataset/configs/synthetic_audio.yaml | 86 ++++++++++ .../dataset/configs/synthetic_images.yaml | 150 +++++++++++++++++ .../dataset/configs/synthetic_videos.yaml | 39 +++++ src/gasbench/dataset/download/core.py | 7 +- src/gasbench/dataset/download/extract.py | 15 +- tests/integration/test_parquet_processing.py | 35 ++++ 10 files changed, 626 insertions(+), 8 deletions(-) diff --git a/src/gasbench/dataset/config.py b/src/gasbench/dataset/config.py index 84c197e..b7d1e07 100644 --- a/src/gasbench/dataset/config.py +++ b/src/gasbench/dataset/config.py @@ -78,6 +78,7 @@ class BenchmarkDatasetConfig: # Examples: filter_column: "label", filter_value: "fake" or 0 filter_column: Optional[str] = None filter_value: Optional[Union[str, int, float, bool]] = None + filter_values: Optional[Dict[str, Union[str, int, float, bool]]] = None # For holdout datasets: stores the original name before obfuscation original_name: Optional[str] = None @@ -357,6 +358,14 @@ def validate_dataset_config( or not prefixes or any(not isinstance(p, str) or not p for p in prefixes)): errors.append(f"Dataset '{dataset_name}': s3_prefixes requires S3 and nonempty key prefixes") + filter_values = config_dict.get("filter_values") + if filter_values is not None and ( + not isinstance(filter_values, dict) + or not filter_values + or any(not isinstance(key, str) or not key or value is None for key, value in filter_values.items()) + ): + errors.append(f"Dataset '{dataset_name}': filter_values must map nonempty column names to values") + numeric_fields = [ "media_per_archive", "archives_per_dataset", @@ -406,6 +415,7 @@ def _dataset_dict_to_config(d: dict, **overrides) -> BenchmarkDatasetConfig: "notes": d.get("notes"), "filter_column": d.get("filter_column"), "filter_value": d.get("filter_value"), + "filter_values": d.get("filter_values"), "generator_family": d.get("generator_family"), "content_category": d.get("content_category"), } diff --git a/src/gasbench/dataset/configs/real_audio.yaml b/src/gasbench/dataset/configs/real_audio.yaml index 4d6f44f..515ab60 100644 --- a/src/gasbench/dataset/configs/real_audio.yaml +++ b/src/gasbench/dataset/configs/real_audio.yaml @@ -896,3 +896,60 @@ datasets: archives_per_dataset: -1 notes: 'Released from v24 bmcore holdouts; exact local subset. ' hf_revision: 0699bb1d85463a1cb5928db83bc61bdd127b4590 + + # Released v26 holdouts — pinned upstream HF sources. + - name: v26-real-galgame-speech-asr-16khz + path: litagin/Galgame_Speech_ASR_16kHz + hf_revision: 3fb86654222b3f0af0f7c332ae6a0ef9752a9451 + media_per_archive: 1000 + archives_per_dataset: 1 + source_format: tar + include_paths: + - data/galgame-speech-asr-16kHz-train- + modality: audio + media_type: real + - name: v26-real-sada2022-arabic-tts + path: mohamedalgmaal/sada2022-arabic-tts + hf_revision: 10870c1a64d17a24a791ee08e8297f216f96a447 + media_per_archive: 1000 + archives_per_dataset: 1 + source_format: wav + include_paths: + - batch_ + modality: audio + media_type: real + - name: v26-real-starrail-voice + path: simon3000/starrail-voice + hf_revision: 1e5f1e8658ab2e618c4815cf64ddf905c361067d + media_per_archive: 1000 + archives_per_dataset: 1 + source_format: parquet + include_paths: + - data/train- + data_columns: + - audio + modality: audio + media_type: real + - name: v26-real-worldspeech + path: disco-eth/WorldSpeech + hf_revision: b76b85395039f50332fbc421f7c098ee2f2653d7 + media_per_archive: 1000 + archives_per_dataset: 1 + source_format: parquet + include_paths: + - data/af_za/test- + - data/am_et/test- + - data/ar_bh/test- + - data/ar_dz/test- + - data/ar_eg/test- + - data/ar_iq/test- + - data/ar_kw/test- + - data/ar_ma/test- + - data/ar_sa/test- + - data/ar_tn/test- + - data/ar_un/test- + - data/az_az/test- + data_columns: + - audio + modality: audio + media_type: real diff --git a/src/gasbench/dataset/configs/real_images.yaml b/src/gasbench/dataset/configs/real_images.yaml index 28c0027..a4acd69 100644 --- a/src/gasbench/dataset/configs/real_images.yaml +++ b/src/gasbench/dataset/configs/real_images.yaml @@ -1288,3 +1288,86 @@ datasets: archives_per_dataset: -1 notes: 'Released from v24 bmcore holdouts; exact local subset. ' hf_revision: a3cf2557e18a832c4b2921c8d20948202eab54f5 + + # Released v26 holdouts — pinned upstream HF sources. + - name: v25-real-beans-test + path: AI-Lab-Makerere/beans + hf_revision: 27aa014ce09b193e1a6f58112d4a66e0eddb69c5 + media_per_archive: 1000 + archives_per_dataset: 1 + source_format: parquet + include_paths: + - data/test- + data_columns: + - image + modality: image + media_type: real + - name: v25-real-beans-train + path: AI-Lab-Makerere/beans + hf_revision: 27aa014ce09b193e1a6f58112d4a66e0eddb69c5 + media_per_archive: 1000 + archives_per_dataset: 1 + source_format: parquet + include_paths: + - data/train- + data_columns: + - image + modality: image + media_type: real + - name: v25-real-beans-validation + path: AI-Lab-Makerere/beans + hf_revision: 27aa014ce09b193e1a6f58112d4a66e0eddb69c5 + media_per_archive: 1000 + archives_per_dataset: 1 + source_format: parquet + include_paths: + - data/validation- + data_columns: + - image + modality: image + media_type: real + - name: v25-real-oxford-iiit-pet-test + path: timm/oxford-iiit-pet + hf_revision: 089695c834a7deb60505b7cc506672db1c31a6aa + media_per_archive: 1000 + archives_per_dataset: 1 + source_format: parquet + include_paths: + - data/test- + data_columns: + - image + modality: image + media_type: real + - name: v25-real-oxford-iiit-pet-train + path: timm/oxford-iiit-pet + hf_revision: 089695c834a7deb60505b7cc506672db1c31a6aa + media_per_archive: 1000 + archives_per_dataset: 1 + source_format: parquet + include_paths: + - data/train- + data_columns: + - image + modality: image + media_type: real + - name: v26-real-dcvlm-pool-large-dvqa + path: mlfoundations/dcvlm_pool_large + hf_revision: 479162ae38203029e06807038cd2e5e545bfd2e0 + media_per_archive: 1000 + archives_per_dataset: 1 + source_format: tar + include_paths: + - dvqa/ + modality: image + media_type: real + - name: v26-real-puffin-16m + path: ACERobotics/Puffin-16M + hf_revision: 2112b3502f9ea0cc1d31d2c5a2b7f2259b52cffd + media_per_archive: 1000 + archives_per_dataset: 1 + source_format: zip + include_paths: + - Puffin-Cam-15M/ + - Puffin-Traj-1M/ + modality: image + media_type: real diff --git a/src/gasbench/dataset/configs/real_videos.yaml b/src/gasbench/dataset/configs/real_videos.yaml index a925063..7ed80f2 100644 --- a/src/gasbench/dataset/configs/real_videos.yaml +++ b/src/gasbench/dataset/configs/real_videos.yaml @@ -1339,3 +1339,155 @@ datasets: archives_per_dataset: -1 notes: 'Released from v24 bmcore holdouts; exact local subset. ' hf_revision: f0af08aece50ff99a9c399f063d71e455d610bbf + + # Released v26 holdouts — pinned upstream HF sources. + - name: v25-rendered-physical-interaction-bowling-00000 + path: nvidia/PhysicalAI-WorldModel-Synthetic-Physical-Interaction-Scenes + hf_revision: bdafca2a9c575629e47509f547e27ab7f3680369 + media_per_archive: 250 + archives_per_dataset: 1 + source_format: tar + include_paths: + - videos/bowling/videos-bowling-00000.tar + archive_include_paths: + - Bowler.mp4 + - Overhead.mp4 + - Pins.mp4 + - Side.mp4 + modality: video + media_type: real + - name: v25-rendered-physical-interaction-bowling-00042 + path: nvidia/PhysicalAI-WorldModel-Synthetic-Physical-Interaction-Scenes + hf_revision: bdafca2a9c575629e47509f547e27ab7f3680369 + media_per_archive: 250 + archives_per_dataset: 1 + source_format: tar + include_paths: + - videos/bowling/videos-bowling-00042.tar + archive_include_paths: + - Bowler.mp4 + - Overhead.mp4 + - Pins.mp4 + - Side.mp4 + modality: video + media_type: real + - name: v25-rendered-physical-interaction-bowling-00063 + path: nvidia/PhysicalAI-WorldModel-Synthetic-Physical-Interaction-Scenes + hf_revision: bdafca2a9c575629e47509f547e27ab7f3680369 + media_per_archive: 250 + archives_per_dataset: 1 + source_format: tar + include_paths: + - videos/bowling/videos-bowling-00063.tar + archive_include_paths: + - Bowler.mp4 + - Overhead.mp4 + - Pins.mp4 + - Side.mp4 + modality: video + media_type: real + - name: v25-rendered-physical-interaction-bowling-00084 + path: nvidia/PhysicalAI-WorldModel-Synthetic-Physical-Interaction-Scenes + hf_revision: bdafca2a9c575629e47509f547e27ab7f3680369 + media_per_archive: 250 + archives_per_dataset: 1 + source_format: tar + include_paths: + - videos/bowling/videos-bowling-00084.tar + archive_include_paths: + - Bowler.mp4 + - Overhead.mp4 + - Pins.mp4 + - Side.mp4 + modality: video + media_type: real + - name: v25-rendered-physicalai-worldmodel-synthetic-warehouse-operations-scenes-artifacts-forklift-human-nearmiss-nearmiss-artifacts-00005 + path: nvidia/PhysicalAI-WorldModel-Synthetic-Warehouse-Operations-Scenes + hf_revision: d5b88d3abcf659f304a107f4336b71b4e2159133 + media_per_archive: 250 + archives_per_dataset: 1 + source_format: tar + include_paths: + - artifacts/forklift_human_nearmiss/nearmiss-artifacts-00005.tar + archive_include_paths: + - .rgb.mp4 + modality: video + media_type: real + - name: v25-rendered-physicalai-worldmodel-synthetic-warehouse-operations-scenes-artifacts-forklift-human-nearmiss-nearmiss-artifacts-00016 + path: nvidia/PhysicalAI-WorldModel-Synthetic-Warehouse-Operations-Scenes + hf_revision: d5b88d3abcf659f304a107f4336b71b4e2159133 + media_per_archive: 250 + archives_per_dataset: 1 + source_format: tar + include_paths: + - artifacts/forklift_human_nearmiss/nearmiss-artifacts-00016.tar + archive_include_paths: + - .rgb.mp4 + modality: video + media_type: real + - name: v25-rendered-physicalai-worldmodel-synthetic-warehouse-operations-scenes-artifacts-forklift-human-nearmiss-nearmiss-artifacts-00028 + path: nvidia/PhysicalAI-WorldModel-Synthetic-Warehouse-Operations-Scenes + hf_revision: d5b88d3abcf659f304a107f4336b71b4e2159133 + media_per_archive: 250 + archives_per_dataset: 1 + source_format: tar + include_paths: + - artifacts/forklift_human_nearmiss/nearmiss-artifacts-00028.tar + archive_include_paths: + - .rgb.mp4 + modality: video + media_type: real + - name: v25-rendered-physicalai-worldmodel-synthetic-warehouse-operations-scenes-artifacts-forklift-human-nearmiss-nearmiss-artifacts-00039 + path: nvidia/PhysicalAI-WorldModel-Synthetic-Warehouse-Operations-Scenes + hf_revision: d5b88d3abcf659f304a107f4336b71b4e2159133 + media_per_archive: 250 + archives_per_dataset: 1 + source_format: tar + include_paths: + - artifacts/forklift_human_nearmiss/nearmiss-artifacts-00039.tar + archive_include_paths: + - .rgb.mp4 + modality: video + media_type: real + - name: v25-rendered-physicalai-worldmodel-synthetic-warehouse-operations-scenes-artifacts-forklift-human-nearmiss-nearmiss-artifacts-00050 + path: nvidia/PhysicalAI-WorldModel-Synthetic-Warehouse-Operations-Scenes + hf_revision: d5b88d3abcf659f304a107f4336b71b4e2159133 + media_per_archive: 250 + archives_per_dataset: 1 + source_format: tar + include_paths: + - artifacts/forklift_human_nearmiss/nearmiss-artifacts-00050.tar + archive_include_paths: + - .rgb.mp4 + modality: video + media_type: real + - name: v26-real-physicalai-robotics-manipulation-objects + path: nvidia/PhysicalAI-Robotics-Manipulation-Objects + hf_revision: 538b6833148b4731e4ea30fe181cc4eecfc13440 + media_per_archive: 250 + archives_per_dataset: 1 + source_format: mp4 + include_paths: + - .rgb/ + modality: video + media_type: real + - name: v26-real-pick-tblock-mp-centered-0701-rendered + path: younghyopark/pick_tblock_mp_centered_0701_rendered + hf_revision: b353fb987b6cc55524fb144ea61632168bc2ca5f + media_per_archive: 250 + archives_per_dataset: 1 + source_format: mp4 + include_paths: + - videos/chunk-000/ + modality: video + media_type: real + - name: v26-real-vidgen-1m + path: Fudan-FUXI/VIDGEN-1M + hf_revision: 9158f218b6b81694a7809b6ee74e30003d58a70e + media_per_archive: 250 + archives_per_dataset: 1 + source_format: zip + include_paths: + - VidGen_video_ + modality: video + media_type: real diff --git a/src/gasbench/dataset/configs/synthetic_audio.yaml b/src/gasbench/dataset/configs/synthetic_audio.yaml index ae8c069..dd5823c 100644 --- a/src/gasbench/dataset/configs/synthetic_audio.yaml +++ b/src/gasbench/dataset/configs/synthetic_audio.yaml @@ -900,3 +900,89 @@ datasets: archives_per_dataset: -1 notes: 'Released from v24 bmcore holdouts; exact local subset. ' hf_revision: 04188209b3135617c0c682c432c2168e01b35011 + + # Released v26 holdouts — pinned upstream HF sources. + - name: v25-Fake-Synthetic-Turkish-TTS-Data + path: Anilosan15/Synthetic_Turkish_TTS_Data + hf_revision: f1d0e40efb27b5e78cdd6dbd6f6b601ab2ce9ae0 + media_per_archive: 1000 + archives_per_dataset: 1 + source_format: parquet + include_paths: + - data/train- + data_columns: + - audio + modality: audio + media_type: synthetic + - name: v25-Fake-qwen-clones-4m-en-audio-train-00000-of-00080 + path: SynDataLab-EN/qwen-clones-4m-en + hf_revision: 38769c7d96e5767035555a66ce57b1076a7dccf1 + media_per_archive: 1000 + archives_per_dataset: 1 + source_format: parquet + include_paths: + - data/train-00000-of-00080.parquet + data_columns: + - audio + modality: audio + media_type: synthetic + - name: v25-Fake-qwen-clones-4m-en-audio-train-00035-of-00080 + path: SynDataLab-EN/qwen-clones-4m-en + hf_revision: 38769c7d96e5767035555a66ce57b1076a7dccf1 + media_per_archive: 1000 + archives_per_dataset: 1 + source_format: parquet + include_paths: + - data/train-00035-of-00080.parquet + data_columns: + - audio + modality: audio + media_type: synthetic + - name: v25-Fake-qwen-clones-4m-en-audio-train-00052-of-00080 + path: SynDataLab-EN/qwen-clones-4m-en + hf_revision: 38769c7d96e5767035555a66ce57b1076a7dccf1 + media_per_archive: 1000 + archives_per_dataset: 1 + source_format: parquet + include_paths: + - data/train-00052-of-00080.parquet + data_columns: + - audio + modality: audio + media_type: synthetic + - name: v25-Fake-qwen-clones-4m-en-audio-train-00074-of-00080 + path: SynDataLab-EN/qwen-clones-4m-en + hf_revision: 38769c7d96e5767035555a66ce57b1076a7dccf1 + media_per_archive: 1000 + archives_per_dataset: 1 + source_format: parquet + include_paths: + - data/train-00074-of-00080.parquet + data_columns: + - audio + modality: audio + media_type: synthetic + - name: v26-synthetic-echo-clones-4m-en + path: SynDataLab-EN/echo-clones-4m-en + hf_revision: aebc7c787c64e918b0130c04d3878f17a602c1df + media_per_archive: 1000 + archives_per_dataset: 1 + source_format: parquet + include_paths: + - data/train- + data_columns: + - audio + modality: audio + media_type: synthetic + - name: v26-synthetic-steak + path: DEEL-AI/STEAK + hf_revision: 4ee840a09e582d23d2ce88b7f99a1996ffe9aec6 + media_per_archive: 1000 + archives_per_dataset: 1 + source_format: parquet + include_paths: + - data/STEAK- + data_columns: + - audio + modality: audio + media_type: synthetic diff --git a/src/gasbench/dataset/configs/synthetic_images.yaml b/src/gasbench/dataset/configs/synthetic_images.yaml index cf0cabb..5c1d9b0 100644 --- a/src/gasbench/dataset/configs/synthetic_images.yaml +++ b/src/gasbench/dataset/configs/synthetic_images.yaml @@ -1173,3 +1173,153 @@ datasets: archives_per_dataset: -1 notes: 'Released from v24 bmcore holdouts; exact local subset. ' hf_revision: 5649a33952bb8faabf3b861ad3b5024374509e72 + + # Released v26 holdouts — pinned upstream HF sources. + - name: v25-synthetic-10D-500K-images-Animals-bagel + path: addisonwu05/10D-500K + hf_revision: d5e2472237a1dc612d9b1d1bfaed8b9b90fbc0e1 + media_per_archive: 1000 + archives_per_dataset: 1 + source_format: parquet + include_paths: + - data/train- + data_columns: + - image + filter_values: + domain: Animals + model: bagel + modality: image + media_type: synthetic + - name: v25-synthetic-10D-500K-images-Buildings-emu + path: addisonwu05/10D-500K + hf_revision: d5e2472237a1dc612d9b1d1bfaed8b9b90fbc0e1 + media_per_archive: 1000 + archives_per_dataset: 1 + source_format: parquet + include_paths: + - data/train- + data_columns: + - image + filter_values: + domain: Buildings + model: emu + modality: image + media_type: synthetic + - name: v25-synthetic-10D-500K-images-Food_and_Drinks-janus + path: addisonwu05/10D-500K + hf_revision: d5e2472237a1dc612d9b1d1bfaed8b9b90fbc0e1 + media_per_archive: 1000 + archives_per_dataset: 1 + source_format: parquet + include_paths: + - data/train- + data_columns: + - image + filter_values: + domain: Food_and_Drinks + model: janus + modality: image + media_type: synthetic + - name: v25-synthetic-10D-500K-images-Interior_Spaces-mmada + path: addisonwu05/10D-500K + hf_revision: d5e2472237a1dc612d9b1d1bfaed8b9b90fbc0e1 + media_per_archive: 1000 + archives_per_dataset: 1 + source_format: parquet + include_paths: + - data/train- + data_columns: + - image + filter_values: + domain: Interior_Spaces + model: mmada + modality: image + media_type: synthetic + - name: v25-synthetic-10D-500K-images-People-showo2 + path: addisonwu05/10D-500K + hf_revision: d5e2472237a1dc612d9b1d1bfaed8b9b90fbc0e1 + media_per_archive: 1000 + archives_per_dataset: 1 + source_format: parquet + include_paths: + - data/train- + data_columns: + - image + filter_values: + domain: People + model: showo2 + modality: image + media_type: synthetic + - name: v25-synthetic-Persona-Fluxed-10k-2608_brazil-train + path: retowyss/Persona-Fluxed-10k-2608 + hf_revision: abd1938c598a1f67a242bb35cd9740a5e1870e78 + media_per_archive: 1000 + archives_per_dataset: 1 + source_format: png + include_paths: + - brazil/train/ + modality: image + media_type: synthetic + - name: v25-synthetic-Persona-Fluxed-10k-2608_france-train + path: retowyss/Persona-Fluxed-10k-2608 + hf_revision: abd1938c598a1f67a242bb35cd9740a5e1870e78 + media_per_archive: 1000 + archives_per_dataset: 1 + source_format: png + include_paths: + - france/train/ + modality: image + media_type: synthetic + - name: v25-synthetic-Persona-Fluxed-10k-2608_japan-train + path: retowyss/Persona-Fluxed-10k-2608 + hf_revision: abd1938c598a1f67a242bb35cd9740a5e1870e78 + media_per_archive: 1000 + archives_per_dataset: 1 + source_format: png + include_paths: + - japan/train/ + modality: image + media_type: synthetic + - name: v25-synthetic-Syn-Vis-v0-images-headshot + path: retowyss/Syn-Vis-v0 + hf_revision: 100262732989e77f38cd831d70a376a93735006a + media_per_archive: 1000 + archives_per_dataset: 1 + source_format: png + include_paths: + - images/headshot/ + modality: image + media_type: synthetic + - name: v26-synthetic-per-400k + path: a1557811266/Per-400k + hf_revision: bbb01675182feca1434ae3fbb97a9355d9337a25 + media_per_archive: 1000 + archives_per_dataset: 1 + source_format: zip + include_paths: + - person_data/img/ + modality: image + media_type: synthetic + - name: v26-synthetic-sora-explore-dataset + path: abrar71/sora-explore-dataset + hf_revision: 79559046d3782bb69147209bbbd72a787a326452 + media_per_archive: 1000 + archives_per_dataset: 1 + source_format: parquet + include_paths: + - dataset/train- + data_columns: + - image + modality: image + media_type: synthetic + - name: v26-synthetic-stylegan-face + path: gojay/StyleGAN-Face + hf_revision: d7592242ec59376dd1c418dd0353c2361631ac6e + media_per_archive: 1000 + archives_per_dataset: 1 + source_format: zip + include_paths: + - age_ + - smile_ + modality: image + media_type: synthetic diff --git a/src/gasbench/dataset/configs/synthetic_videos.yaml b/src/gasbench/dataset/configs/synthetic_videos.yaml index e6c5b6d..626b709 100644 --- a/src/gasbench/dataset/configs/synthetic_videos.yaml +++ b/src/gasbench/dataset/configs/synthetic_videos.yaml @@ -1297,3 +1297,42 @@ datasets: archives_per_dataset: -1 notes: 'Released from v24 bmcore holdouts; exact local subset. ' hf_revision: fa7713f57249caf009200ab0f875c07cd3e20986 + + # Released v26 holdouts — pinned upstream HF sources. + - name: v26-semisynthetic-vera-layered-video-dataset + path: netflix/Vera-Layered-Video-Dataset + hf_revision: 8e0b98ee9bce66fdae345e75aa766ef7c0a04d4e + media_per_archive: 250 + archives_per_dataset: 1 + source_format: mp4 + include_paths: + - train-set/49-frames/synthetic-obj-add/composite/ + - train-set/81-frames/synthetic-obj-add/composite/ + modality: video + media_type: semisynthetic + - name: v26-synthetic-text-2-video-human-preferences-genmo-mochi-1 + path: Rapidata/text-2-video-human-preferences-genmo-mochi-1 + hf_revision: 9b8c6dbba6ba4e034adaa509550e53d81e3b7148 + media_per_archive: 250 + archives_per_dataset: 1 + source_format: mp4 + include_paths: + - Videos/ + modality: video + media_type: synthetic + - name: v26-synthetic-vbench-2-0-sampled-videos + path: Vchitect/VBench-2.0_sampled_videos + hf_revision: f9ee15efcfb86d96bd5379e8d85f8d69b901aad1 + media_per_archive: 250 + archives_per_dataset: 1 + source_format: mp4 + modality: video + media_type: synthetic + - name: v26-synthetic-vbench-i2v-sampled-video + path: Vchitect/VBench-I2V_sampled_video + hf_revision: d1977eff951fad22ac7ac11059f6939f570a78eb + media_per_archive: 250 + archives_per_dataset: 1 + source_format: mp4 + modality: video + media_type: synthetic diff --git a/src/gasbench/dataset/download/core.py b/src/gasbench/dataset/download/core.py index 9bc3115..84b23b7 100644 --- a/src/gasbench/dataset/download/core.py +++ b/src/gasbench/dataset/download/core.py @@ -141,11 +141,14 @@ def download_and_extract( max_files_to_list = n_files if n_files > 0 else None # ── Filtered sequential mode ───────────────────────────────────── - # When filter_column/filter_value are set, download parquet shards + # When row filters are set, download parquet shards # one at a time and stop as soon as the sample target is hit. filter_column = getattr(dataset, "filter_column", None) filter_value = getattr(dataset, "filter_value", None) + filter_values = dict(getattr(dataset, "filter_values", None) or {}) if filter_column and filter_value is not None: + filter_values[filter_column] = filter_value + if filter_values: try: all_filenames = _list_remote_dataset_files( dataset.path, @@ -172,7 +175,7 @@ def download_and_extract( target_total = media_per_archive * archives_per_dataset logger.info( - f"Filtered mode: {dataset.name} — {filter_column}=={filter_value}, " + f"Filtered mode: {dataset.name} — {filter_values}, " f"target {target_total} samples from {len(all_filenames)} shards" ) yield from _download_filtered_sequential( diff --git a/src/gasbench/dataset/download/extract.py b/src/gasbench/dataset/download/extract.py index 318000e..9b89269 100644 --- a/src/gasbench/dataset/download/extract.py +++ b/src/gasbench/dataset/download/extract.py @@ -426,15 +426,19 @@ def _process_parquet( filter_column = getattr(dataset, "filter_column", None) filter_value = getattr(dataset, "filter_value", None) + filter_values = dict(getattr(dataset, "filter_values", None) or {}) if filter_column and filter_value is not None: - if filter_column not in df.columns: - logger.warning(f"filter_column '{filter_column}' not in {source_path.name}; skipping") + filter_values[filter_column] = filter_value + for column, value in filter_values.items(): + if column not in df.columns: + logger.warning(f"filter column '{column}' not in {source_path.name}; skipping") return - df = df[df[filter_column] == filter_value] + df = df[df[column] == value] if df.empty: - logger.info(f"No rows matching {filter_column}=={filter_value} in {source_path.name}") + logger.info(f"No rows matching {filter_values} in {source_path.name}") return - logger.info(f" {source_path.name}: {len(df)} rows after filter ({filter_column}=={filter_value})") + if filter_values: + logger.info(f" {source_path.name}: {len(df)} rows after filter ({filter_values})") # Frames-parquet: video dataset stored as one row per frame, grouped by video_id if dataset.modality == "video" and "video_id" in df.columns: @@ -689,4 +693,3 @@ def extract_number(filepath): except Exception as e: logger.warning(f"Error processing frame directory {frame_dir}: {e}") return - diff --git a/tests/integration/test_parquet_processing.py b/tests/integration/test_parquet_processing.py index 6054863..14dae8e 100644 --- a/tests/integration/test_parquet_processing.py +++ b/tests/integration/test_parquet_processing.py @@ -144,6 +144,41 @@ def download(paths, root, **kwargs): assert all(sample["label"] == label and sample["media_type"] == media_type for sample in samples) +def test_multiple_yaml_filters_collect_only_matching_rows_across_shards(tmp_path, monkeypatch): + config_path = tmp_path / "datasets.yaml" + config_path.write_text(yaml.safe_dump({"datasets": [{ + "name": "domain-model", "path": "test/images", "modality": "image", + "media_type": "synthetic", "source_format": "parquet", + "filter_values": {"domain": "Animals", "model": "bagel"}, + }]})) + (config,) = load_datasets_from_yaml(str(config_path))["image"] + filenames = [f"shard-{i}.parquet" for i in range(4)] + monkeypatch.setattr(core, "_list_remote_dataset_files", lambda *a, **k: filenames) + monkeypatch.setattr(core, "_get_download_urls", lambda path, files, *a: list(files)) + downloaded = [] + + def download(paths, root, **kwargs): + for filename in paths: + path = root / filename + pd.DataFrame([ + {"image": png(10), "domain": "Animals", "model": "bagel"}, + {"image": png(20), "domain": "Animals", "model": "emu"}, + {"image": png(30), "domain": "Buildings", "model": "bagel"}, + ]).to_parquet(path) + downloaded.append(filename) + yield path + + monkeypatch.setattr(core, "_stream_downloads", download) + monkeypatch.setattr(core, "download_files", lambda *a, **k: list(download(*a, **k))) + samples = list(core.download_and_extract( + config, media_per_archive=3, archives_per_dataset=1, + temp_dir=str(tmp_path), force_download=True, seed=7, + )) + + assert len(samples) == len(downloaded) == 3 + assert all(sample["domain"] == "Animals" and sample["model"] == "bagel" for sample in samples) + + @pytest.mark.parametrize("metadata", [{}, {"label": 1}]) def test_zero_filter_yields_nothing_when_label_is_missing_or_unmatched(parquet, metadata): config = BenchmarkDatasetConfig( From c0f3df0f7c861bf4a5bec2ab53017ee8be25cbc5 Mon Sep 17 00:00:00 2001 From: dylan Date: Tue, 6 Oct 2026 12:11:35 -0500 Subject: [PATCH 4/5] Use concise names for newly public datasets --- src/gasbench/dataset/configs/real_audio.yaml | 8 +++---- src/gasbench/dataset/configs/real_images.yaml | 14 +++++------ src/gasbench/dataset/configs/real_videos.yaml | 24 +++++++++---------- .../dataset/configs/synthetic_audio.yaml | 14 +++++------ .../dataset/configs/synthetic_images.yaml | 24 +++++++++---------- .../dataset/configs/synthetic_videos.yaml | 8 +++---- 6 files changed, 46 insertions(+), 46 deletions(-) diff --git a/src/gasbench/dataset/configs/real_audio.yaml b/src/gasbench/dataset/configs/real_audio.yaml index 515ab60..71517f0 100644 --- a/src/gasbench/dataset/configs/real_audio.yaml +++ b/src/gasbench/dataset/configs/real_audio.yaml @@ -898,7 +898,7 @@ datasets: hf_revision: 0699bb1d85463a1cb5928db83bc61bdd127b4590 # Released v26 holdouts — pinned upstream HF sources. - - name: v26-real-galgame-speech-asr-16khz + - name: galgame-speech-asr-16khz path: litagin/Galgame_Speech_ASR_16kHz hf_revision: 3fb86654222b3f0af0f7c332ae6a0ef9752a9451 media_per_archive: 1000 @@ -908,7 +908,7 @@ datasets: - data/galgame-speech-asr-16kHz-train- modality: audio media_type: real - - name: v26-real-sada2022-arabic-tts + - name: sada2022-arabic-tts path: mohamedalgmaal/sada2022-arabic-tts hf_revision: 10870c1a64d17a24a791ee08e8297f216f96a447 media_per_archive: 1000 @@ -918,7 +918,7 @@ datasets: - batch_ modality: audio media_type: real - - name: v26-real-starrail-voice + - name: starrail-voice path: simon3000/starrail-voice hf_revision: 1e5f1e8658ab2e618c4815cf64ddf905c361067d media_per_archive: 1000 @@ -930,7 +930,7 @@ datasets: - audio modality: audio media_type: real - - name: v26-real-worldspeech + - name: worldspeech path: disco-eth/WorldSpeech hf_revision: b76b85395039f50332fbc421f7c098ee2f2653d7 media_per_archive: 1000 diff --git a/src/gasbench/dataset/configs/real_images.yaml b/src/gasbench/dataset/configs/real_images.yaml index a4acd69..eee90b8 100644 --- a/src/gasbench/dataset/configs/real_images.yaml +++ b/src/gasbench/dataset/configs/real_images.yaml @@ -1290,7 +1290,7 @@ datasets: hf_revision: a3cf2557e18a832c4b2921c8d20948202eab54f5 # Released v26 holdouts — pinned upstream HF sources. - - name: v25-real-beans-test + - name: beans-test path: AI-Lab-Makerere/beans hf_revision: 27aa014ce09b193e1a6f58112d4a66e0eddb69c5 media_per_archive: 1000 @@ -1302,7 +1302,7 @@ datasets: - image modality: image media_type: real - - name: v25-real-beans-train + - name: beans-train path: AI-Lab-Makerere/beans hf_revision: 27aa014ce09b193e1a6f58112d4a66e0eddb69c5 media_per_archive: 1000 @@ -1314,7 +1314,7 @@ datasets: - image modality: image media_type: real - - name: v25-real-beans-validation + - name: beans-validation path: AI-Lab-Makerere/beans hf_revision: 27aa014ce09b193e1a6f58112d4a66e0eddb69c5 media_per_archive: 1000 @@ -1326,7 +1326,7 @@ datasets: - image modality: image media_type: real - - name: v25-real-oxford-iiit-pet-test + - name: oxford-iiit-pet-test path: timm/oxford-iiit-pet hf_revision: 089695c834a7deb60505b7cc506672db1c31a6aa media_per_archive: 1000 @@ -1338,7 +1338,7 @@ datasets: - image modality: image media_type: real - - name: v25-real-oxford-iiit-pet-train + - name: oxford-iiit-pet-train path: timm/oxford-iiit-pet hf_revision: 089695c834a7deb60505b7cc506672db1c31a6aa media_per_archive: 1000 @@ -1350,7 +1350,7 @@ datasets: - image modality: image media_type: real - - name: v26-real-dcvlm-pool-large-dvqa + - name: dcvlm-pool-large-dvqa path: mlfoundations/dcvlm_pool_large hf_revision: 479162ae38203029e06807038cd2e5e545bfd2e0 media_per_archive: 1000 @@ -1360,7 +1360,7 @@ datasets: - dvqa/ modality: image media_type: real - - name: v26-real-puffin-16m + - name: puffin-16m path: ACERobotics/Puffin-16M hf_revision: 2112b3502f9ea0cc1d31d2c5a2b7f2259b52cffd media_per_archive: 1000 diff --git a/src/gasbench/dataset/configs/real_videos.yaml b/src/gasbench/dataset/configs/real_videos.yaml index 7ed80f2..9670962 100644 --- a/src/gasbench/dataset/configs/real_videos.yaml +++ b/src/gasbench/dataset/configs/real_videos.yaml @@ -1341,7 +1341,7 @@ datasets: hf_revision: f0af08aece50ff99a9c399f063d71e455d610bbf # Released v26 holdouts — pinned upstream HF sources. - - name: v25-rendered-physical-interaction-bowling-00000 + - name: physical-interaction-bowling-00000 path: nvidia/PhysicalAI-WorldModel-Synthetic-Physical-Interaction-Scenes hf_revision: bdafca2a9c575629e47509f547e27ab7f3680369 media_per_archive: 250 @@ -1356,7 +1356,7 @@ datasets: - Side.mp4 modality: video media_type: real - - name: v25-rendered-physical-interaction-bowling-00042 + - name: physical-interaction-bowling-00042 path: nvidia/PhysicalAI-WorldModel-Synthetic-Physical-Interaction-Scenes hf_revision: bdafca2a9c575629e47509f547e27ab7f3680369 media_per_archive: 250 @@ -1371,7 +1371,7 @@ datasets: - Side.mp4 modality: video media_type: real - - name: v25-rendered-physical-interaction-bowling-00063 + - name: physical-interaction-bowling-00063 path: nvidia/PhysicalAI-WorldModel-Synthetic-Physical-Interaction-Scenes hf_revision: bdafca2a9c575629e47509f547e27ab7f3680369 media_per_archive: 250 @@ -1386,7 +1386,7 @@ datasets: - Side.mp4 modality: video media_type: real - - name: v25-rendered-physical-interaction-bowling-00084 + - name: physical-interaction-bowling-00084 path: nvidia/PhysicalAI-WorldModel-Synthetic-Physical-Interaction-Scenes hf_revision: bdafca2a9c575629e47509f547e27ab7f3680369 media_per_archive: 250 @@ -1401,7 +1401,7 @@ datasets: - Side.mp4 modality: video media_type: real - - name: v25-rendered-physicalai-worldmodel-synthetic-warehouse-operations-scenes-artifacts-forklift-human-nearmiss-nearmiss-artifacts-00005 + - name: physicalai-warehouse-forklift-nearmiss-00005 path: nvidia/PhysicalAI-WorldModel-Synthetic-Warehouse-Operations-Scenes hf_revision: d5b88d3abcf659f304a107f4336b71b4e2159133 media_per_archive: 250 @@ -1413,7 +1413,7 @@ datasets: - .rgb.mp4 modality: video media_type: real - - name: v25-rendered-physicalai-worldmodel-synthetic-warehouse-operations-scenes-artifacts-forklift-human-nearmiss-nearmiss-artifacts-00016 + - name: physicalai-warehouse-forklift-nearmiss-00016 path: nvidia/PhysicalAI-WorldModel-Synthetic-Warehouse-Operations-Scenes hf_revision: d5b88d3abcf659f304a107f4336b71b4e2159133 media_per_archive: 250 @@ -1425,7 +1425,7 @@ datasets: - .rgb.mp4 modality: video media_type: real - - name: v25-rendered-physicalai-worldmodel-synthetic-warehouse-operations-scenes-artifacts-forklift-human-nearmiss-nearmiss-artifacts-00028 + - name: physicalai-warehouse-forklift-nearmiss-00028 path: nvidia/PhysicalAI-WorldModel-Synthetic-Warehouse-Operations-Scenes hf_revision: d5b88d3abcf659f304a107f4336b71b4e2159133 media_per_archive: 250 @@ -1437,7 +1437,7 @@ datasets: - .rgb.mp4 modality: video media_type: real - - name: v25-rendered-physicalai-worldmodel-synthetic-warehouse-operations-scenes-artifacts-forklift-human-nearmiss-nearmiss-artifacts-00039 + - name: physicalai-warehouse-forklift-nearmiss-00039 path: nvidia/PhysicalAI-WorldModel-Synthetic-Warehouse-Operations-Scenes hf_revision: d5b88d3abcf659f304a107f4336b71b4e2159133 media_per_archive: 250 @@ -1449,7 +1449,7 @@ datasets: - .rgb.mp4 modality: video media_type: real - - name: v25-rendered-physicalai-worldmodel-synthetic-warehouse-operations-scenes-artifacts-forklift-human-nearmiss-nearmiss-artifacts-00050 + - name: physicalai-warehouse-forklift-nearmiss-00050 path: nvidia/PhysicalAI-WorldModel-Synthetic-Warehouse-Operations-Scenes hf_revision: d5b88d3abcf659f304a107f4336b71b4e2159133 media_per_archive: 250 @@ -1461,7 +1461,7 @@ datasets: - .rgb.mp4 modality: video media_type: real - - name: v26-real-physicalai-robotics-manipulation-objects + - name: physicalai-robotics-manipulation-objects path: nvidia/PhysicalAI-Robotics-Manipulation-Objects hf_revision: 538b6833148b4731e4ea30fe181cc4eecfc13440 media_per_archive: 250 @@ -1471,7 +1471,7 @@ datasets: - .rgb/ modality: video media_type: real - - name: v26-real-pick-tblock-mp-centered-0701-rendered + - name: pick-tblock-mp-centered-0701-rendered path: younghyopark/pick_tblock_mp_centered_0701_rendered hf_revision: b353fb987b6cc55524fb144ea61632168bc2ca5f media_per_archive: 250 @@ -1481,7 +1481,7 @@ datasets: - videos/chunk-000/ modality: video media_type: real - - name: v26-real-vidgen-1m + - name: vidgen-1m path: Fudan-FUXI/VIDGEN-1M hf_revision: 9158f218b6b81694a7809b6ee74e30003d58a70e media_per_archive: 250 diff --git a/src/gasbench/dataset/configs/synthetic_audio.yaml b/src/gasbench/dataset/configs/synthetic_audio.yaml index dd5823c..d76cca2 100644 --- a/src/gasbench/dataset/configs/synthetic_audio.yaml +++ b/src/gasbench/dataset/configs/synthetic_audio.yaml @@ -902,7 +902,7 @@ datasets: hf_revision: 04188209b3135617c0c682c432c2168e01b35011 # Released v26 holdouts — pinned upstream HF sources. - - name: v25-Fake-Synthetic-Turkish-TTS-Data + - name: turkish-tts-data path: Anilosan15/Synthetic_Turkish_TTS_Data hf_revision: f1d0e40efb27b5e78cdd6dbd6f6b601ab2ce9ae0 media_per_archive: 1000 @@ -914,7 +914,7 @@ datasets: - audio modality: audio media_type: synthetic - - name: v25-Fake-qwen-clones-4m-en-audio-train-00000-of-00080 + - name: qwen-clones-4m-en-audio-train-00000-of-00080 path: SynDataLab-EN/qwen-clones-4m-en hf_revision: 38769c7d96e5767035555a66ce57b1076a7dccf1 media_per_archive: 1000 @@ -926,7 +926,7 @@ datasets: - audio modality: audio media_type: synthetic - - name: v25-Fake-qwen-clones-4m-en-audio-train-00035-of-00080 + - name: qwen-clones-4m-en-audio-train-00035-of-00080 path: SynDataLab-EN/qwen-clones-4m-en hf_revision: 38769c7d96e5767035555a66ce57b1076a7dccf1 media_per_archive: 1000 @@ -938,7 +938,7 @@ datasets: - audio modality: audio media_type: synthetic - - name: v25-Fake-qwen-clones-4m-en-audio-train-00052-of-00080 + - name: qwen-clones-4m-en-audio-train-00052-of-00080 path: SynDataLab-EN/qwen-clones-4m-en hf_revision: 38769c7d96e5767035555a66ce57b1076a7dccf1 media_per_archive: 1000 @@ -950,7 +950,7 @@ datasets: - audio modality: audio media_type: synthetic - - name: v25-Fake-qwen-clones-4m-en-audio-train-00074-of-00080 + - name: qwen-clones-4m-en-audio-train-00074-of-00080 path: SynDataLab-EN/qwen-clones-4m-en hf_revision: 38769c7d96e5767035555a66ce57b1076a7dccf1 media_per_archive: 1000 @@ -962,7 +962,7 @@ datasets: - audio modality: audio media_type: synthetic - - name: v26-synthetic-echo-clones-4m-en + - name: echo-clones-4m-en path: SynDataLab-EN/echo-clones-4m-en hf_revision: aebc7c787c64e918b0130c04d3878f17a602c1df media_per_archive: 1000 @@ -974,7 +974,7 @@ datasets: - audio modality: audio media_type: synthetic - - name: v26-synthetic-steak + - name: steak path: DEEL-AI/STEAK hf_revision: 4ee840a09e582d23d2ce88b7f99a1996ffe9aec6 media_per_archive: 1000 diff --git a/src/gasbench/dataset/configs/synthetic_images.yaml b/src/gasbench/dataset/configs/synthetic_images.yaml index 5c1d9b0..bf891b9 100644 --- a/src/gasbench/dataset/configs/synthetic_images.yaml +++ b/src/gasbench/dataset/configs/synthetic_images.yaml @@ -1175,7 +1175,7 @@ datasets: hf_revision: 5649a33952bb8faabf3b861ad3b5024374509e72 # Released v26 holdouts — pinned upstream HF sources. - - name: v25-synthetic-10D-500K-images-Animals-bagel + - name: 10d-500k-animals-bagel path: addisonwu05/10D-500K hf_revision: d5e2472237a1dc612d9b1d1bfaed8b9b90fbc0e1 media_per_archive: 1000 @@ -1190,7 +1190,7 @@ datasets: model: bagel modality: image media_type: synthetic - - name: v25-synthetic-10D-500K-images-Buildings-emu + - name: 10d-500k-buildings-emu path: addisonwu05/10D-500K hf_revision: d5e2472237a1dc612d9b1d1bfaed8b9b90fbc0e1 media_per_archive: 1000 @@ -1205,7 +1205,7 @@ datasets: model: emu modality: image media_type: synthetic - - name: v25-synthetic-10D-500K-images-Food_and_Drinks-janus + - name: 10d-500k-food-and-drinks-janus path: addisonwu05/10D-500K hf_revision: d5e2472237a1dc612d9b1d1bfaed8b9b90fbc0e1 media_per_archive: 1000 @@ -1220,7 +1220,7 @@ datasets: model: janus modality: image media_type: synthetic - - name: v25-synthetic-10D-500K-images-Interior_Spaces-mmada + - name: 10d-500k-interior-spaces-mmada path: addisonwu05/10D-500K hf_revision: d5e2472237a1dc612d9b1d1bfaed8b9b90fbc0e1 media_per_archive: 1000 @@ -1235,7 +1235,7 @@ datasets: model: mmada modality: image media_type: synthetic - - name: v25-synthetic-10D-500K-images-People-showo2 + - name: 10d-500k-people-showo2 path: addisonwu05/10D-500K hf_revision: d5e2472237a1dc612d9b1d1bfaed8b9b90fbc0e1 media_per_archive: 1000 @@ -1250,7 +1250,7 @@ datasets: model: showo2 modality: image media_type: synthetic - - name: v25-synthetic-Persona-Fluxed-10k-2608_brazil-train + - name: persona-fluxed-10k-2608-brazil-train path: retowyss/Persona-Fluxed-10k-2608 hf_revision: abd1938c598a1f67a242bb35cd9740a5e1870e78 media_per_archive: 1000 @@ -1260,7 +1260,7 @@ datasets: - brazil/train/ modality: image media_type: synthetic - - name: v25-synthetic-Persona-Fluxed-10k-2608_france-train + - name: persona-fluxed-10k-2608-france-train path: retowyss/Persona-Fluxed-10k-2608 hf_revision: abd1938c598a1f67a242bb35cd9740a5e1870e78 media_per_archive: 1000 @@ -1270,7 +1270,7 @@ datasets: - france/train/ modality: image media_type: synthetic - - name: v25-synthetic-Persona-Fluxed-10k-2608_japan-train + - name: persona-fluxed-10k-2608-japan-train path: retowyss/Persona-Fluxed-10k-2608 hf_revision: abd1938c598a1f67a242bb35cd9740a5e1870e78 media_per_archive: 1000 @@ -1280,7 +1280,7 @@ datasets: - japan/train/ modality: image media_type: synthetic - - name: v25-synthetic-Syn-Vis-v0-images-headshot + - name: syn-vis-v0-headshot path: retowyss/Syn-Vis-v0 hf_revision: 100262732989e77f38cd831d70a376a93735006a media_per_archive: 1000 @@ -1290,7 +1290,7 @@ datasets: - images/headshot/ modality: image media_type: synthetic - - name: v26-synthetic-per-400k + - name: per-400k path: a1557811266/Per-400k hf_revision: bbb01675182feca1434ae3fbb97a9355d9337a25 media_per_archive: 1000 @@ -1300,7 +1300,7 @@ datasets: - person_data/img/ modality: image media_type: synthetic - - name: v26-synthetic-sora-explore-dataset + - name: sora-explore-dataset path: abrar71/sora-explore-dataset hf_revision: 79559046d3782bb69147209bbbd72a787a326452 media_per_archive: 1000 @@ -1312,7 +1312,7 @@ datasets: - image modality: image media_type: synthetic - - name: v26-synthetic-stylegan-face + - name: stylegan-face path: gojay/StyleGAN-Face hf_revision: d7592242ec59376dd1c418dd0353c2361631ac6e media_per_archive: 1000 diff --git a/src/gasbench/dataset/configs/synthetic_videos.yaml b/src/gasbench/dataset/configs/synthetic_videos.yaml index 626b709..91f731e 100644 --- a/src/gasbench/dataset/configs/synthetic_videos.yaml +++ b/src/gasbench/dataset/configs/synthetic_videos.yaml @@ -1299,7 +1299,7 @@ datasets: hf_revision: fa7713f57249caf009200ab0f875c07cd3e20986 # Released v26 holdouts — pinned upstream HF sources. - - name: v26-semisynthetic-vera-layered-video-dataset + - name: vera-layered-video-dataset path: netflix/Vera-Layered-Video-Dataset hf_revision: 8e0b98ee9bce66fdae345e75aa766ef7c0a04d4e media_per_archive: 250 @@ -1310,7 +1310,7 @@ datasets: - train-set/81-frames/synthetic-obj-add/composite/ modality: video media_type: semisynthetic - - name: v26-synthetic-text-2-video-human-preferences-genmo-mochi-1 + - name: text-2-video-human-preferences-genmo-mochi-1 path: Rapidata/text-2-video-human-preferences-genmo-mochi-1 hf_revision: 9b8c6dbba6ba4e034adaa509550e53d81e3b7148 media_per_archive: 250 @@ -1320,7 +1320,7 @@ datasets: - Videos/ modality: video media_type: synthetic - - name: v26-synthetic-vbench-2-0-sampled-videos + - name: vbench-2-0-sampled-videos path: Vchitect/VBench-2.0_sampled_videos hf_revision: f9ee15efcfb86d96bd5379e8d85f8d69b901aad1 media_per_archive: 250 @@ -1328,7 +1328,7 @@ datasets: source_format: mp4 modality: video media_type: synthetic - - name: v26-synthetic-vbench-i2v-sampled-video + - name: vbench-i2v-sampled-video path: Vchitect/VBench-I2V_sampled_video hf_revision: d1977eff951fad22ac7ac11059f6939f570a78eb media_per_archive: 250 From 0ca259a8d936de146b1ff9418e8807fa28a04e8f Mon Sep 17 00:00:00 2001 From: dylan Date: Tue, 6 Oct 2026 12:08:06 -0500 Subject: [PATCH 5/5] Bump GasBench version to 0.11.2 --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index a4243d8..7396950 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta" [project] name = "gasbench" -version = "0.11.1" +version = "0.11.2" description = "GASBench - ML model benchmark evaluation package" readme = "README.md" requires-python = ">=3.10"