From 6301780053182f75afb6e1d6efcb13a82522878c Mon Sep 17 00:00:00 2001 From: Shaw Date: Sat, 19 Sep 2026 22:12:58 -0700 Subject: [PATCH 01/12] tests: apply explicit thread limit to initialization and reference backend --- tests/test-backend-ops.cpp | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/tests/test-backend-ops.cpp b/tests/test-backend-ops.cpp index eddea9555..830d5bb03 100644 --- a/tests/test-backend-ops.cpp +++ b/tests/test-backend-ops.cpp @@ -57,7 +57,7 @@ static void init_tensor_uniform(ggml_tensor * tensor, float min = -1.0f, float m std::vector data(nels); { // parallel initialization - static const size_t n_threads = N_THREADS; + static const size_t n_threads = g_n_threads > 0 ? (size_t) g_n_threads : N_THREADS; // static RNG initialization (revisit if n_threads stops being constant) static std::vector generators = []() { std::random_device rd; @@ -10051,6 +10051,11 @@ static bool test_backend(ggml_backend_t backend, test_mode mode, const char * op // Use reference implementation on the CPU backend for comparison using ggml_backend_cpu_set_use_ref_t = void (*)(ggml_backend_t, bool); auto * reg = ggml_backend_dev_backend_reg(ggml_backend_get_device(backend_cpu)); + if (g_n_threads > 0) { + auto * set_n_threads = (ggml_backend_set_n_threads_t) ggml_backend_reg_get_proc_address(reg, "ggml_backend_set_n_threads"); + GGML_ASSERT(set_n_threads != nullptr); + set_n_threads(backend_cpu, g_n_threads); + } auto * set_use_ref = (ggml_backend_cpu_set_use_ref_t) ggml_backend_reg_get_proc_address(reg, "ggml_backend_cpu_set_use_ref"); if (set_use_ref) { set_use_ref(backend_cpu, true); From 9ed72696d7e138f6570f6dbcfeb86ef43f0222f7 Mon Sep 17 00:00:00 2001 From: Shaw Date: Sun, 20 Sep 2026 00:32:36 -0700 Subject: [PATCH 02/12] ci: require complete Vulkan occurrence coverage across shards --- .github/workflows/eliza-vulkan-validation.yml | 239 ++++++++++-------- scripts/ci_verify_vulkan.py | 151 +++++++++++ scripts/test_ci_verify_vulkan.py | 158 ++++++++++++ tests/test-backend-ops.cpp | 49 +++- 4 files changed, 486 insertions(+), 111 deletions(-) create mode 100644 scripts/ci_verify_vulkan.py create mode 100644 scripts/test_ci_verify_vulkan.py diff --git a/.github/workflows/eliza-vulkan-validation.yml b/.github/workflows/eliza-vulkan-validation.yml index 2847ff3df..7c4a14291 100644 --- a/.github/workflows/eliza-vulkan-validation.yml +++ b/.github/workflows/eliza-vulkan-validation.yml @@ -16,6 +16,8 @@ on: types: [opened, synchronize, reopened] paths: - '.github/workflows/eliza-vulkan-validation.yml' + - 'scripts/ci_verify_vulkan.py' + - 'scripts/test_ci_verify_vulkan.py' - 'ggml/src/ggml-vulkan/**' - 'ggml/include/**' - 'src/**' @@ -34,9 +36,12 @@ env: LLAMA_LOG_TIMESTAMPS: 1 jobs: - ubuntu-vulkan-llvmpipe-validation: + build-vulkan: runs-on: ubuntu-24.04 - timeout-minutes: 60 + timeout-minutes: 25 + outputs: + head: ${{ steps.package.outputs.head }} + build-sha: ${{ steps.package.outputs.build-sha }} steps: - name: Clone @@ -99,120 +104,138 @@ jobs: source ./vulkan_sdk/setup-env.sh vulkaninfo --summary | head -40 - - name: test-quantize-fns (CPU; smoke for Eliza custom quants) + - name: Check result verifier and CPU quantizers + shell: bash run: | - ./build-vulkan/bin/test-quantize-fns | tee /tmp/quantize-fns.log - # Must mention the Eliza-divergent quant types. + set -euo pipefail + python3 scripts/test_ci_verify_vulkan.py + ./build-vulkan/bin/test-quantize-fns | tee quantize-fns.log for t in qjl1_256 q4_polar tbq3_0 tbq4_0 tbq3_tcq; do - grep -q "Testing $t" /tmp/quantize-fns.log \ - || (echo "MISSING quant: $t" && exit 1) + grep -q "Testing $t" quantize-fns.log done - - name: test-backend-ops on Vulkan (filtered subset) - env: - GGML_VK_VISIBLE_DEVICES: '0' - GGML_VK_DISABLE_F16: '1' - GGML_VK_DISABLE_COOPMAT: '1' + - name: Package one runtime for every shard + id: package + shell: bash run: | - source ./vulkan_sdk/setup-env.sh - # Filter to ops that matter most for inference correctness. - # Full sweep is too slow on llvmpipe (~hours). - for op in MUL_MAT MUL_MAT_ID ADD MUL SOFT_MAX RMS_NORM CPY ROPE FLASH_ATTN_EXT; do - echo "=== $op ===" - timeout 600 ./build-vulkan/bin/test-backend-ops \ - -b Vulkan0 -o "$op" \ - 2>&1 | tee -a /tmp/backend-ops.log || true - done - # Summary lines for the artifact viewer. - echo "=== FAIL SUMMARY ===" - grep -E "Backend Vulkan0:.*FAIL|tests passed|Failing tests:" \ - /tmp/backend-ops.log || true - - - name: test-backend-ops Eliza custom-quant parity (Vulkan) - env: - GGML_VK_VISIBLE_DEVICES: '0' - GGML_VK_DISABLE_F16: '1' - GGML_VK_DISABLE_COOPMAT: '1' + set -euo pipefail + tar -czf vulkan-runtime.tar.gz -C build-vulkan bin + sha256sum vulkan-runtime.tar.gz > vulkan-runtime.sha256 + echo "head=$(git rev-parse HEAD)" >> "$GITHUB_OUTPUT" + echo "build-sha=$(cut -d ' ' -f 1 vulkan-runtime.sha256)" >> "$GITHUB_OUTPUT" + + - uses: actions/upload-artifact@v4 + with: + name: vulkan-validation-runtime + path: | + vulkan-runtime.tar.gz + vulkan-runtime.sha256 + quantize-fns.log + retention-days: 14 + + numerical-shards: + needs: build-vulkan + runs-on: ubuntu-24.04 + timeout-minutes: 55 + strategy: + fail-fast: false + max-parallel: 4 + matrix: + shard: [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15] + env: + GGML_VK_VISIBLE_DEVICES: '0' + GGML_VK_DISABLE_F16: '1' + GGML_VK_DISABLE_COOPMAT: '1' + SHARD: ${{ matrix.shard }} + QUALIFIED_HEAD: ${{ needs.build-vulkan.outputs.head }} + BUILD_SHA: ${{ needs.build-vulkan.outputs.build-sha }} + steps: + - uses: actions/checkout@v6 + - name: System Vulkan runtime run: | - source ./vulkan_sdk/setup-env.sh - # Eliza custom quants (Q1_0_g32/g128, TBQ3_0/4_0, TBQ3_K/4_K, - # QJL1_256, Q4_POLAR, TBQ3_TCQ). These have CPU+Metal kernels - # but no Vulkan kernels yet. The harness emits one - # "not supported" line per (op,type,shape) — that is the - # intended baseline. If a future PR adds a Vulkan kernel for - # any of these types, this step will start emitting real - # PASS/FAIL lines automatically because supports_op() will - # return true and the test harness will compare against CPU. - # - # We do NOT fail the job on "not supported" lines (expected). - # We DO fail on any "FAIL" line for these types — that would - # signal a Vulkan kernel that diverges from the CPU reference. - for op in GET_ROWS CPY MUL_MAT \ - ATTN_SCORE_TBQ ATTN_SCORE_POLAR \ - ISTFT; do - echo "=== $op (Eliza custom quants) ===" - timeout 300 ./build-vulkan/bin/test-backend-ops \ - -b Vulkan0 -o "$op" \ - 2>&1 | tee -a /tmp/eliza-custom-quants.log || true - done - # Count skip/pass/fail among lines mentioning custom quant - # type names. The grep patterns target the type prefixes used - # in the test_case::vars() output. - echo "=== Eliza custom-quant summary ===" - for t in q1_0_g32 q1_0_g128 tbq3_0 tbq4_0 tbq3_k tbq4_k qjl1_256 q4_polar tbq3_tcq; do - n_skip=$(grep -c "type=$t.*not supported" /tmp/eliza-custom-quants.log || true) - n_fail=$(grep -c "type=$t.*FAIL" /tmp/eliza-custom-quants.log || true) - n_pass=$(grep -c "type=$t.*OK" /tmp/eliza-custom-quants.log || true) - echo " $t: skip=$n_skip pass=$n_pass fail=$n_fail" - if [ "$n_fail" -gt 0 ]; then - echo "::error::Vulkan kernel divergence for type=$t (see /tmp/eliza-custom-quants.log)" - exit 1 - fi - done - # ATTN_SCORE_TBQ / ATTN_SCORE_POLAR: CPU oracle landed at - # 4ab3e26f6. Vulkan kernels were corrected at 2e5094b2c - # (turbo4 uncondition) and 1cbc7e0f8 (turbo3 layout + codebook - # + uncondition); all 3 TBQ shapes now pass parity vs CPU. - # Promoted from warning to hard-fail so future shader changes - # cannot silently regress attention scores. - echo "=== Eliza custom-attention (Vulkan) ===" - for op_name in ATTN_SCORE_TBQ ATTN_SCORE_POLAR; do - n_fail=$(grep -c "$op_name.*FAIL" /tmp/eliza-custom-quants.log || true) - n_skip=$(grep -c "$op_name.*not supported" /tmp/eliza-custom-quants.log || true) - n_pass=$(grep -c "$op_name.*OK" /tmp/eliza-custom-quants.log || true) - echo " $op_name: skip=$n_skip pass=$n_pass fail=$n_fail" - if [ "$n_fail" -gt 0 ]; then - echo "::error::Vulkan kernel diverges from CPU oracle for $op_name (see /tmp/eliza-custom-quants.log)" - exit 1 - fi - done - # ISTFT: Vulkan kernel at vulkan-shaders/istft.comp + ggml_vk_istft - # dispatch in ggml-vulkan.cpp. All 4 parity cases against the CPU - # reference must PASS — any FAIL or "not supported" line indicates - # a regression in the Vulkan iSTFT dispatch path or shader. - echo "=== ISTFT (Vulkan, gated) ===" - for op_name in ISTFT; do - n_fail=$(grep -c "$op_name.*FAIL" /tmp/eliza-custom-quants.log || true) - n_skip=$(grep -c "$op_name.*not supported" /tmp/eliza-custom-quants.log || true) - n_pass=$(grep -c "$op_name.*OK" /tmp/eliza-custom-quants.log || true) - echo " $op_name: skip=$n_skip pass=$n_pass fail=$n_fail" - if [ "$n_fail" -gt 0 ]; then - echo "::error::Vulkan iSTFT diverges from CPU reference (see ggml-cpu/ops.cpp:ggml_compute_forward_istft_f32)" - exit 1 - fi - if [ "$n_skip" -gt 0 ]; then - echo "::error::Vulkan unexpectedly reports ISTFT as not supported (regression in supports_op)" - exit 1 + sudo add-apt-repository -y ppa:kisak/kisak-mesa + sudo apt-get update -y + sudo apt-get install -y mesa-vulkan-drivers libvulkan1 + - uses: actions/download-artifact@v4 + with: + name: vulkan-validation-runtime + - name: Verify and unpack shared build + shell: bash + run: | + set -euo pipefail + test "$(git rev-parse HEAD)" = "$QUALIFIED_HEAD" + test "$(cut -d ' ' -f 1 vulkan-runtime.sha256)" = "$BUILD_SHA" + sha256sum --check vulkan-runtime.sha256 + mkdir build-vulkan + tar -xzf vulkan-runtime.tar.gz -C build-vulkan + mkdir results + : > results/ledger.jsonl + - name: Complete assigned numerical occurrences + shell: bash + run: | + set -euo pipefail + for group in broad custom; do + if [ "$group" = broad ]; then + ops="MUL_MAT MUL_MAT_ID ADD MUL SOFT_MAX RMS_NORM CPY ROPE FLASH_ATTN_EXT" + limit=600 + else + ops="GET_ROWS CPY MUL_MAT ATTN_SCORE_TBQ ATTN_SCORE_POLAR ISTFT" + limit=300 fi + for op in $ops; do + shard_args=() + verify_args=() + if [ "$op" = MUL_MAT ] || [ "$op" = FLASH_ATTN_EXT ]; then + shard_args=(--test-shard "$SHARD/16") + verify_args=(--shard "$SHARD") + elif [ "$SHARD" != 0 ]; then + continue + fi + log="results/$group-$op.log" + timeout "$limit" ./build-vulkan/bin/test-backend-ops \ + -b Vulkan0 -o "$op" "${shard_args[@]}" \ + 2>"results/$group-$op.stderr" | tee "$log" + python3 scripts/ci_verify_vulkan.py record "$group" "$op" "$log" results/ledger.jsonl \ + --head "$QUALIFIED_HEAD" --build-sha "$BUILD_SHA" "${verify_args[@]}" + done done - - - name: Upload logs + - uses: actions/upload-artifact@v4 if: always() - uses: actions/upload-artifact@v4 with: - name: eliza-vulkan-validation-logs - path: | - /tmp/quantize-fns.log - /tmp/backend-ops.log - /tmp/eliza-custom-quants.log + name: vulkan-shard-${{ matrix.shard }} + path: results/ + if-no-files-found: error + retention-days: 14 + + ubuntu-vulkan-llvmpipe-validation: + if: always() + needs: [build-vulkan, numerical-shards] + runs-on: ubuntu-24.04 + timeout-minutes: 5 + env: + BUILD_RESULT: ${{ needs.build-vulkan.result }} + SHARD_RESULT: ${{ needs.numerical-shards.result }} + QUALIFIED_HEAD: ${{ needs.build-vulkan.outputs.head }} + BUILD_SHA: ${{ needs.build-vulkan.outputs.build-sha }} + steps: + - name: Require every producer to finish successfully + run: | + test "$BUILD_RESULT" = success + test "$SHARD_RESULT" = success + - uses: actions/checkout@v6 + - uses: actions/download-artifact@v4 + with: + pattern: vulkan-shard-* + path: shard-results + - name: Reconcile all occurrences against the complete inventories + shell: bash + run: | + set -euo pipefail + test "$(git rev-parse HEAD)" = "$QUALIFIED_HEAD" + python3 scripts/ci_verify_vulkan.py aggregate shard-results \ + --head "$QUALIFIED_HEAD" --build-sha "$BUILD_SHA" | tee vulkan-coverage.json + - uses: actions/upload-artifact@v4 + with: + name: vulkan-validation-coverage + path: vulkan-coverage.json retention-days: 14 diff --git a/scripts/ci_verify_vulkan.py b/scripts/ci_verify_vulkan.py new file mode 100644 index 000000000..85c79e8fe --- /dev/null +++ b/scripts/ci_verify_vulkan.py @@ -0,0 +1,151 @@ +#!/usr/bin/env python3 +"""Verify complete Vulkan results and the union of deterministic test shards. + +Unsupported outcomes remain explicit and never count as numerical passes. +Occurrence indices preserve duplicate cases and whole-graph tests; support-mode +inventories are deliberately not used because they omit whole-graph tests. +""" +import argparse +from collections import Counter +import hashlib +import json +from pathlib import Path +import re + +ANSI = re.compile(r"\x1b\[[0-9;]*m") +LANES = { + "broad": ["MUL_MAT", "MUL_MAT_ID", "ADD", "MUL", "SOFT_MAX", "RMS_NORM", "CPY", "ROPE", "FLASH_ATTN_EXT"], + "custom": ["GET_ROWS", "CPY", "MUL_MAT", "ATTN_SCORE_TBQ", "ATTN_SCORE_POLAR", "ISTFT"], +} +SHARDED = {("broad", "MUL_MAT"), ("broad", "FLASH_ATTN_EXT"), ("custom", "MUL_MAT")} +SHARDS = 16 + + +def sha(path): + return hashlib.sha256(path.read_bytes()).hexdigest() + + +def verify(text, operation, shard=None): + text = ANSI.sub("", text) + results = re.findall(r"^ ([A-Z0-9_]+)\((.*)\): (.*)$", text, re.M) + passed, unsupported, observed = [], [], [] + for op, params, status in results: + if op != operation: + raise ValueError("Unexpected operation: " + op) + observed.append(params) + if status.strip() == "OK": + passed.append(params) + elif status.strip() == "not supported [Vulkan0]": + unsupported.append(params) + else: + raise ValueError("Nonpassing or incomplete outcome: " + op + "(" + params + "): " + status) + inventory = [] + selected = [] + inventory_rows = re.findall(r"^Shard inventory (\d+): ([A-Z0-9_]+)\((.*)\)$", text, re.M) + shard_rows = re.findall(r"^Shard (\d+)/(\d+): (\d+) selected of (\d+) matching cases$", text, re.M) + if shard is not None: + index, count = shard + if not 0 <= index < count <= 256: + raise ValueError("Invalid shard index/count") + for ordinal, (actual, op, params) in enumerate(inventory_rows): + if int(actual) != ordinal or op != operation: + raise ValueError("Noncontiguous or unexpected inventory") + inventory.append(params) + if not inventory: + raise ValueError("Missing full occurrence inventory") + selected = list(range(index, len(inventory), count)) + if shard_rows != [(str(index), str(count), str(len(selected)), str(len(inventory)))]: + raise ValueError("Missing or inconsistent shard completion") + if observed != [inventory[i] for i in selected]: + raise ValueError("Shard results do not equal assigned occurrence sequence") + elif inventory_rows or shard_rows or not passed: + raise ValueError("Unexpected shard output or no numerical cases") + summaries = re.findall(r"^ (\d+)/(\d+) tests passed$", text, re.M) + if summaries != [(str(len(passed)), str(len(passed)))]: + raise ValueError("Missing, repeated or inconsistent numerical completion summary") + if len(re.findall(r"^ Backend Vulkan0: OK$", text, re.M)) != 1: + raise ValueError("Vulkan backend did not complete exactly once") + if operation == "ISTFT" and (unsupported or len(passed) < 4): + raise ValueError("All four required ISTFT cases must execute numerically") + return {"operation": operation, "numericalPasses": len(passed), + "unsupportedCases": unsupported, "numericalCases": passed, + "inventory": inventory, "selectedIndices": selected, + "shard": list(shard) if shard is not None else None} + + +def aggregate(records, head, build_sha, shards=SHARDS): + expected = {(group, op, index if (group, op) in SHARDED else None) + for group, ops in LANES.items() for op in ops + for index in (range(shards) if (group, op) in SHARDED else [0])} + found = {} + for r in records: + if r["head"] != head or r["buildSha256"] != build_sha: + raise ValueError("Source or build artifact mismatch") + shard = r["shard"] + key = (r["group"], r["operation"], shard[0] if shard else None) + if key not in expected or key in found or (shard and shard[1] != shards): + raise ValueError("Unexpected, duplicate or misconfigured invocation") + found[key] = r + if set(found) != expected: + raise ValueError("Missing terminal invocations") + totals = [] + for group, ops in LANES.items(): + for op in ops: + family = [r for (g, o, _), r in found.items() if (g, o) == (group, op)] + numeric = sum(r["numericalPasses"] for r in family) + if numeric == 0: + raise ValueError("No numerical execution for " + group + "/" + op) + if (group, op) in SHARDED: + inventory = family[0]["inventory"] + indices = [] + for r in family: + if r["inventory"] != inventory: + raise ValueError("Workers generated different complete inventories") + indices.extend(r["selectedIndices"]) + if Counter(indices) != Counter(range(len(inventory))): + raise ValueError("Shard union omits or repeats occurrences") + totals.append({"group": group, "operation": op, "numericalPasses": numeric, + "explicitUnsupported": sum(len(r["unsupportedCases"]) for r in family)}) + return totals + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + sub = parser.add_subparsers(dest="mode", required=True) + record = sub.add_parser("record") + record.add_argument("group", choices=LANES) + record.add_argument("operation") + record.add_argument("log", type=Path) + record.add_argument("ledger", type=Path) + record.add_argument("--shard", type=int) + collect = sub.add_parser("aggregate") + collect.add_argument("root", type=Path) + for command in (record, collect): + command.add_argument("--head", required=True) + command.add_argument("--build-sha", required=True) + args = parser.parse_args() + if args.mode == "record": + shard = (args.shard, SHARDS) if args.shard is not None else None + r = verify(args.log.read_text(), args.operation, shard) + r.update(group=args.group, head=args.head, buildSha256=args.build_sha, + log=args.log.name, logSha256=sha(args.log)) + with args.ledger.open("a") as output: + output.write(json.dumps(r) + "\n") + print(json.dumps({k: r[k] for k in ("group", "operation", "shard", "numericalPasses")})) + else: + records = [] + for ledger in sorted(args.root.glob("*/ledger.jsonl")): + for line in ledger.read_text().splitlines(): + r = json.loads(line) + log = ledger.parent / r["log"] + if log.name != r["log"] or sha(log) != r["logSha256"]: + raise ValueError("Log path or digest mismatch") + verified = verify(log.read_text(), r["operation"], r["shard"]) + if any(r[key] != value for key, value in verified.items()): + raise ValueError("Receipt does not match raw output") + records.append(r) + print(json.dumps(aggregate(records, args.head, args.build_sha), indent=2)) + + +if __name__ == "__main__": + main() diff --git a/scripts/test_ci_verify_vulkan.py b/scripts/test_ci_verify_vulkan.py new file mode 100644 index 000000000..97bf5a945 --- /dev/null +++ b/scripts/test_ci_verify_vulkan.py @@ -0,0 +1,158 @@ +#!/usr/bin/env python3 +"""Exercise the backend console boundary: incomplete and unsupported runs cannot qualify.""" +import copy +import hashlib +import json +from pathlib import Path +import subprocess +import sys +import tempfile +import unittest +from ci_verify_vulkan import LANES, SHARDED, SHARDS, aggregate, verify + + +def completed(operation="ADD", statuses=("OK",)): + rows = [f" {operation}(case={i}): {status}" for i, status in enumerate(statuses)] + count = statuses.count("OK") + return "\n".join(rows + [f" {count}/{count} tests passed", " Backend Vulkan0: OK", ""]) + + +class VulkanCompletionTests(unittest.TestCase): + def test_counts_numeric_and_explicit_unsupported_separately(self): + text = completed(statuses=("OK", "not supported [Vulkan0]")) + result = verify(text.replace("OK", "\x1b[32mOK\x1b[0m"), "ADD") + self.assertEqual(result["numericalPasses"], 1) + self.assertEqual(result["unsupportedCases"], ["case=1"]) + + def test_rejects_invalid_or_incomplete_backend_output(self): + valid = completed() + invalid = { + "empty": "", + "unsupported-only": completed(statuses=("not supported [Vulkan0]",)), + "divergence": completed(statuses=("FAIL",)), + "partial-result": " ADD(case=0): ", + "missing-summary": valid.replace(" 1/1 tests passed\n", ""), + "repeated-summary": valid + " 1/1 tests passed\n", + "wrong-count": valid.replace("1/1", "2/2"), + "missing-backend": valid.replace(" Backend Vulkan0: OK\n", ""), + "wrong-operation": completed("MUL"), + "unexpected-status": completed(statuses=("skipping large tensors for speed",)), + } + for name, text in invalid.items(): + with self.subTest(name=name), self.assertRaises(ValueError): + verify(text, "ADD") + + def test_istft_requires_four_numeric_cases_without_unsupported(self): + self.assertEqual(verify(completed("ISTFT", ("OK",) * 4), "ISTFT")["numericalPasses"], 4) + for statuses in [("OK",) * 3, ("OK",) * 4 + ("not supported [Vulkan0]",)]: + with self.subTest(statuses=statuses), self.assertRaises(ValueError): + verify(completed("ISTFT", statuses), "ISTFT") + + +def shard_output(op, index, count, inventory): + rows = [f"Shard inventory {i}: {op}({params})" for i, params in enumerate(inventory)] + selected = inventory[index::count] + rows += [f" {op}({params}): OK" for params in selected] + rows += [f"Shard {index}/{count}: {len(selected)} selected of {len(inventory)} matching cases", + f" {len(selected)}/{len(selected)} tests passed", " Backend Vulkan0: OK", ""] + return "\n".join(rows) + + +class ShardCoverageTests(unittest.TestCase): + def records(self): + records = [] + for group, ops in LANES.items(): + for op in ops: + for index in (range(2) if (group, op) in SHARDED else [None]): + if index is None: + r = verify(completed(op, ("OK",) * (4 if op == "ISTFT" else 1)), op) + else: + # Duplicate identities and a whole-graph case must remain occurrences. + r = verify(shard_output(op, index, 2, ["o=1", "o=1", "o=3"]), op, (index, 2)) + r.update(group=group, head="head", buildSha256="build") + records.append(r) + return records + + def test_union_preserves_duplicate_and_whole_graph_occurrences(self): + result = aggregate(self.records(), "head", "build", shards=2) + for r in result: + if (r["group"], r["operation"]) in SHARDED: + self.assertEqual(r["numericalPasses"], 3) + + def test_rejects_incomplete_or_inconsistent_worker_sets(self): + original = self.records() + controls = {} + controls["missing-shard"] = original[1:] + controls["duplicate-shard"] = original + [original[0]] + for name, key, value in [ + ("wrong-source", "head", "other"), + ("wrong-build", "buildSha256", "other"), + ("wrong-inventory", "inventory", ["different"]), + ("duplicate-index", "selectedIndices", [0, 0]), + ("wrong-count", "shard", [0, 3]), + ]: + records = copy.deepcopy(original) + records[0][key] = value + controls[name] = records + records = copy.deepcopy(original) + for r in records: + if r["operation"] == "MUL_MAT": + r["numericalPasses"] = 0 + controls["all-unsupported-family"] = records + for name, records in controls.items(): + with self.subTest(name=name), self.assertRaises(ValueError): + aggregate(records, "head", "build", shards=2) + + def test_rejects_missing_extra_or_reordered_occurrence_output(self): + valid = shard_output("MUL_MAT", 0, 2, ["o=1", "o=1", "o=3"]) + for text in [valid.replace(" MUL_MAT(o=3): OK\n", ""), + valid.replace(" MUL_MAT(o=3): OK", " MUL_MAT(o=1): OK"), + valid.replace("Shard inventory 1:", "Shard inventory 2:"), + valid.replace("2 selected of 3", "1 selected of 3")]: + with self.subTest(text=text), self.assertRaises(ValueError): + verify(text, "MUL_MAT", (0, 2)) + for shard in [(-1, 2), (2, 2), (0, 0), (0, 257)]: + with self.subTest(shard=shard), self.assertRaises(ValueError): + verify(valid, "MUL_MAT", shard) + + +class AggregateCommandTests(unittest.TestCase): + def test_cli_rechecks_files_and_rejects_changed_or_missing_artifacts(self): + script = Path(__file__).with_name("ci_verify_vulkan.py") + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + for index in range(SHARDS): + folder = root / f"worker-{index}" + folder.mkdir() + records = [] + for group, ops in LANES.items(): + for op in ops: + sharded = (group, op) in SHARDED + if not sharded and index != 0: + continue + # Some shards have no assigned occurrences; the complete family must still execute. + text = (shard_output(op, index, SHARDS, ["o=1", "o=1", "o=3"]) + if sharded else completed(op, ("OK",) * (4 if op == "ISTFT" else 1))) + log = folder / f"{group}-{op}.log" + log.write_text(text) + record = verify(text, op, (index, SHARDS) if sharded else None) + record.update(group=group, head="head", buildSha256="build", log=log.name, + logSha256=hashlib.sha256(log.read_bytes()).hexdigest()) + records.append(record) + (folder / "ledger.jsonl").write_text("".join(json.dumps(r) + "\n" for r in records)) + command = [sys.executable, str(script), "aggregate", str(root), "--head", "head", "--build-sha", "build"] + result = subprocess.run(command, capture_output=True, text=True) + self.assertEqual(result.returncode, 0, result.stderr) + self.assertEqual(len(json.loads(result.stdout)), sum(map(len, LANES.values()))) + log = root / "worker-0" / "broad-MUL_MAT.log" + original = log.read_bytes() + log.write_text("changed artifact") + self.assertNotEqual(subprocess.run(command, capture_output=True).returncode, 0) + log.write_bytes(original) + ledger = root / "worker-1" / "ledger.jsonl" + ledger.unlink() + self.assertNotEqual(subprocess.run(command, capture_output=True).returncode, 0) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test-backend-ops.cpp b/tests/test-backend-ops.cpp index 830d5bb03..3147e5ce0 100644 --- a/tests/test-backend-ops.cpp +++ b/tests/test-backend-ops.cpp @@ -51,6 +51,9 @@ static int64_t g_perf_duration_usec = 1000000; // default: 1 second in microseconds static int g_n_threads = -1; // -1 means use backend default (N_THREADS) +static bool g_test_shard_requested = false; +static size_t g_test_shard_index = 0; +static size_t g_test_shard_count = 1; static void init_tensor_uniform(ggml_tensor * tensor, float min = -1.0f, float max = 1.0f) { size_t nels = ggml_nelements(tensor); @@ -1303,7 +1306,9 @@ struct test_case { test_status_t eval(ggml_backend_t backend1, ggml_backend_t backend2, const char * op_names_filter, - printer * output_printer) { + printer * output_printer, + size_t & matching_cases, + size_t & selected_cases) { mode = MODE_TEST; ggml_init_params params = { @@ -1329,6 +1334,18 @@ struct test_case { return test_status_t::SKIPPED; } + // Partition after the normal op filter, before capability checks or execution. + // Whole-graph cases and duplicate parameter strings retain distinct indices. + const size_t case_index = matching_cases++; + if (g_test_shard_requested) { + printf("Shard inventory %zu: %s(%s)\n", case_index, current_op_name.c_str(), vars().c_str()); + } + if (case_index % g_test_shard_count != g_test_shard_index) { + ggml_free(ctx); + return test_status_t::SKIPPED; + } + selected_cases++; + // check if the backends support the ops bool supported = true; for (ggml_backend_t backend : {backend1, backend2}) { @@ -10063,9 +10080,11 @@ static bool test_backend(ggml_backend_t backend, test_mode mode, const char * op size_t n_ok = 0; size_t tests_run = 0; + size_t matching_cases = 0; + size_t selected_cases = 0; std::vector failed_tests; for (auto & test : test_cases) { - test_status_t status = test->eval(backend, backend_cpu, op_names_filter, output_printer); + test_status_t status = test->eval(backend, backend_cpu, op_names_filter, output_printer, matching_cases, selected_cases); if (status == test_status_t::SKIPPED || status == test_status_t::NOT_SUPPORTED) { continue; } @@ -10076,6 +10095,10 @@ static bool test_backend(ggml_backend_t backend, test_mode mode, const char * op failed_tests.push_back(test->current_op_name + "(" + test->vars() + ")"); } } + if (g_test_shard_requested) { + printf("Shard %zu/%zu: %zu selected of %zu matching cases\n", + g_test_shard_index, g_test_shard_count, selected_cases, matching_cases); + } output_printer->print_summary(test_summary_info(n_ok, tests_run, false)); output_printer->print_failed_tests(failed_tests); @@ -10223,7 +10246,7 @@ static void show_test_coverage() { static void usage(char ** argv) { printf("Usage: %s [mode] [-o ] [-b ] [-p ] [--output ] [--list-ops]", argv[0]); - printf(" [--show-coverage] [--test-file ] [--perf-duration ] [--threads ]\n"); + printf(" [--show-coverage] [--test-file ] [--perf-duration ] [--threads ] [--test-shard ]\n"); printf(" valid modes:\n"); printf(" - test (default, compare with CPU backend for correctness)\n"); printf(" - grad (compare gradients from backpropagation with method of finite differences)\n"); @@ -10313,6 +10336,21 @@ int main(int argc, char ** argv) { usage(argv); return 1; } + } else if (strcmp(argv[i], "--test-shard") == 0) { + std::cmatch parts; + if (g_test_shard_requested || i + 1 >= argc || + !std::regex_match(argv[i + 1], parts, std::regex("([0-9]{1,3})/([0-9]{1,3})"))) { + fprintf(stderr, "error: --test-shard requires one zero-based index/count pair\n"); + return 1; + } + g_test_shard_index = std::stoul(parts[1].str()); + g_test_shard_count = std::stoul(parts[2].str()); + if (g_test_shard_count == 0 || g_test_shard_count > 256 || g_test_shard_index >= g_test_shard_count) { + fprintf(stderr, "error: --test-shard requires index < count and count in 1..256\n"); + return 1; + } + g_test_shard_requested = true; + ++i; } else if (strcmp(argv[i], "--threads") == 0 || strcmp(argv[i], "-t") == 0) { if (i + 1 < argc) { g_n_threads = atoi(argv[++i]); @@ -10331,6 +10369,11 @@ int main(int argc, char ** argv) { } } + if (g_test_shard_requested && (mode != MODE_TEST || output_format != output_formats::CONSOLE)) { + fprintf(stderr, "error: --test-shard requires test mode and console output\n"); + return 1; + } + // load and enumerate backends ggml_backend_load_all(); From 2bdcad806ca9d299741f4cd0fdec5e808435cd92 Mon Sep 17 00:00:00 2001 From: Shaw Date: Sun, 20 Sep 2026 04:07:34 -0700 Subject: [PATCH 03/12] ci: split Vulkan numerical coverage across 32 partitions --- .github/workflows/eliza-vulkan-validation.yml | 4 ++-- scripts/ci_verify_vulkan.py | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/eliza-vulkan-validation.yml b/.github/workflows/eliza-vulkan-validation.yml index 7c4a14291..5afe0e505 100644 --- a/.github/workflows/eliza-vulkan-validation.yml +++ b/.github/workflows/eliza-vulkan-validation.yml @@ -141,7 +141,7 @@ jobs: fail-fast: false max-parallel: 4 matrix: - shard: [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15] + shard: [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31] env: GGML_VK_VISIBLE_DEVICES: '0' GGML_VK_DISABLE_F16: '1' @@ -186,7 +186,7 @@ jobs: shard_args=() verify_args=() if [ "$op" = MUL_MAT ] || [ "$op" = FLASH_ATTN_EXT ]; then - shard_args=(--test-shard "$SHARD/16") + shard_args=(--test-shard "$SHARD/32") verify_args=(--shard "$SHARD") elif [ "$SHARD" != 0 ]; then continue diff --git a/scripts/ci_verify_vulkan.py b/scripts/ci_verify_vulkan.py index 85c79e8fe..510afa8cd 100644 --- a/scripts/ci_verify_vulkan.py +++ b/scripts/ci_verify_vulkan.py @@ -18,7 +18,7 @@ "custom": ["GET_ROWS", "CPY", "MUL_MAT", "ATTN_SCORE_TBQ", "ATTN_SCORE_POLAR", "ISTFT"], } SHARDED = {("broad", "MUL_MAT"), ("broad", "FLASH_ATTN_EXT"), ("custom", "MUL_MAT")} -SHARDS = 16 +SHARDS = 32 def sha(path): From c204681298b5e55a5a2e93c542864c7e44fb8b45 Mon Sep 17 00:00:00 2001 From: Shaw Date: Sun, 20 Sep 2026 09:31:51 -0700 Subject: [PATCH 04/12] ci: diagnose fixed Vulkan artifact with bounded exact cases --- ...eliza-vulkan-fixed-artifact-diagnostic.yml | 59 ++++++++++++++++ scripts/vulkan-fixed-artifact-cases.json | 69 +++++++++++++++++++ scripts/vulkan-fixed-artifact-diagnostic.py | 34 +++++++++ 3 files changed, 162 insertions(+) create mode 100644 .github/workflows/eliza-vulkan-fixed-artifact-diagnostic.yml create mode 100644 scripts/vulkan-fixed-artifact-cases.json create mode 100644 scripts/vulkan-fixed-artifact-diagnostic.py diff --git a/.github/workflows/eliza-vulkan-fixed-artifact-diagnostic.yml b/.github/workflows/eliza-vulkan-fixed-artifact-diagnostic.yml new file mode 100644 index 000000000..d30b7754a --- /dev/null +++ b/.github/workflows/eliza-vulkan-fixed-artifact-diagnostic.yml @@ -0,0 +1,59 @@ +name: Eliza Vulkan Fixed Artifact Diagnostic +on: + workflow_dispatch: + pull_request: + paths: + - '.github/workflows/eliza-vulkan-fixed-artifact-diagnostic.yml' + - 'scripts/vulkan-fixed-artifact-diagnostic.py' + - 'scripts/vulkan-fixed-artifact-cases.json' +permissions: + contents: read + actions: read +jobs: + diagnostic: + runs-on: ubuntu-24.04 + timeout-minutes: 85 + env: + GGML_VK_VISIBLE_DEVICES: '0' + GGML_VK_DISABLE_F16: '1' + steps: + - uses: actions/checkout@v6 + with: + fetch-depth: 0 + - name: Install matching Vulkan runtime family + run: | + sudo add-apt-repository -y ppa:kisak/kisak-mesa + sudo apt-get update -y + sudo apt-get install -y mesa-vulkan-drivers libvulkan1 vulkan-tools + - uses: actions/download-artifact@v4 + with: + name: vulkan-validation-runtime + run-id: 35506984389 + github-token: ${{ github.token }} + - name: Bind exact runtime and verifier + run: | + set -euo pipefail + test "$(sha256sum vulkan-runtime.tar.gz | cut -d ' ' -f 1)" = 4a8824b685224613468c7be9211244058e2d49e5e7a5de31f3f7d887d03e102a + sha256sum --check vulkan-runtime.sha256 + mkdir build-vulkan + tar -xzf vulkan-runtime.tar.gz -C build-vulkan + cp scripts/vulkan-fixed-artifact-cases.json diagnostic-cases.json + git show b3538ea8ac12e576c3b44f5b5015708322c446b1:scripts/ci_verify_vulkan.py > scripts/ci_verify_vulkan.py + - name: Record host and run serial cases + run: | + set -euo pipefail + mkdir diagnostic-host + uname -a > diagnostic-host/uname.txt + lscpu > diagnostic-host/cpu.txt + vulkaninfo --summary > diagnostic-host/vulkan.txt 2>&1 + dpkg-query -W mesa-vulkan-drivers libvulkan1 > diagnostic-host/versions.txt + python3 scripts/vulkan-fixed-artifact-diagnostic.py + - uses: actions/upload-artifact@v4 + if: always() + with: + name: vulkan-fixed-artifact-diagnostic + path: | + diagnostic-results + diagnostic-host + diagnostic-cases.json + if-no-files-found: error diff --git a/scripts/vulkan-fixed-artifact-cases.json b/scripts/vulkan-fixed-artifact-cases.json new file mode 100644 index 000000000..349cb49e7 --- /dev/null +++ b/scripts/vulkan-fixed-artifact-cases.json @@ -0,0 +1,69 @@ +{ + "sourceRun": 35506984389, + "artifactId": 10604247295, + "artifactName": "vulkan-validation-runtime", + "qualifiedMergeHead": "fded54caaf2e7fbb40be8059baf8ab491348b1a9", + "buildSha256": "4a8824b685224613468c7be9211244058e2d49e5e7a5de31f3f7d887d03e102a", + "pullRequestHead": "b3538ea8ac12e576c3b44f5b5015708322c446b1", + "cases": [ + { + "id": "shard1-last-completed-control", + "sourceShard": 1, + "ordinal": 4801, + "params": "hsk=512,hsv=512,nh=4,nr23=[1,1],kv=113,nb=3,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]", + "expectedOccurrences": 1, + "interpretation": "last-completed-control", + "sourceLogSha256": "98f9c5ba63bcdde61e88ace97603768b238d9ee6cdb179058dc124fe2d6ae499" + }, + { + "id": "shard1-next-unconfirmed-candidate", + "sourceShard": 1, + "ordinal": 4833, + "params": "hsk=576,hsv=512,nh=1,nr23=[1,1],kv=113,nb=3,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]", + "expectedOccurrences": 1, + "interpretation": "next-unconfirmed-candidate", + "sourceLogSha256": "98f9c5ba63bcdde61e88ace97603768b238d9ee6cdb179058dc124fe2d6ae499" + }, + { + "id": "shard3-last-completed-control", + "sourceShard": 3, + "ordinal": 4995, + "params": "hsk=576,hsv=512,nh=1,nr23=[4,1],kv=512,nb=75,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]", + "expectedOccurrences": 1, + "interpretation": "last-completed-control", + "sourceLogSha256": "579c1e6965ae89587d74fba3b466f39101693ab61a4e3160c2f7df6d3f8611b8" + }, + { + "id": "shard3-next-unconfirmed-candidate", + "sourceShard": 3, + "ordinal": 5027, + "params": "hsk=576,hsv=512,nh=1,nr23=[1,1],kv=1024,nb=75,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]", + "expectedOccurrences": 1, + "interpretation": "next-unconfirmed-candidate", + "sourceLogSha256": "579c1e6965ae89587d74fba3b466f39101693ab61a4e3160c2f7df6d3f8611b8" + }, + { + "id": "shard30-last-completed-control", + "sourceShard": 30, + "ordinal": 4446, + "params": "hsk=256,hsv=256,nh=4,nr23=[4,1],kv=512,nb=32,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]", + "expectedOccurrences": 1, + "interpretation": "last-completed-control", + "sourceLogSha256": "c8619f4a79a551c822d9b52afe1fa090fdbf033a58859393e293995a061e392a" + }, + { + "id": "shard30-next-unconfirmed-candidate", + "sourceShard": 30, + "ordinal": 4478, + "params": "hsk=320,hsv=256,nh=1,nr23=[1,1],kv=1024,nb=32,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]", + "expectedOccurrences": 1, + "interpretation": "next-unconfirmed-candidate", + "sourceLogSha256": "c8619f4a79a551c822d9b52afe1fa090fdbf033a58859393e293995a061e392a" + } + ], + "replayShard": [ + 1, + 32 + ], + "scope": "Diagnostic only. Next unconfirmed case is inferred from complete inventory and confirmed result prefix; buffered logs do not prove it was executing at timeout." +} diff --git a/scripts/vulkan-fixed-artifact-diagnostic.py b/scripts/vulkan-fixed-artifact-diagnostic.py new file mode 100644 index 000000000..a0d527c59 --- /dev/null +++ b/scripts/vulkan-fixed-artifact-diagnostic.py @@ -0,0 +1,34 @@ +"""Diagnose exact hosted Vulkan cases without rebuilding or qualifying the matrix.""" +from pathlib import Path +import hashlib, importlib.util, json, re, subprocess, time +root=Path.cwd(); out=root/'diagnostic-results'; out.mkdir(exist_ok=False) +capsule=json.loads((root/'diagnostic-cases.json').read_text()) +sha=lambda p:hashlib.sha256(p.read_bytes()).hexdigest() +assert sha(root/'vulkan-runtime.tar.gz')==capsule['buildSha256'] +spec=importlib.util.spec_from_file_location('verifier',root/'scripts/ci_verify_vulkan.py'); verifier=importlib.util.module_from_spec(spec);spec.loader.exec_module(verifier) +binary=root/'build-vulkan/bin/test-backend-ops' +artifacts={str(p.relative_to(root)):sha(p) for p in (root/'build-vulkan').rglob('*') if p.is_file()} +(out/'runtime-hashes.json').write_text(json.dumps(artifacts,indent=2)+'\n') +rows=[] +inputs=[(c['id'],['-p','^'+re.sub(r'([\\.^$|?*+()\[\]{}])',r'\\\1',c['params'])+'$'],c) for c in capsule['cases']] +inputs.append(('original-shard1',['--test-shard','1/32'],None)) +for name,extra,case in inputs: + command=['timeout','600',str(binary),'-b','Vulkan0','-o','FLASH_ATTN_EXT',*extra] + start=time.monotonic() + with (out/(name+'.stdout')).open('x') as stdout,(out/(name+'.stderr')).open('x') as stderr: + result=subprocess.run(command,stdout=stdout,stderr=stderr) + row={'id':name,'command':command,'seconds':time.monotonic()-start,'exit':result.returncode,'stdoutSha256':sha(out/(name+'.stdout')),'stderrSha256':sha(out/(name+'.stderr')),'completeNumericalResult':False} + if result.returncode==0: + try: + verified=verifier.verify((out/(name+'.stdout')).read_text(),'FLASH_ATTN_EXT',None if case else [1,32]) + if case: + assert verified['numericalPasses']==case['expectedOccurrences']>0 + assert verified['numericalCases']==[case['params']]*case['expectedOccurrences'] + assert not verified['unsupportedCases'] + else: assert verified['numericalPasses']>0 + row.update(completeNumericalResult=True,numericalPasses=verified['numericalPasses']) + except (ValueError,AssertionError) as error: + row['verificationFailure']=str(error) or 'Exact numerical occurrence mismatch' + rows.append(row);(out/'receipt.json').write_text(json.dumps({'scope':'Diagnostic only, not matrix qualification','cases':rows,'runtimeHashesUnchanged':all(sha(root/p)==h for p,h in artifacts.items())},indent=2)+'\n') +assert all(sha(root/p)==h for p,h in artifacts.items()) +raise SystemExit(0 if all(r['completeNumericalResult'] for r in rows) else 1) From 939216253780903dc341b21a4c349c08bca88be1 Mon Sep 17 00:00:00 2001 From: Shaw Date: Sun, 20 Sep 2026 09:35:54 -0700 Subject: [PATCH 05/12] ci: preserve structured Vulkan output while satisfying Python lint --- scripts/ci_verify_vulkan.py | 6 +- scripts/vulkan-fixed-artifact-diagnostic.py | 138 +++++++++++++++----- 2 files changed, 110 insertions(+), 34 deletions(-) diff --git a/scripts/ci_verify_vulkan.py b/scripts/ci_verify_vulkan.py index 510afa8cd..5286f7830 100644 --- a/scripts/ci_verify_vulkan.py +++ b/scripts/ci_verify_vulkan.py @@ -11,6 +11,7 @@ import json from pathlib import Path import re +import sys ANSI = re.compile(r"\x1b\[[0-9;]*m") LANES = { @@ -131,7 +132,8 @@ def main(): log=args.log.name, logSha256=sha(args.log)) with args.ledger.open("a") as output: output.write(json.dumps(r) + "\n") - print(json.dumps({k: r[k] for k in ("group", "operation", "shard", "numericalPasses")})) + # stdout is the machine-readable CLI result; failures retain stderr and nonzero exit. + sys.stdout.write(json.dumps({k: r[k] for k in ("group", "operation", "shard", "numericalPasses")}) + "\n") else: records = [] for ledger in sorted(args.root.glob("*/ledger.jsonl")): @@ -144,7 +146,7 @@ def main(): if any(r[key] != value for key, value in verified.items()): raise ValueError("Receipt does not match raw output") records.append(r) - print(json.dumps(aggregate(records, args.head, args.build_sha), indent=2)) + sys.stdout.write(json.dumps(aggregate(records, args.head, args.build_sha), indent=2) + "\n") if __name__ == "__main__": diff --git a/scripts/vulkan-fixed-artifact-diagnostic.py b/scripts/vulkan-fixed-artifact-diagnostic.py index a0d527c59..098359109 100644 --- a/scripts/vulkan-fixed-artifact-diagnostic.py +++ b/scripts/vulkan-fixed-artifact-diagnostic.py @@ -1,34 +1,108 @@ """Diagnose exact hosted Vulkan cases without rebuilding or qualifying the matrix.""" + from pathlib import Path -import hashlib, importlib.util, json, re, subprocess, time -root=Path.cwd(); out=root/'diagnostic-results'; out.mkdir(exist_ok=False) -capsule=json.loads((root/'diagnostic-cases.json').read_text()) -sha=lambda p:hashlib.sha256(p.read_bytes()).hexdigest() -assert sha(root/'vulkan-runtime.tar.gz')==capsule['buildSha256'] -spec=importlib.util.spec_from_file_location('verifier',root/'scripts/ci_verify_vulkan.py'); verifier=importlib.util.module_from_spec(spec);spec.loader.exec_module(verifier) -binary=root/'build-vulkan/bin/test-backend-ops' -artifacts={str(p.relative_to(root)):sha(p) for p in (root/'build-vulkan').rglob('*') if p.is_file()} -(out/'runtime-hashes.json').write_text(json.dumps(artifacts,indent=2)+'\n') -rows=[] -inputs=[(c['id'],['-p','^'+re.sub(r'([\\.^$|?*+()\[\]{}])',r'\\\1',c['params'])+'$'],c) for c in capsule['cases']] -inputs.append(('original-shard1',['--test-shard','1/32'],None)) -for name,extra,case in inputs: - command=['timeout','600',str(binary),'-b','Vulkan0','-o','FLASH_ATTN_EXT',*extra] - start=time.monotonic() - with (out/(name+'.stdout')).open('x') as stdout,(out/(name+'.stderr')).open('x') as stderr: - result=subprocess.run(command,stdout=stdout,stderr=stderr) - row={'id':name,'command':command,'seconds':time.monotonic()-start,'exit':result.returncode,'stdoutSha256':sha(out/(name+'.stdout')),'stderrSha256':sha(out/(name+'.stderr')),'completeNumericalResult':False} - if result.returncode==0: - try: - verified=verifier.verify((out/(name+'.stdout')).read_text(),'FLASH_ATTN_EXT',None if case else [1,32]) - if case: - assert verified['numericalPasses']==case['expectedOccurrences']>0 - assert verified['numericalCases']==[case['params']]*case['expectedOccurrences'] - assert not verified['unsupportedCases'] - else: assert verified['numericalPasses']>0 - row.update(completeNumericalResult=True,numericalPasses=verified['numericalPasses']) - except (ValueError,AssertionError) as error: - row['verificationFailure']=str(error) or 'Exact numerical occurrence mismatch' - rows.append(row);(out/'receipt.json').write_text(json.dumps({'scope':'Diagnostic only, not matrix qualification','cases':rows,'runtimeHashesUnchanged':all(sha(root/p)==h for p,h in artifacts.items())},indent=2)+'\n') -assert all(sha(root/p)==h for p,h in artifacts.items()) -raise SystemExit(0 if all(r['completeNumericalResult'] for r in rows) else 1) +import hashlib +import importlib.util +import json +import re +import subprocess +import time + + +def sha(p): + return hashlib.sha256(p.read_bytes()).hexdigest() + + +root = Path.cwd() +out = root / "diagnostic-results" +out.mkdir(exist_ok=False) +capsule = json.loads((root / "diagnostic-cases.json").read_text()) +assert sha(root / "vulkan-runtime.tar.gz") == capsule["buildSha256"] +spec = importlib.util.spec_from_file_location( + "verifier", root / "scripts/ci_verify_vulkan.py" +) +verifier = importlib.util.module_from_spec(spec) +spec.loader.exec_module(verifier) +binary = root / "build-vulkan/bin/test-backend-ops" +artifacts = { + str(p.relative_to(root)): sha(p) + for p in (root / "build-vulkan").rglob("*") + if p.is_file() +} +(out / "runtime-hashes.json").write_text(json.dumps(artifacts, indent=2) + "\n") +rows = [] +inputs = [ + ( + c["id"], + ["-p", "^" + re.sub(r"([\\.^$|?*+()\[\]{}])", r"\\\1", c["params"]) + "$"], + c, + ) + for c in capsule["cases"] +] +inputs.append(("original-shard1", ["--test-shard", "1/32"], None)) +for name, extra, case in inputs: + command = [ + "timeout", + "600", + str(binary), + "-b", + "Vulkan0", + "-o", + "FLASH_ATTN_EXT", + *extra, + ] + start = time.monotonic() + with ( + (out / (name + ".stdout")).open("x") as stdout, + (out / (name + ".stderr")).open("x") as stderr, + ): + result = subprocess.run(command, stdout=stdout, stderr=stderr) + row = { + "id": name, + "command": command, + "seconds": time.monotonic() - start, + "exit": result.returncode, + "stdoutSha256": sha(out / (name + ".stdout")), + "stderrSha256": sha(out / (name + ".stderr")), + "completeNumericalResult": False, + } + if result.returncode == 0: + try: + verified = verifier.verify( + (out / (name + ".stdout")).read_text(), + "FLASH_ATTN_EXT", + None if case else [1, 32], + ) + if case: + assert verified["numericalPasses"] == case["expectedOccurrences"] > 0 + assert ( + verified["numericalCases"] + == [case["params"]] * case["expectedOccurrences"] + ) + assert not verified["unsupportedCases"] + else: + assert verified["numericalPasses"] > 0 + row.update( + completeNumericalResult=True, + numericalPasses=verified["numericalPasses"], + ) + except (ValueError, AssertionError) as error: + row["verificationFailure"] = ( + str(error) or "Exact numerical occurrence mismatch" + ) + rows.append(row) + (out / "receipt.json").write_text( + json.dumps( + { + "scope": "Diagnostic only, not matrix qualification", + "cases": rows, + "runtimeHashesUnchanged": all( + sha(root / p) == h for p, h in artifacts.items() + ), + }, + indent=2, + ) + + "\n" + ) +assert all(sha(root / p) == h for p, h in artifacts.items()) +raise SystemExit(0 if all(r["completeNumericalResult"] for r in rows) else 1) From 6039ba0f001e2674fc11ac2583c555268f64b43e Mon Sep 17 00:00:00 2001 From: Shaw Date: Sun, 20 Sep 2026 10:20:38 -0700 Subject: [PATCH 06/12] ci: measure fixed Vulkan shard children with complete occurrence accounting --- ...a-vulkan-partition-children-diagnostic.yml | 63 +++++++++ .../vulkan-partition-children-diagnostic.py | 128 ++++++++++++++++++ 2 files changed, 191 insertions(+) create mode 100644 .github/workflows/eliza-vulkan-partition-children-diagnostic.yml create mode 100644 scripts/vulkan-partition-children-diagnostic.py diff --git a/.github/workflows/eliza-vulkan-partition-children-diagnostic.yml b/.github/workflows/eliza-vulkan-partition-children-diagnostic.yml new file mode 100644 index 000000000..b972cd241 --- /dev/null +++ b/.github/workflows/eliza-vulkan-partition-children-diagnostic.yml @@ -0,0 +1,63 @@ +name: Eliza Vulkan Partition Children Diagnostic +on: + workflow_dispatch: + pull_request: + paths: + - '.github/workflows/eliza-vulkan-partition-children-diagnostic.yml' + - 'scripts/vulkan-partition-children-diagnostic.py' +permissions: + contents: read + actions: read +jobs: + diagnostic: + runs-on: ubuntu-24.04 + timeout-minutes: 30 + env: + GGML_VK_VISIBLE_DEVICES: '0' + GGML_VK_DISABLE_F16: '1' + steps: + - uses: actions/checkout@v6 + with: + fetch-depth: 0 + - name: Install matching Vulkan runtime family + run: | + sudo add-apt-repository -y ppa:kisak/kisak-mesa + sudo apt-get update -y + sudo apt-get install -y mesa-vulkan-drivers libvulkan1 vulkan-tools + - uses: actions/download-artifact@v4 + with: + name: vulkan-validation-runtime + run-id: 35506984389 + github-token: ${{ github.token }} + - uses: actions/download-artifact@v4 + with: + name: vulkan-shard-0 + run-id: 35506984389 + github-token: ${{ github.token }} + path: reference-shard + - name: Bind exact runtime and verifier + run: | + set -euo pipefail + test "$(sha256sum vulkan-runtime.tar.gz | cut -d ' ' -f 1)" = 4a8824b685224613468c7be9211244058e2d49e5e7a5de31f3f7d887d03e102a + sha256sum --check vulkan-runtime.sha256 + mkdir build-vulkan + tar -xzf vulkan-runtime.tar.gz -C build-vulkan + git show b3538ea8ac12e576c3b44f5b5015708322c446b1:scripts/ci_verify_vulkan.py > scripts/ci_verify_vulkan.py + - name: Record host and run serial cases + run: | + set -euo pipefail + mkdir diagnostic-host + uname -a > diagnostic-host/uname.txt + lscpu > diagnostic-host/cpu.txt + vulkaninfo --summary > diagnostic-host/vulkan.txt 2>&1 + dpkg-query -W mesa-vulkan-drivers libvulkan1 > diagnostic-host/versions.txt + python3 scripts/vulkan-partition-children-diagnostic.py + - uses: actions/upload-artifact@v4 + if: always() + with: + name: vulkan-partition-children-diagnostic + path: | + partition-child-results + diagnostic-host + reference-shard/broad-FLASH_ATTN_EXT.log + if-no-files-found: error diff --git a/scripts/vulkan-partition-children-diagnostic.py b/scripts/vulkan-partition-children-diagnostic.py new file mode 100644 index 000000000..3198123eb --- /dev/null +++ b/scripts/vulkan-partition-children-diagnostic.py @@ -0,0 +1,128 @@ +"""Measure two smaller partitions of one timed-out shard without rebuilding the runtime.""" + +from pathlib import Path +import hashlib +import importlib.util +import json +import subprocess +import time + +ROOT = Path.cwd() +OUT = ROOT / "partition-child-results" +OUT.mkdir(exist_ok=False) +BUILD_SHA = "4a8824b685224613468c7be9211244058e2d49e5e7a5de31f3f7d887d03e102a" +REFERENCE_SHA = "19fb24bdc4fa6fef547c62435492040de832aca929bf1aece6200c989f552136" + + +def sha(path): + return hashlib.sha256(path.read_bytes()).hexdigest() + + +def save(name, value): + (OUT / name).write_text(json.dumps(value, indent=2) + "\n") + + +assert sha(ROOT / "vulkan-runtime.tar.gz") == BUILD_SHA +reference = ROOT / "reference-shard/broad-FLASH_ATTN_EXT.log" +assert sha(reference) == REFERENCE_SHA +spec = importlib.util.spec_from_file_location( + "verifier", ROOT / "scripts/ci_verify_vulkan.py" +) +verifier = importlib.util.module_from_spec(spec) +spec.loader.exec_module(verifier) +canonical = verifier.verify(reference.read_text(), "FLASH_ATTN_EXT", [0, 32])[ + "inventory" +] +assert len(canonical) == 5096 +parent_indices = list(range(1, len(canonical), 32)) +expected_children = {index: list(range(index, len(canonical), 64)) for index in [1, 33]} +assert sorted(expected_children[1] + expected_children[33]) == parent_indices +assert set(expected_children[1]).isdisjoint(expected_children[33]) +runtime = { + str(f.relative_to(ROOT)): sha(f) + for f in (ROOT / "build-vulkan").rglob("*") + if f.is_file() +} +save( + "binding.json", + { + "buildSha256": BUILD_SHA, + "referenceLogSha256": REFERENCE_SHA, + "runtime": runtime, + "parentIndices": parent_indices, + "childIndices": expected_children, + "sourceInventory": canonical, + "scope": "Fixed-artifact partition experiment, not full matrix qualification", + }, +) +rows = [] +verified_indices = [] +for index in [1, 33]: + name = f"child-{index}-of-64" + command = [ + "timeout", + "--kill-after=10s", + "600", + str(ROOT / "build-vulkan/bin/test-backend-ops"), + "-b", + "Vulkan0", + "-o", + "FLASH_ATTN_EXT", + "--test-shard", + f"{index}/64", + ] + start = time.monotonic() + with ( + (OUT / (name + ".stdout")).open("x") as stdout, + (OUT / (name + ".stderr")).open("x") as stderr, + ): + result = subprocess.run(command, stdout=stdout, stderr=stderr) + row = { + "id": name, + "command": command, + "seconds": time.monotonic() - start, + "exit": result.returncode, + "stdoutSha256": sha(OUT / (name + ".stdout")), + "stderrSha256": sha(OUT / (name + ".stderr")), + "completeNumericalResult": False, + } + if result.returncode == 0: + try: + verified = verifier.verify( + (OUT / (name + ".stdout")).read_text(), "FLASH_ATTN_EXT", [index, 64] + ) + assert verified["inventory"] == canonical + assert verified["selectedIndices"] == expected_children[index] + assert verified["numericalPasses"] > 0 + verified_indices.extend(verified["selectedIndices"]) + row.update( + completeNumericalResult=True, + numericalPasses=verified["numericalPasses"], + explicitUnsupported=len(verified["unsupportedCases"]), + selectedIndices=verified["selectedIndices"], + ) + except (ValueError, AssertionError) as error: + row["verificationFailure"] = ( + str(error) or "Exact occurrence inventory mismatch" + ) + rows.append(row) + save( + "receipt.json", + { + "cases": rows, + "runtimeHashesUnchanged": all( + sha(ROOT / f) == h for f, h in runtime.items() + ), + "completeParentOccurrenceUnion": len(rows) == 2 + and sorted(verified_indices) == parent_indices, + "scope": "Diagnostic only; all other partitions remain unqualified", + }, + ) +assert all(sha(ROOT / f) == h for f, h in runtime.items()) +assert sha(reference) == REFERENCE_SHA +raise SystemExit( + 0 + if all(r["completeNumericalResult"] for r in rows) + and sorted(verified_indices) == parent_indices + else 1 +) From 073959fe8f1eb4198399748af9e66fb1eab143c1 Mon Sep 17 00:00:00 2001 From: Shaw Date: Sun, 20 Sep 2026 10:31:56 -0700 Subject: [PATCH 07/12] ci: reject unavailable Vulkan verifier loaders explicitly --- scripts/vulkan-fixed-artifact-diagnostic.py | 2 ++ scripts/vulkan-partition-children-diagnostic.py | 2 ++ 2 files changed, 4 insertions(+) diff --git a/scripts/vulkan-fixed-artifact-diagnostic.py b/scripts/vulkan-fixed-artifact-diagnostic.py index 098359109..507850bba 100644 --- a/scripts/vulkan-fixed-artifact-diagnostic.py +++ b/scripts/vulkan-fixed-artifact-diagnostic.py @@ -21,6 +21,8 @@ def sha(p): spec = importlib.util.spec_from_file_location( "verifier", root / "scripts/ci_verify_vulkan.py" ) +if spec is None or spec.loader is None: + raise RuntimeError("Cannot load the pinned Vulkan verifier") verifier = importlib.util.module_from_spec(spec) spec.loader.exec_module(verifier) binary = root / "build-vulkan/bin/test-backend-ops" diff --git a/scripts/vulkan-partition-children-diagnostic.py b/scripts/vulkan-partition-children-diagnostic.py index 3198123eb..4b3a6b30d 100644 --- a/scripts/vulkan-partition-children-diagnostic.py +++ b/scripts/vulkan-partition-children-diagnostic.py @@ -28,6 +28,8 @@ def save(name, value): spec = importlib.util.spec_from_file_location( "verifier", ROOT / "scripts/ci_verify_vulkan.py" ) +if spec is None or spec.loader is None: + raise RuntimeError("Cannot load the pinned Vulkan verifier") verifier = importlib.util.module_from_spec(spec) spec.loader.exec_module(verifier) canonical = verifier.verify(reference.read_text(), "FLASH_ATTN_EXT", [0, 32])[ From 41465ee07e075fa1770654b36d3afa405acec441 Mon Sep 17 00:00:00 2001 From: Shaw Date: Sun, 20 Sep 2026 10:42:58 -0700 Subject: [PATCH 08/12] ci: partition Vulkan numerical qualification into 64 shards --- .github/workflows/eliza-vulkan-validation.yml | 4 ++-- scripts/ci_verify_vulkan.py | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/eliza-vulkan-validation.yml b/.github/workflows/eliza-vulkan-validation.yml index 5afe0e505..132e61d05 100644 --- a/.github/workflows/eliza-vulkan-validation.yml +++ b/.github/workflows/eliza-vulkan-validation.yml @@ -141,7 +141,7 @@ jobs: fail-fast: false max-parallel: 4 matrix: - shard: [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31] + shard: [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47, 48, 49, 50, 51, 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, 62, 63] env: GGML_VK_VISIBLE_DEVICES: '0' GGML_VK_DISABLE_F16: '1' @@ -186,7 +186,7 @@ jobs: shard_args=() verify_args=() if [ "$op" = MUL_MAT ] || [ "$op" = FLASH_ATTN_EXT ]; then - shard_args=(--test-shard "$SHARD/32") + shard_args=(--test-shard "$SHARD/64") verify_args=(--shard "$SHARD") elif [ "$SHARD" != 0 ]; then continue diff --git a/scripts/ci_verify_vulkan.py b/scripts/ci_verify_vulkan.py index 5286f7830..2aade6158 100644 --- a/scripts/ci_verify_vulkan.py +++ b/scripts/ci_verify_vulkan.py @@ -19,7 +19,7 @@ "custom": ["GET_ROWS", "CPY", "MUL_MAT", "ATTN_SCORE_TBQ", "ATTN_SCORE_POLAR", "ISTFT"], } SHARDED = {("broad", "MUL_MAT"), ("broad", "FLASH_ATTN_EXT"), ("custom", "MUL_MAT")} -SHARDS = 32 +SHARDS = 64 def sha(path): From 3b652b4900ae0926c73e9c30c7e4da79a023153e Mon Sep 17 00:00:00 2001 From: Shaw Date: Sun, 20 Sep 2026 11:04:52 -0700 Subject: [PATCH 09/12] ci: make shared Vulkan CPU runtime portable across runners --- .github/workflows/eliza-vulkan-validation.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.github/workflows/eliza-vulkan-validation.yml b/.github/workflows/eliza-vulkan-validation.yml index 132e61d05..5183c7e39 100644 --- a/.github/workflows/eliza-vulkan-validation.yml +++ b/.github/workflows/eliza-vulkan-validation.yml @@ -85,7 +85,9 @@ jobs: - name: Configure run: | source ./vulkan_sdk/setup-env.sh + # This runtime is shared across runners with different CPU features. cmake -B build-vulkan \ + -DGGML_NATIVE=OFF \ -DGGML_VULKAN=ON \ -DGGML_METAL=OFF \ -DLLAMA_CURL=OFF \ @@ -95,6 +97,7 @@ jobs: - name: Build run: | source ./vulkan_sdk/setup-env.sh + grep -q "^GGML_NATIVE:BOOL=OFF$" build-vulkan/CMakeCache.txt cmake --build build-vulkan -j $(nproc) \ --target llama-cli llama-completion \ test-backend-ops test-quantize-fns @@ -169,6 +172,7 @@ jobs: mkdir build-vulkan tar -xzf vulkan-runtime.tar.gz -C build-vulkan mkdir results + lscpu > results/cpu.txt : > results/ledger.jsonl - name: Complete assigned numerical occurrences shell: bash From 8662bea3bd816f128092b677037ed9778064fcea Mon Sep 17 00:00:00 2001 From: Shaw Date: Mon, 21 Sep 2026 10:09:06 -0700 Subject: [PATCH 10/12] ci: split Vulkan numerical workload into 128 complete shards --- .github/workflows/eliza-vulkan-validation.yml | 4 ++-- scripts/ci_verify_vulkan.py | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/eliza-vulkan-validation.yml b/.github/workflows/eliza-vulkan-validation.yml index 5183c7e39..180eb7873 100644 --- a/.github/workflows/eliza-vulkan-validation.yml +++ b/.github/workflows/eliza-vulkan-validation.yml @@ -144,7 +144,7 @@ jobs: fail-fast: false max-parallel: 4 matrix: - shard: [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47, 48, 49, 50, 51, 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, 62, 63] + shard: [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47, 48, 49, 50, 51, 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, 62, 63, 64, 65, 66, 67, 68, 69, 70, 71, 72, 73, 74, 75, 76, 77, 78, 79, 80, 81, 82, 83, 84, 85, 86, 87, 88, 89, 90, 91, 92, 93, 94, 95, 96, 97, 98, 99, 100, 101, 102, 103, 104, 105, 106, 107, 108, 109, 110, 111, 112, 113, 114, 115, 116, 117, 118, 119, 120, 121, 122, 123, 124, 125, 126, 127] env: GGML_VK_VISIBLE_DEVICES: '0' GGML_VK_DISABLE_F16: '1' @@ -190,7 +190,7 @@ jobs: shard_args=() verify_args=() if [ "$op" = MUL_MAT ] || [ "$op" = FLASH_ATTN_EXT ]; then - shard_args=(--test-shard "$SHARD/64") + shard_args=(--test-shard "$SHARD/128") verify_args=(--shard "$SHARD") elif [ "$SHARD" != 0 ]; then continue diff --git a/scripts/ci_verify_vulkan.py b/scripts/ci_verify_vulkan.py index 2aade6158..a2eb1cb27 100644 --- a/scripts/ci_verify_vulkan.py +++ b/scripts/ci_verify_vulkan.py @@ -19,7 +19,7 @@ "custom": ["GET_ROWS", "CPY", "MUL_MAT", "ATTN_SCORE_TBQ", "ATTN_SCORE_POLAR", "ISTFT"], } SHARDED = {("broad", "MUL_MAT"), ("broad", "FLASH_ATTN_EXT"), ("custom", "MUL_MAT")} -SHARDS = 64 +SHARDS = 128 def sha(path): From 7a7f559c1838b6d1f34c06cc6366b5049c422c27 Mon Sep 17 00:00:00 2001 From: Shaw Date: Tue, 13 Oct 2026 23:18:05 -0400 Subject: [PATCH 11/12] tests: balance Vulkan test shards by estimated case cost --test-shard assigned occurrences by index modulo the shard count, so the expensive hsk=512/576 FLASH_ATTN_EXT cases could stack on one worker. On llvmpipe shard 86 of 128 exceeded the 600 s limit (exit 124) while most shards finished in a few minutes. test-backend-ops now estimates each matching case (op FLOPs plus a per-byte and per-case term, from the same metadata-only graph eval() builds) and assigns longest-first to the lightest shard. The assignment is deterministic, printed in every inventory line ("Shard inventory N -> S: ..."), and the verifier requires every worker to report the same assignment, run exactly its own occurrences, and cover each occurrence once across the union. The workflow gives sharded invocations 1200 s (worker 0 also runs every unsharded op, so the job limit rises to 75 minutes) and reports each invocation's elapsed time. Co-Authored-By: Claude Opus 5.5 --- .github/workflows/eliza-vulkan-validation.yml | 14 +++- scripts/ci_verify_vulkan.py | 22 +++++- scripts/test_ci_verify_vulkan.py | 27 +++++-- tests/test-backend-ops.cpp | 77 ++++++++++++++++++- 4 files changed, 125 insertions(+), 15 deletions(-) diff --git a/.github/workflows/eliza-vulkan-validation.yml b/.github/workflows/eliza-vulkan-validation.yml index 180eb7873..7a9908f06 100644 --- a/.github/workflows/eliza-vulkan-validation.yml +++ b/.github/workflows/eliza-vulkan-validation.yml @@ -136,10 +136,16 @@ jobs: quantize-fns.log retention-days: 14 + # MUL_MAT and FLASH_ATTN_EXT are split across 128 workers. test-backend-ops + # assigns every matching occurrence to one worker by estimated cost (longest + # first onto the lightest worker), so the large hsk=512/576 FLASH_ATTN_EXT + # cases no longer cluster on one worker the way index-modulo assignment let + # them (shard 86 exceeded the 600 s limit on llvmpipe). Sharded invocations + # get 1200 s; worker 0 additionally runs every unsharded operation. numerical-shards: needs: build-vulkan runs-on: ubuntu-24.04 - timeout-minutes: 55 + timeout-minutes: 75 strategy: fail-fast: false max-parallel: 4 @@ -189,16 +195,20 @@ jobs: for op in $ops; do shard_args=() verify_args=() + op_limit="$limit" if [ "$op" = MUL_MAT ] || [ "$op" = FLASH_ATTN_EXT ]; then shard_args=(--test-shard "$SHARD/128") verify_args=(--shard "$SHARD") + op_limit=1200 elif [ "$SHARD" != 0 ]; then continue fi log="results/$group-$op.log" - timeout "$limit" ./build-vulkan/bin/test-backend-ops \ + start=$(date +%s) + timeout "$op_limit" ./build-vulkan/bin/test-backend-ops \ -b Vulkan0 -o "$op" "${shard_args[@]}" \ 2>"results/$group-$op.stderr" | tee "$log" + echo "::notice::$group/$op shard $SHARD finished in $(( $(date +%s) - start ))s of ${op_limit}s" python3 scripts/ci_verify_vulkan.py record "$group" "$op" "$log" results/ledger.jsonl \ --head "$QUALIFIED_HEAD" --build-sha "$BUILD_SHA" "${verify_args[@]}" done diff --git a/scripts/ci_verify_vulkan.py b/scripts/ci_verify_vulkan.py index a2eb1cb27..d474d84fa 100644 --- a/scripts/ci_verify_vulkan.py +++ b/scripts/ci_verify_vulkan.py @@ -1,6 +1,11 @@ #!/usr/bin/env python3 """Verify complete Vulkan results and the union of deterministic test shards. +test-backend-ops assigns each matching occurrence to one shard by estimated cost +(longest first onto the lightest shard) and prints that assignment in every +worker's inventory; all workers must agree on it and their union must cover +every occurrence exactly once. + Unsupported outcomes remain explicit and never count as numerical passes. Occurrence indices preserve duplicate cases and whole-graph tests; support-mode inventories are deliberately not used because they omit whole-graph tests. @@ -41,20 +46,24 @@ def verify(text, operation, shard=None): else: raise ValueError("Nonpassing or incomplete outcome: " + op + "(" + params + "): " + status) inventory = [] + assignment = [] selected = [] - inventory_rows = re.findall(r"^Shard inventory (\d+): ([A-Z0-9_]+)\((.*)\)$", text, re.M) + inventory_rows = re.findall(r"^Shard inventory (\d+) -> (\d+): ([A-Z0-9_]+)\((.*)\)$", text, re.M) shard_rows = re.findall(r"^Shard (\d+)/(\d+): (\d+) selected of (\d+) matching cases$", text, re.M) if shard is not None: index, count = shard if not 0 <= index < count <= 256: raise ValueError("Invalid shard index/count") - for ordinal, (actual, op, params) in enumerate(inventory_rows): + for ordinal, (actual, owner, op, params) in enumerate(inventory_rows): if int(actual) != ordinal or op != operation: raise ValueError("Noncontiguous or unexpected inventory") + if not 0 <= int(owner) < count: + raise ValueError("Occurrence assigned outside the shard range") inventory.append(params) + assignment.append(int(owner)) if not inventory: raise ValueError("Missing full occurrence inventory") - selected = list(range(index, len(inventory), count)) + selected = [i for i, owner in enumerate(assignment) if owner == index] if shard_rows != [(str(index), str(count), str(len(selected)), str(len(inventory)))]: raise ValueError("Missing or inconsistent shard completion") if observed != [inventory[i] for i in selected]: @@ -70,7 +79,7 @@ def verify(text, operation, shard=None): raise ValueError("All four required ISTFT cases must execute numerically") return {"operation": operation, "numericalPasses": len(passed), "unsupportedCases": unsupported, "numericalCases": passed, - "inventory": inventory, "selectedIndices": selected, + "inventory": inventory, "assignment": assignment, "selectedIndices": selected, "shard": list(shard) if shard is not None else None} @@ -98,10 +107,15 @@ def aggregate(records, head, build_sha, shards=SHARDS): raise ValueError("No numerical execution for " + group + "/" + op) if (group, op) in SHARDED: inventory = family[0]["inventory"] + assignment = family[0]["assignment"] indices = [] for r in family: if r["inventory"] != inventory: raise ValueError("Workers generated different complete inventories") + if r["assignment"] != assignment: + raise ValueError("Workers computed different shard assignments") + if r["selectedIndices"] != [i for i, owner in enumerate(assignment) if owner == r["shard"][0]]: + raise ValueError("Shard ran occurrences it was not assigned") indices.extend(r["selectedIndices"]) if Counter(indices) != Counter(range(len(inventory))): raise ValueError("Shard union omits or repeats occurrences") diff --git a/scripts/test_ci_verify_vulkan.py b/scripts/test_ci_verify_vulkan.py index 97bf5a945..a955e6587 100644 --- a/scripts/test_ci_verify_vulkan.py +++ b/scripts/test_ci_verify_vulkan.py @@ -49,9 +49,16 @@ def test_istft_requires_four_numeric_cases_without_unsupported(self): verify(completed("ISTFT", statuses), "ISTFT") -def shard_output(op, index, count, inventory): - rows = [f"Shard inventory {i}: {op}({params})" for i, params in enumerate(inventory)] - selected = inventory[index::count] +def balanced(n, count): + # Not round-robin: the verifier must follow the printed assignment. + return [(count - 1 - i) % count for i in range(n)] + + +def shard_output(op, index, count, inventory, assignment=None): + assignment = balanced(len(inventory), count) if assignment is None else assignment + rows = [f"Shard inventory {i} -> {owner}: {op}({params})" + for i, (params, owner) in enumerate(zip(inventory, assignment))] + selected = [params for params, owner in zip(inventory, assignment) if owner == index] rows += [f" {op}({params}): OK" for params in selected] rows += [f"Shard {index}/{count}: {len(selected)} selected of {len(inventory)} matching cases", f" {len(selected)}/{len(selected)} tests passed", " Backend Vulkan0: OK", ""] @@ -89,6 +96,7 @@ def test_rejects_incomplete_or_inconsistent_worker_sets(self): ("wrong-build", "buildSha256", "other"), ("wrong-inventory", "inventory", ["different"]), ("duplicate-index", "selectedIndices", [0, 0]), + ("different-assignment", "assignment", [0, 0, 0]), ("wrong-count", "shard", [0, 3]), ]: records = copy.deepcopy(original) @@ -105,10 +113,15 @@ def test_rejects_incomplete_or_inconsistent_worker_sets(self): def test_rejects_missing_extra_or_reordered_occurrence_output(self): valid = shard_output("MUL_MAT", 0, 2, ["o=1", "o=1", "o=3"]) - for text in [valid.replace(" MUL_MAT(o=3): OK\n", ""), - valid.replace(" MUL_MAT(o=3): OK", " MUL_MAT(o=1): OK"), - valid.replace("Shard inventory 1:", "Shard inventory 2:"), - valid.replace("2 selected of 3", "1 selected of 3")]: + # balanced(3, 2) == [1, 0, 1]: shard 0 owns only the second occurrence. + self.assertEqual(verify(valid, "MUL_MAT", (0, 2))["selectedIndices"], [1]) + for text in [valid.replace(" MUL_MAT(o=1): OK\n", ""), + valid.replace(" MUL_MAT(o=1): OK", " MUL_MAT(o=3): OK"), + valid.replace("Shard inventory 1 ->", "Shard inventory 2 ->"), + valid.replace("Shard inventory 2 -> 1:", "Shard inventory 2 -> 0:"), + valid.replace("Shard inventory 2 -> 1:", "Shard inventory 2 -> 2:"), + valid.replace("Shard inventory 2 -> 1:", "Shard inventory 2:"), + valid.replace("1 selected of 3", "2 selected of 3")]: with self.subTest(text=text), self.assertRaises(ValueError): verify(text, "MUL_MAT", (0, 2)) for shard in [(-1, 2), (2, 2), (0, 0), (0, 257)]: diff --git a/tests/test-backend-ops.cpp b/tests/test-backend-ops.cpp index 3147e5ce0..aa3d88452 100644 --- a/tests/test-backend-ops.cpp +++ b/tests/test-backend-ops.cpp @@ -54,6 +54,8 @@ static int g_n_threads = -1; // -1 means use backend default (N_THREADS) static bool g_test_shard_requested = false; static size_t g_test_shard_index = 0; static size_t g_test_shard_count = 1; +// Shard owning each matching case, indexed by occurrence (see assign_test_shards). +static std::vector g_test_shard_assignment; static void init_tensor_uniform(ggml_tensor * tensor, float min = -1.0f, float max = 1.0f) { size_t nels = ggml_nelements(tensor); @@ -1303,6 +1305,43 @@ struct test_case { } } + // Estimated relative cost of one numerical case, used only to balance --test-shard + // partitions. Builds the same metadata-only graph as eval() and applies the same op + // filter; returns false when the case does not match. The estimate is the op's + // FLOP count plus a per-byte and per-case term, so FLOP-free ops still weigh in. + bool shard_cost(const char * op_names_filter, double & cost) { + mode = MODE_SUPPORT; // no sentinels: eval() rebuilds the graph from scratch + + ggml_init_params params = { + /* .mem_size = */ ggml_tensor_overhead()*128 + ggml_graph_overhead(), + /* .mem_base = */ NULL, + /* .no_alloc = */ true, + }; + ggml_context * ctx = ggml_init(params); + GGML_ASSERT(ctx); + gf = ggml_new_graph(ctx); + + ggml_tensor * out = build_graph(ctx); + const bool match = matches_filter(out, op_names_filter); + if (match) { + ggml_build_forward_expand(gf, out); + double bytes = 0.0; + for (int i = 0; i < ggml_graph_n_nodes(gf); ++i) { + const ggml_tensor * node = ggml_graph_node(gf, i); + bytes += (double) ggml_nbytes(node); + for (int j = 0; j < GGML_MAX_SRC; ++j) { + if (node->src[j]) { + bytes += (double) ggml_nbytes(node->src[j]); + } + } + } + cost = (double) op_flops(out) + 16.0*bytes + 1e7; + } + ggml_free(ctx); + gf = nullptr; + return match; + } + test_status_t eval(ggml_backend_t backend1, ggml_backend_t backend2, const char * op_names_filter, @@ -1337,10 +1376,13 @@ struct test_case { // Partition after the normal op filter, before capability checks or execution. // Whole-graph cases and duplicate parameter strings retain distinct indices. const size_t case_index = matching_cases++; + size_t case_shard = 0; if (g_test_shard_requested) { - printf("Shard inventory %zu: %s(%s)\n", case_index, current_op_name.c_str(), vars().c_str()); + GGML_ASSERT(case_index < g_test_shard_assignment.size()); + case_shard = g_test_shard_assignment[case_index]; + printf("Shard inventory %zu -> %zu: %s(%s)\n", case_index, case_shard, current_op_name.c_str(), vars().c_str()); } - if (case_index % g_test_shard_count != g_test_shard_index) { + if (case_shard != g_test_shard_index) { ggml_free(ctx); return test_status_t::SKIPPED; } @@ -10019,6 +10061,32 @@ static std::vector> make_test_cases_from_file(const c return test_cases; } +// Deterministic cost-balanced partition of the matching cases for --test-shard: longest +// estimated case first, each to the currently lightest shard (ties: lower case index, +// lower shard). Every worker computes the same assignment from the same inventory, and +// each occurrence (including duplicates and whole-graph cases) belongs to one shard. +static void assign_test_shards(std::vector> & test_cases, const char * op_names_filter) { + std::vector costs; + for (auto & test : test_cases) { + double cost = 0.0; + if (test->shard_cost(op_names_filter, cost)) { + costs.push_back(cost); + } + } + std::vector order(costs.size()); + for (size_t i = 0; i < order.size(); ++i) { + order[i] = i; + } + std::stable_sort(order.begin(), order.end(), [&](size_t a, size_t b) { return costs[a] > costs[b]; }); + std::vector load(g_test_shard_count, 0.0); + g_test_shard_assignment.assign(costs.size(), 0); + for (size_t i : order) { + const size_t shard = (size_t) (std::min_element(load.begin(), load.end()) - load.begin()); + g_test_shard_assignment[i] = shard; + load[shard] += costs[i]; + } +} + static bool test_backend(ggml_backend_t backend, test_mode mode, const char * op_names_filter, const char * params_filter, printer * output_printer, const char * test_file_path) { auto filter_test_cases = [](std::vector> & test_cases, const char * params_filter) { @@ -10078,6 +10146,10 @@ static bool test_backend(ggml_backend_t backend, test_mode mode, const char * op set_use_ref(backend_cpu, true); } + if (g_test_shard_requested) { + assign_test_shards(test_cases, op_names_filter); + } + size_t n_ok = 0; size_t tests_run = 0; size_t matching_cases = 0; @@ -10096,6 +10168,7 @@ static bool test_backend(ggml_backend_t backend, test_mode mode, const char * op } } if (g_test_shard_requested) { + GGML_ASSERT(matching_cases == g_test_shard_assignment.size()); printf("Shard %zu/%zu: %zu selected of %zu matching cases\n", g_test_shard_index, g_test_shard_count, selected_cases, matching_cases); } From b8fa7fe732440f1eb0b7b5ac8e4ea0d5f9306a60 Mon Sep 17 00:00:00 2001 From: Shaw Date: Tue, 13 Oct 2026 23:18:05 -0400 Subject: [PATCH 12/12] ci: remove one-off Vulkan diagnostic workflows The fixed-artifact and partition-children diagnostics replayed one runtime artifact from run 35506984389 to time cases from the shard that timed out. They qualify nothing ("Diagnostic only, not matrix qualification"), the fixed-artifact run exits 1 whenever one of its pinned slow cases does not complete in its bound (as on the last push), and the pinned artifact has a 14-day retention that has since expired, so both would now fail on any pull request touching them. The shard timing they measured is addressed by cost-balanced sharding. Co-Authored-By: Claude Opus 5.5 --- ...eliza-vulkan-fixed-artifact-diagnostic.yml | 59 -------- ...a-vulkan-partition-children-diagnostic.yml | 63 --------- scripts/vulkan-fixed-artifact-cases.json | 69 ---------- scripts/vulkan-fixed-artifact-diagnostic.py | 110 --------------- .../vulkan-partition-children-diagnostic.py | 130 ------------------ 5 files changed, 431 deletions(-) delete mode 100644 .github/workflows/eliza-vulkan-fixed-artifact-diagnostic.yml delete mode 100644 .github/workflows/eliza-vulkan-partition-children-diagnostic.yml delete mode 100644 scripts/vulkan-fixed-artifact-cases.json delete mode 100644 scripts/vulkan-fixed-artifact-diagnostic.py delete mode 100644 scripts/vulkan-partition-children-diagnostic.py diff --git a/.github/workflows/eliza-vulkan-fixed-artifact-diagnostic.yml b/.github/workflows/eliza-vulkan-fixed-artifact-diagnostic.yml deleted file mode 100644 index d30b7754a..000000000 --- a/.github/workflows/eliza-vulkan-fixed-artifact-diagnostic.yml +++ /dev/null @@ -1,59 +0,0 @@ -name: Eliza Vulkan Fixed Artifact Diagnostic -on: - workflow_dispatch: - pull_request: - paths: - - '.github/workflows/eliza-vulkan-fixed-artifact-diagnostic.yml' - - 'scripts/vulkan-fixed-artifact-diagnostic.py' - - 'scripts/vulkan-fixed-artifact-cases.json' -permissions: - contents: read - actions: read -jobs: - diagnostic: - runs-on: ubuntu-24.04 - timeout-minutes: 85 - env: - GGML_VK_VISIBLE_DEVICES: '0' - GGML_VK_DISABLE_F16: '1' - steps: - - uses: actions/checkout@v6 - with: - fetch-depth: 0 - - name: Install matching Vulkan runtime family - run: | - sudo add-apt-repository -y ppa:kisak/kisak-mesa - sudo apt-get update -y - sudo apt-get install -y mesa-vulkan-drivers libvulkan1 vulkan-tools - - uses: actions/download-artifact@v4 - with: - name: vulkan-validation-runtime - run-id: 35506984389 - github-token: ${{ github.token }} - - name: Bind exact runtime and verifier - run: | - set -euo pipefail - test "$(sha256sum vulkan-runtime.tar.gz | cut -d ' ' -f 1)" = 4a8824b685224613468c7be9211244058e2d49e5e7a5de31f3f7d887d03e102a - sha256sum --check vulkan-runtime.sha256 - mkdir build-vulkan - tar -xzf vulkan-runtime.tar.gz -C build-vulkan - cp scripts/vulkan-fixed-artifact-cases.json diagnostic-cases.json - git show b3538ea8ac12e576c3b44f5b5015708322c446b1:scripts/ci_verify_vulkan.py > scripts/ci_verify_vulkan.py - - name: Record host and run serial cases - run: | - set -euo pipefail - mkdir diagnostic-host - uname -a > diagnostic-host/uname.txt - lscpu > diagnostic-host/cpu.txt - vulkaninfo --summary > diagnostic-host/vulkan.txt 2>&1 - dpkg-query -W mesa-vulkan-drivers libvulkan1 > diagnostic-host/versions.txt - python3 scripts/vulkan-fixed-artifact-diagnostic.py - - uses: actions/upload-artifact@v4 - if: always() - with: - name: vulkan-fixed-artifact-diagnostic - path: | - diagnostic-results - diagnostic-host - diagnostic-cases.json - if-no-files-found: error diff --git a/.github/workflows/eliza-vulkan-partition-children-diagnostic.yml b/.github/workflows/eliza-vulkan-partition-children-diagnostic.yml deleted file mode 100644 index b972cd241..000000000 --- a/.github/workflows/eliza-vulkan-partition-children-diagnostic.yml +++ /dev/null @@ -1,63 +0,0 @@ -name: Eliza Vulkan Partition Children Diagnostic -on: - workflow_dispatch: - pull_request: - paths: - - '.github/workflows/eliza-vulkan-partition-children-diagnostic.yml' - - 'scripts/vulkan-partition-children-diagnostic.py' -permissions: - contents: read - actions: read -jobs: - diagnostic: - runs-on: ubuntu-24.04 - timeout-minutes: 30 - env: - GGML_VK_VISIBLE_DEVICES: '0' - GGML_VK_DISABLE_F16: '1' - steps: - - uses: actions/checkout@v6 - with: - fetch-depth: 0 - - name: Install matching Vulkan runtime family - run: | - sudo add-apt-repository -y ppa:kisak/kisak-mesa - sudo apt-get update -y - sudo apt-get install -y mesa-vulkan-drivers libvulkan1 vulkan-tools - - uses: actions/download-artifact@v4 - with: - name: vulkan-validation-runtime - run-id: 35506984389 - github-token: ${{ github.token }} - - uses: actions/download-artifact@v4 - with: - name: vulkan-shard-0 - run-id: 35506984389 - github-token: ${{ github.token }} - path: reference-shard - - name: Bind exact runtime and verifier - run: | - set -euo pipefail - test "$(sha256sum vulkan-runtime.tar.gz | cut -d ' ' -f 1)" = 4a8824b685224613468c7be9211244058e2d49e5e7a5de31f3f7d887d03e102a - sha256sum --check vulkan-runtime.sha256 - mkdir build-vulkan - tar -xzf vulkan-runtime.tar.gz -C build-vulkan - git show b3538ea8ac12e576c3b44f5b5015708322c446b1:scripts/ci_verify_vulkan.py > scripts/ci_verify_vulkan.py - - name: Record host and run serial cases - run: | - set -euo pipefail - mkdir diagnostic-host - uname -a > diagnostic-host/uname.txt - lscpu > diagnostic-host/cpu.txt - vulkaninfo --summary > diagnostic-host/vulkan.txt 2>&1 - dpkg-query -W mesa-vulkan-drivers libvulkan1 > diagnostic-host/versions.txt - python3 scripts/vulkan-partition-children-diagnostic.py - - uses: actions/upload-artifact@v4 - if: always() - with: - name: vulkan-partition-children-diagnostic - path: | - partition-child-results - diagnostic-host - reference-shard/broad-FLASH_ATTN_EXT.log - if-no-files-found: error diff --git a/scripts/vulkan-fixed-artifact-cases.json b/scripts/vulkan-fixed-artifact-cases.json deleted file mode 100644 index 349cb49e7..000000000 --- a/scripts/vulkan-fixed-artifact-cases.json +++ /dev/null @@ -1,69 +0,0 @@ -{ - "sourceRun": 35506984389, - "artifactId": 10604247295, - "artifactName": "vulkan-validation-runtime", - "qualifiedMergeHead": "fded54caaf2e7fbb40be8059baf8ab491348b1a9", - "buildSha256": "4a8824b685224613468c7be9211244058e2d49e5e7a5de31f3f7d887d03e102a", - "pullRequestHead": "b3538ea8ac12e576c3b44f5b5015708322c446b1", - "cases": [ - { - "id": "shard1-last-completed-control", - "sourceShard": 1, - "ordinal": 4801, - "params": "hsk=512,hsv=512,nh=4,nr23=[1,1],kv=113,nb=3,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]", - "expectedOccurrences": 1, - "interpretation": "last-completed-control", - "sourceLogSha256": "98f9c5ba63bcdde61e88ace97603768b238d9ee6cdb179058dc124fe2d6ae499" - }, - { - "id": "shard1-next-unconfirmed-candidate", - "sourceShard": 1, - "ordinal": 4833, - "params": "hsk=576,hsv=512,nh=1,nr23=[1,1],kv=113,nb=3,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]", - "expectedOccurrences": 1, - "interpretation": "next-unconfirmed-candidate", - "sourceLogSha256": "98f9c5ba63bcdde61e88ace97603768b238d9ee6cdb179058dc124fe2d6ae499" - }, - { - "id": "shard3-last-completed-control", - "sourceShard": 3, - "ordinal": 4995, - "params": "hsk=576,hsv=512,nh=1,nr23=[4,1],kv=512,nb=75,mask=1,sinks=0,max_bias=8.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]", - "expectedOccurrences": 1, - "interpretation": "last-completed-control", - "sourceLogSha256": "579c1e6965ae89587d74fba3b466f39101693ab61a4e3160c2f7df6d3f8611b8" - }, - { - "id": "shard3-next-unconfirmed-candidate", - "sourceShard": 3, - "ordinal": 5027, - "params": "hsk=576,hsv=512,nh=1,nr23=[1,1],kv=1024,nb=75,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]", - "expectedOccurrences": 1, - "interpretation": "next-unconfirmed-candidate", - "sourceLogSha256": "579c1e6965ae89587d74fba3b466f39101693ab61a4e3160c2f7df6d3f8611b8" - }, - { - "id": "shard30-last-completed-control", - "sourceShard": 30, - "ordinal": 4446, - "params": "hsk=256,hsv=256,nh=4,nr23=[4,1],kv=512,nb=32,mask=0,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]", - "expectedOccurrences": 1, - "interpretation": "last-completed-control", - "sourceLogSha256": "c8619f4a79a551c822d9b52afe1fa090fdbf033a58859393e293995a061e392a" - }, - { - "id": "shard30-next-unconfirmed-candidate", - "sourceShard": 30, - "ordinal": 4478, - "params": "hsk=320,hsv=256,nh=1,nr23=[1,1],kv=1024,nb=32,mask=1,sinks=1,max_bias=0.000000,logit_softcap=0.000000,prec=f32,type_K=f16,type_V=f16,permute=[0,1,2,3]", - "expectedOccurrences": 1, - "interpretation": "next-unconfirmed-candidate", - "sourceLogSha256": "c8619f4a79a551c822d9b52afe1fa090fdbf033a58859393e293995a061e392a" - } - ], - "replayShard": [ - 1, - 32 - ], - "scope": "Diagnostic only. Next unconfirmed case is inferred from complete inventory and confirmed result prefix; buffered logs do not prove it was executing at timeout." -} diff --git a/scripts/vulkan-fixed-artifact-diagnostic.py b/scripts/vulkan-fixed-artifact-diagnostic.py deleted file mode 100644 index 507850bba..000000000 --- a/scripts/vulkan-fixed-artifact-diagnostic.py +++ /dev/null @@ -1,110 +0,0 @@ -"""Diagnose exact hosted Vulkan cases without rebuilding or qualifying the matrix.""" - -from pathlib import Path -import hashlib -import importlib.util -import json -import re -import subprocess -import time - - -def sha(p): - return hashlib.sha256(p.read_bytes()).hexdigest() - - -root = Path.cwd() -out = root / "diagnostic-results" -out.mkdir(exist_ok=False) -capsule = json.loads((root / "diagnostic-cases.json").read_text()) -assert sha(root / "vulkan-runtime.tar.gz") == capsule["buildSha256"] -spec = importlib.util.spec_from_file_location( - "verifier", root / "scripts/ci_verify_vulkan.py" -) -if spec is None or spec.loader is None: - raise RuntimeError("Cannot load the pinned Vulkan verifier") -verifier = importlib.util.module_from_spec(spec) -spec.loader.exec_module(verifier) -binary = root / "build-vulkan/bin/test-backend-ops" -artifacts = { - str(p.relative_to(root)): sha(p) - for p in (root / "build-vulkan").rglob("*") - if p.is_file() -} -(out / "runtime-hashes.json").write_text(json.dumps(artifacts, indent=2) + "\n") -rows = [] -inputs = [ - ( - c["id"], - ["-p", "^" + re.sub(r"([\\.^$|?*+()\[\]{}])", r"\\\1", c["params"]) + "$"], - c, - ) - for c in capsule["cases"] -] -inputs.append(("original-shard1", ["--test-shard", "1/32"], None)) -for name, extra, case in inputs: - command = [ - "timeout", - "600", - str(binary), - "-b", - "Vulkan0", - "-o", - "FLASH_ATTN_EXT", - *extra, - ] - start = time.monotonic() - with ( - (out / (name + ".stdout")).open("x") as stdout, - (out / (name + ".stderr")).open("x") as stderr, - ): - result = subprocess.run(command, stdout=stdout, stderr=stderr) - row = { - "id": name, - "command": command, - "seconds": time.monotonic() - start, - "exit": result.returncode, - "stdoutSha256": sha(out / (name + ".stdout")), - "stderrSha256": sha(out / (name + ".stderr")), - "completeNumericalResult": False, - } - if result.returncode == 0: - try: - verified = verifier.verify( - (out / (name + ".stdout")).read_text(), - "FLASH_ATTN_EXT", - None if case else [1, 32], - ) - if case: - assert verified["numericalPasses"] == case["expectedOccurrences"] > 0 - assert ( - verified["numericalCases"] - == [case["params"]] * case["expectedOccurrences"] - ) - assert not verified["unsupportedCases"] - else: - assert verified["numericalPasses"] > 0 - row.update( - completeNumericalResult=True, - numericalPasses=verified["numericalPasses"], - ) - except (ValueError, AssertionError) as error: - row["verificationFailure"] = ( - str(error) or "Exact numerical occurrence mismatch" - ) - rows.append(row) - (out / "receipt.json").write_text( - json.dumps( - { - "scope": "Diagnostic only, not matrix qualification", - "cases": rows, - "runtimeHashesUnchanged": all( - sha(root / p) == h for p, h in artifacts.items() - ), - }, - indent=2, - ) - + "\n" - ) -assert all(sha(root / p) == h for p, h in artifacts.items()) -raise SystemExit(0 if all(r["completeNumericalResult"] for r in rows) else 1) diff --git a/scripts/vulkan-partition-children-diagnostic.py b/scripts/vulkan-partition-children-diagnostic.py deleted file mode 100644 index 4b3a6b30d..000000000 --- a/scripts/vulkan-partition-children-diagnostic.py +++ /dev/null @@ -1,130 +0,0 @@ -"""Measure two smaller partitions of one timed-out shard without rebuilding the runtime.""" - -from pathlib import Path -import hashlib -import importlib.util -import json -import subprocess -import time - -ROOT = Path.cwd() -OUT = ROOT / "partition-child-results" -OUT.mkdir(exist_ok=False) -BUILD_SHA = "4a8824b685224613468c7be9211244058e2d49e5e7a5de31f3f7d887d03e102a" -REFERENCE_SHA = "19fb24bdc4fa6fef547c62435492040de832aca929bf1aece6200c989f552136" - - -def sha(path): - return hashlib.sha256(path.read_bytes()).hexdigest() - - -def save(name, value): - (OUT / name).write_text(json.dumps(value, indent=2) + "\n") - - -assert sha(ROOT / "vulkan-runtime.tar.gz") == BUILD_SHA -reference = ROOT / "reference-shard/broad-FLASH_ATTN_EXT.log" -assert sha(reference) == REFERENCE_SHA -spec = importlib.util.spec_from_file_location( - "verifier", ROOT / "scripts/ci_verify_vulkan.py" -) -if spec is None or spec.loader is None: - raise RuntimeError("Cannot load the pinned Vulkan verifier") -verifier = importlib.util.module_from_spec(spec) -spec.loader.exec_module(verifier) -canonical = verifier.verify(reference.read_text(), "FLASH_ATTN_EXT", [0, 32])[ - "inventory" -] -assert len(canonical) == 5096 -parent_indices = list(range(1, len(canonical), 32)) -expected_children = {index: list(range(index, len(canonical), 64)) for index in [1, 33]} -assert sorted(expected_children[1] + expected_children[33]) == parent_indices -assert set(expected_children[1]).isdisjoint(expected_children[33]) -runtime = { - str(f.relative_to(ROOT)): sha(f) - for f in (ROOT / "build-vulkan").rglob("*") - if f.is_file() -} -save( - "binding.json", - { - "buildSha256": BUILD_SHA, - "referenceLogSha256": REFERENCE_SHA, - "runtime": runtime, - "parentIndices": parent_indices, - "childIndices": expected_children, - "sourceInventory": canonical, - "scope": "Fixed-artifact partition experiment, not full matrix qualification", - }, -) -rows = [] -verified_indices = [] -for index in [1, 33]: - name = f"child-{index}-of-64" - command = [ - "timeout", - "--kill-after=10s", - "600", - str(ROOT / "build-vulkan/bin/test-backend-ops"), - "-b", - "Vulkan0", - "-o", - "FLASH_ATTN_EXT", - "--test-shard", - f"{index}/64", - ] - start = time.monotonic() - with ( - (OUT / (name + ".stdout")).open("x") as stdout, - (OUT / (name + ".stderr")).open("x") as stderr, - ): - result = subprocess.run(command, stdout=stdout, stderr=stderr) - row = { - "id": name, - "command": command, - "seconds": time.monotonic() - start, - "exit": result.returncode, - "stdoutSha256": sha(OUT / (name + ".stdout")), - "stderrSha256": sha(OUT / (name + ".stderr")), - "completeNumericalResult": False, - } - if result.returncode == 0: - try: - verified = verifier.verify( - (OUT / (name + ".stdout")).read_text(), "FLASH_ATTN_EXT", [index, 64] - ) - assert verified["inventory"] == canonical - assert verified["selectedIndices"] == expected_children[index] - assert verified["numericalPasses"] > 0 - verified_indices.extend(verified["selectedIndices"]) - row.update( - completeNumericalResult=True, - numericalPasses=verified["numericalPasses"], - explicitUnsupported=len(verified["unsupportedCases"]), - selectedIndices=verified["selectedIndices"], - ) - except (ValueError, AssertionError) as error: - row["verificationFailure"] = ( - str(error) or "Exact occurrence inventory mismatch" - ) - rows.append(row) - save( - "receipt.json", - { - "cases": rows, - "runtimeHashesUnchanged": all( - sha(ROOT / f) == h for f, h in runtime.items() - ), - "completeParentOccurrenceUnion": len(rows) == 2 - and sorted(verified_indices) == parent_indices, - "scope": "Diagnostic only; all other partitions remain unqualified", - }, - ) -assert all(sha(ROOT / f) == h for f, h in runtime.items()) -assert sha(reference) == REFERENCE_SHA -raise SystemExit( - 0 - if all(r["completeNumericalResult"] for r in rows) - and sorted(verified_indices) == parent_indices - else 1 -)