From 9911399d9f2192971252273833f042a447943c86 Mon Sep 17 00:00:00 2001 From: Zachary Ferguson Date: Mon, 12 Oct 2026 14:49:18 -0400 Subject: [PATCH] Fix CI ccache SIGILL by keying on resolved ISA - Hash the flags `c++ -march=native` expands to, alongside the CPU model name, in the Linux ccache keys of all four workflows - Log the resolved ISA so a future failure shows which flags the runner had - Prefix the CUDA cache key so it no longer shares an entry with the Build workflow Co-Authored-By: Claude Sonnet 5.5 --- .github/workflows/continuous.yml | 19 ++++++++++++------- .github/workflows/coverage.yml | 20 ++++++++++++-------- .github/workflows/cuda.yml | 25 ++++++++++++++++--------- .github/workflows/python.yml | 20 ++++++++++++-------- 4 files changed, 52 insertions(+), 32 deletions(-) diff --git a/.github/workflows/continuous.yml b/.github/workflows/continuous.yml index 0442a24e2..56a8d5056 100644 --- a/.github/workflows/continuous.yml +++ b/.github/workflows/continuous.yml @@ -74,18 +74,23 @@ jobs: uses: SimenB/github-actions-cpu-cores@v2 id: cpu-cores - # FindSIMD compiles with -march=native, which bakes the building runner's - # ISA into every cached object file. The cache key had no CPU component, - # so objects built on one runner model were restored onto another and - # executed instructions it does not implement, failing the test step with - # SIGILL. Key the cache on the CPU so it only restores onto matching - # hardware. + # FindSIMD, MeshFEMCore, and Tight-Inclusion all compile with + # -march=native, which bakes the building runner's ISA into every cached + # object file. ccache hashes the literal flag string, not what it expands + # to, so restored objects are only safe on a runner with the same ISA. + # The CPU model name has not been enough to guarantee that: EPYC 9V74 + # runners restored each other's objects and crashed with SIGILL. On Linux + # we also hash the flags the compiler expands -march=native to, which is + # exactly what ends up in the object files. We keep the model name in the + # hash too, so the key is never weaker than before. - name: CPU fingerprint (Linux/macOS) if: runner.os != 'Windows' run: | if [ "$RUNNER_OS" = "Linux" ]; then CPU=$(grep -m1 '^model name' /proc/cpuinfo | cut -d: -f2-) - KEY=$(printf '%s' "$CPU" | sha256sum | cut -c1-12) + ISA=$(echo | c++ -march=native -E -v - 2>&1 | grep -oE -- ' -m[a-z0-9=._+-]+' | sort -u | tr -d '\n') + echo "ISA:$ISA" + KEY=$(printf '%s\n%s' "$CPU" "$ISA" | sha256sum | cut -c1-12) else CPU=$(sysctl -n machdep.cpu.brand_string) KEY=$(printf '%s' "$CPU" | shasum -a 256 | cut -c1-12) diff --git a/.github/workflows/coverage.yml b/.github/workflows/coverage.yml index 8a3c78854..c39c230a6 100644 --- a/.github/workflows/coverage.yml +++ b/.github/workflows/coverage.yml @@ -37,18 +37,22 @@ jobs: uses: SimenB/github-actions-cpu-cores@v2 id: cpu-cores - # FindSIMD compiles with -march=native, which bakes the building runner's - # ISA into every cached object file, but ccache hashes the flag string and - # not the CPU it expands to. Without a CPU component in the cache key, - # objects built on one runner model get restored onto another, and a - # translation unit compiled fresh on the new CPU then disagrees with the - # cached ones about xsimd::default_arch. Key the cache on the CPU so it - # only restores onto matching hardware. + # FindSIMD, MeshFEMCore, and Tight-Inclusion all compile with + # -march=native, which bakes the building runner's ISA into every cached + # object file. ccache hashes the literal flag string, not what it expands + # to, so restored objects are only safe on a runner with the same ISA. + # The CPU model name has not been enough to guarantee that: EPYC 9V74 + # runners restored each other's objects and crashed with SIGILL. We also + # hash the flags the compiler expands -march=native to, which is exactly + # what ends up in the object files. We keep the model name in the hash too, + # so the key is never weaker than before. - name: CPU fingerprint run: | CPU=$(grep -m1 '^model name' /proc/cpuinfo | cut -d: -f2-) + ISA=$(echo | c++ -march=native -E -v - 2>&1 | grep -oE -- ' -m[a-z0-9=._+-]+' | sort -u | tr -d '\n') echo "CPU: $CPU" - echo "CPU_KEY=$(printf '%s' "$CPU" | sha256sum | cut -c1-12)" >> "$GITHUB_ENV" + echo "ISA:$ISA" + echo "CPU_KEY=$(printf '%s\n%s' "$CPU" "$ISA" | sha256sum | cut -c1-12)" >> "$GITHUB_ENV" - name: Cache Build id: cache-build diff --git a/.github/workflows/cuda.yml b/.github/workflows/cuda.yml index ce36f64a3..8f96fcf14 100644 --- a/.github/workflows/cuda.yml +++ b/.github/workflows/cuda.yml @@ -51,25 +51,32 @@ jobs: uses: SimenB/github-actions-cpu-cores@v2 id: cpu-cores - # FindSIMD compiles with -march=native, which bakes the building runner's - # ISA into every cached object file, but ccache hashes the flag string and - # not the CPU it expands to. Without a CPU component in the cache key, - # objects built on one runner model get restored onto another, and a - # translation unit compiled fresh on the new CPU then disagrees with the - # cached ones about xsimd::default_arch. Key the cache on the CPU so it - # only restores onto matching hardware. + # FindSIMD, MeshFEMCore, and Tight-Inclusion all compile with + # -march=native, which bakes the building runner's ISA into every cached + # object file. ccache hashes the literal flag string, not what it expands + # to, so restored objects are only safe on a runner with the same ISA. + # The CPU model name has not been enough to guarantee that: EPYC 9V74 + # runners restored each other's objects and crashed with SIGILL. We also + # hash the flags the compiler expands -march=native to, which is exactly + # what ends up in the object files. We keep the model name in the hash too, + # so the key is never weaker than before. - name: CPU fingerprint run: | CPU=$(grep -m1 '^model name' /proc/cpuinfo | cut -d: -f2-) + ISA=$(echo | c++ -march=native -E -v - 2>&1 | grep -oE -- ' -m[a-z0-9=._+-]+' | sort -u | tr -d '\n') echo "CPU: $CPU" - echo "CPU_KEY=$(printf '%s' "$CPU" | sha256sum | cut -c1-12)" >> "$GITHUB_ENV" + echo "ISA:$ISA" + echo "CPU_KEY=$(printf '%s\n%s' "$CPU" "$ISA" | sha256sum | cut -c1-12)" >> "$GITHUB_ENV" + # The CUDA key gets its own prefix because continuous.yml uses the same + # -- names, and cache entries are immutable. Whichever + # workflow saved first would otherwise own the entry for both builds. - name: Cache Build id: cache-build uses: actions/cache@v5 with: path: ${{ env.CACHE_PATH }} - key: ${{ runner.os }}-${{ matrix.config }}-${{ env.CPU_KEY }}-cache + key: cuda-${{ runner.os }}-${{ matrix.config }}-${{ env.CPU_KEY }}-cache - name: Prepare ccache run: | diff --git a/.github/workflows/python.yml b/.github/workflows/python.yml index b9f529d98..5f18ea154 100644 --- a/.github/workflows/python.yml +++ b/.github/workflows/python.yml @@ -64,19 +64,23 @@ jobs: choco install ccache "CACHE_PATH=${env:LOCALAPPDATA}\ccache" | Out-File -FilePath $env:GITHUB_ENV -Append - # FindSIMD compiles with -march=native, which bakes the building runner's - # ISA into every cached object file, but ccache hashes the flag string and - # not the CPU it expands to. Without a CPU component in the cache key, - # objects built on one runner model get restored onto another, and a - # translation unit compiled fresh on the new CPU then disagrees with the - # cached ones about xsimd::default_arch. Key the cache on the CPU so it - # only restores onto matching hardware. + # FindSIMD, MeshFEMCore, and Tight-Inclusion all compile with + # -march=native, which bakes the building runner's ISA into every cached + # object file. ccache hashes the literal flag string, not what it expands + # to, so restored objects are only safe on a runner with the same ISA. + # The CPU model name has not been enough to guarantee that: EPYC 9V74 + # runners restored each other's objects and crashed with SIGILL. On Linux + # we also hash the flags the compiler expands -march=native to, which is + # exactly what ends up in the object files. We keep the model name in the + # hash too, so the key is never weaker than before. - name: CPU fingerprint (Linux/macOS) if: runner.os != 'Windows' run: | if [ "$RUNNER_OS" = "Linux" ]; then CPU=$(grep -m1 '^model name' /proc/cpuinfo | cut -d: -f2-) - KEY=$(printf '%s' "$CPU" | sha256sum | cut -c1-12) + ISA=$(echo | c++ -march=native -E -v - 2>&1 | grep -oE -- ' -m[a-z0-9=._+-]+' | sort -u | tr -d '\n') + echo "ISA:$ISA" + KEY=$(printf '%s\n%s' "$CPU" "$ISA" | sha256sum | cut -c1-12) else CPU=$(sysctl -n machdep.cpu.brand_string) KEY=$(printf '%s' "$CPU" | shasum -a 256 | cut -c1-12)