From 12d966e654cf3559b6ebeadac12c0f5b379ccd82 Mon Sep 17 00:00:00 2001 From: Zachary Ferguson Date: Mon, 12 Oct 2026 16:35:58 -0400 Subject: [PATCH 01/11] Fix the -march=native source in CI comments --- .github/workflows/continuous.yml | 8 ++++---- .github/workflows/coverage.yml | 8 ++++---- .github/workflows/cuda.yml | 8 ++++---- .github/workflows/docs.yml | 13 +++++++------ .github/workflows/python.yml | 8 ++++---- 5 files changed, 23 insertions(+), 22 deletions(-) diff --git a/.github/workflows/continuous.yml b/.github/workflows/continuous.yml index 56a8d5056..a31da758d 100644 --- a/.github/workflows/continuous.yml +++ b/.github/workflows/continuous.yml @@ -74,10 +74,10 @@ jobs: uses: SimenB/github-actions-cpu-cores@v2 id: cpu-cores - # FindSIMD, MeshFEMCore, and Tight-Inclusion all compile with - # -march=native, which bakes the building runner's ISA into every cached - # object file. ccache hashes the literal flag string, not what it expands - # to, so restored objects are only safe on a runner with the same ISA. + # FindSIMD and Tight-Inclusion both compile with -march=native, which + # bakes the building runner's ISA into every cached object file. ccache + # hashes the literal flag string, not what it expands to, so restored + # objects are only safe on a runner with the same ISA. # The CPU model name has not been enough to guarantee that: EPYC 9V74 # runners restored each other's objects and crashed with SIGILL. On Linux # we also hash the flags the compiler expands -march=native to, which is diff --git a/.github/workflows/coverage.yml b/.github/workflows/coverage.yml index c39c230a6..f0cb92454 100644 --- a/.github/workflows/coverage.yml +++ b/.github/workflows/coverage.yml @@ -37,10 +37,10 @@ jobs: uses: SimenB/github-actions-cpu-cores@v2 id: cpu-cores - # FindSIMD, MeshFEMCore, and Tight-Inclusion all compile with - # -march=native, which bakes the building runner's ISA into every cached - # object file. ccache hashes the literal flag string, not what it expands - # to, so restored objects are only safe on a runner with the same ISA. + # FindSIMD and Tight-Inclusion both compile with -march=native, which + # bakes the building runner's ISA into every cached object file. ccache + # hashes the literal flag string, not what it expands to, so restored + # objects are only safe on a runner with the same ISA. # The CPU model name has not been enough to guarantee that: EPYC 9V74 # runners restored each other's objects and crashed with SIGILL. We also # hash the flags the compiler expands -march=native to, which is exactly diff --git a/.github/workflows/cuda.yml b/.github/workflows/cuda.yml index 8f96fcf14..a18e0351c 100644 --- a/.github/workflows/cuda.yml +++ b/.github/workflows/cuda.yml @@ -51,10 +51,10 @@ jobs: uses: SimenB/github-actions-cpu-cores@v2 id: cpu-cores - # FindSIMD, MeshFEMCore, and Tight-Inclusion all compile with - # -march=native, which bakes the building runner's ISA into every cached - # object file. ccache hashes the literal flag string, not what it expands - # to, so restored objects are only safe on a runner with the same ISA. + # FindSIMD and Tight-Inclusion both compile with -march=native, which + # bakes the building runner's ISA into every cached object file. ccache + # hashes the literal flag string, not what it expands to, so restored + # objects are only safe on a runner with the same ISA. # The CPU model name has not been enough to guarantee that: EPYC 9V74 # runners restored each other's objects and crashed with SIGILL. We also # hash the flags the compiler expands -march=native to, which is exactly diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml index 280f68c22..371650f53 100644 --- a/.github/workflows/docs.yml +++ b/.github/workflows/docs.yml @@ -90,12 +90,13 @@ jobs: done cd .. - # We build ipctk without ccache on purpose. Its dependencies compile with - # -march=native, and ccache hashes that flag string, not the CPU it expands - # to. A cached object could therefore be restored onto a runner that lacks - # some of its instructions, which crashed `import ipctk` with SIGILL when - # Sphinx started. The tradeoff is a cold build of about 5 minutes instead of - # about 1, which is fine because docs only build on pushes to main. + # We build ipctk without ccache on purpose. It compiles with -march=native + # (from FindSIMD, plus Tight-Inclusion's own flags), and ccache hashes that + # flag string, not the CPU it expands to. A cached object could therefore + # be restored onto a runner that lacks some of its instructions, which + # crashed `import ipctk` with SIGILL when Sphinx started. The tradeoff is a + # cold build of about 5 minutes instead of about 1, which is fine because + # docs only build on pushes to main. - name: Build ipctk run: pip install --verbose . diff --git a/.github/workflows/python.yml b/.github/workflows/python.yml index 5f18ea154..c91bab426 100644 --- a/.github/workflows/python.yml +++ b/.github/workflows/python.yml @@ -64,10 +64,10 @@ jobs: choco install ccache "CACHE_PATH=${env:LOCALAPPDATA}\ccache" | Out-File -FilePath $env:GITHUB_ENV -Append - # FindSIMD, MeshFEMCore, and Tight-Inclusion all compile with - # -march=native, which bakes the building runner's ISA into every cached - # object file. ccache hashes the literal flag string, not what it expands - # to, so restored objects are only safe on a runner with the same ISA. + # FindSIMD and Tight-Inclusion both compile with -march=native, which + # bakes the building runner's ISA into every cached object file. ccache + # hashes the literal flag string, not what it expands to, so restored + # objects are only safe on a runner with the same ISA. # The CPU model name has not been enough to guarantee that: EPYC 9V74 # runners restored each other's objects and crashed with SIGILL. On Linux # we also hash the flags the compiler expands -march=native to, which is From d1e1e3ab7c47147656dba03b347d5fd581637d5b Mon Sep 17 00:00:00 2001 From: Zachary Ferguson Date: Mon, 12 Oct 2026 16:46:58 -0400 Subject: [PATCH 02/11] Build Linux CI for x86-64-v3 and share one cache - Pass -DSIMD_CXX_FLAGS=-march=x86-64-v3 in the Linux continuous and CUDA jobs so ccache's flag hash describes the generated code - Key those caches on x86-64-v3 instead of the CPU, so one cache serves every runner - Fail early if a runner lacks an x86-64-v3 feature - Keep coverage on -march=native so CI still compiles the default path Co-Authored-By: Claude Sonnet 5.5 --- .github/workflows/continuous.yml | 30 ++++++++++++++------------ .github/workflows/coverage.yml | 3 +++ .github/workflows/cuda.yml | 36 ++++++++++++++++---------------- .github/workflows/docs.yml | 12 +++++------ 4 files changed, 44 insertions(+), 37 deletions(-) diff --git a/.github/workflows/continuous.yml b/.github/workflows/continuous.yml index a31da758d..2de33faf3 100644 --- a/.github/workflows/continuous.yml +++ b/.github/workflows/continuous.yml @@ -74,23 +74,26 @@ jobs: uses: SimenB/github-actions-cpu-cores@v2 id: cpu-cores - # FindSIMD and Tight-Inclusion both compile with -march=native, which - # bakes the building runner's ISA into every cached object file. ccache - # hashes the literal flag string, not what it expands to, so restored - # objects are only safe on a runner with the same ISA. - # The CPU model name has not been enough to guarantee that: EPYC 9V74 - # runners restored each other's objects and crashed with SIGILL. On Linux - # we also hash the flags the compiler expands -march=native to, which is - # exactly what ends up in the object files. We keep the model name in the - # hash too, so the key is never weaker than before. - - name: CPU fingerprint (Linux/macOS) + # On Linux we build for x86-64-v3 (AVX2, FMA, BMI1/2, LZCNT, F16C) instead + # of -march=native. ccache hashes the literal flag string, not what + # -march=native expands to, so native objects are only safe on a runner + # with the same ISA, and EPYC 9V74 runners that restored each other's + # objects crashed with SIGILL. An explicit baseline makes the flag string + # describe the generated code, so one cache works on every runner CPU. + # Tight-Inclusion still adds -march=native for its own objects, but it + # also passes -mno-avx512f, which caps them at AVX2. coverage.yml keeps + # -march=native so CI still compiles the code users get by default. + # macOS and Windows keep a cache keyed on the CPU. + - name: CPU key (Linux/macOS) if: runner.os != 'Windows' run: | if [ "$RUNNER_OS" = "Linux" ]; then + for f in avx2 bmi1 bmi2 f16c fma abm movbe xsave; do + grep -qw "$f" /proc/cpuinfo || { echo "::error::Runner CPU lacks $f, which x86-64-v3 requires"; exit 1; } + done CPU=$(grep -m1 '^model name' /proc/cpuinfo | cut -d: -f2-) - ISA=$(echo | c++ -march=native -E -v - 2>&1 | grep -oE -- ' -m[a-z0-9=._+-]+' | sort -u | tr -d '\n') - echo "ISA:$ISA" - KEY=$(printf '%s\n%s' "$CPU" "$ISA" | sha256sum | cut -c1-12) + KEY=x86-64-v3 + echo "SIMD_ARGS=-DSIMD_CXX_FLAGS=-march=x86-64-v3" >> "$GITHUB_ENV" else CPU=$(sysctl -n machdep.cpu.brand_string) KEY=$(printf '%s' "$CPU" | shasum -a 256 | cut -c1-12) @@ -127,6 +130,7 @@ jobs: cd build cmake .. \ -DIPC_TOOLKIT_BUILD_TESTS=ON \ + $SIMD_ARGS \ -DCMAKE_BUILD_TYPE=${{ matrix.config }} - name: Configure (Windows) diff --git a/.github/workflows/coverage.yml b/.github/workflows/coverage.yml index f0cb92454..f4ba50ce9 100644 --- a/.github/workflows/coverage.yml +++ b/.github/workflows/coverage.yml @@ -46,6 +46,9 @@ jobs: # hash the flags the compiler expands -march=native to, which is exactly # what ends up in the object files. We keep the model name in the hash too, # so the key is never weaker than before. + # This job stays on -march=native on purpose, so CI still compiles the + # code users get by default. The other Linux jobs build for x86-64-v3 + # instead and share one cache across runner CPUs. - name: CPU fingerprint run: | CPU=$(grep -m1 '^model name' /proc/cpuinfo | cut -d: -f2-) diff --git a/.github/workflows/cuda.yml b/.github/workflows/cuda.yml index a18e0351c..a96a5c70b 100644 --- a/.github/workflows/cuda.yml +++ b/.github/workflows/cuda.yml @@ -51,32 +51,31 @@ jobs: uses: SimenB/github-actions-cpu-cores@v2 id: cpu-cores - # FindSIMD and Tight-Inclusion both compile with -march=native, which - # bakes the building runner's ISA into every cached object file. ccache - # hashes the literal flag string, not what it expands to, so restored - # objects are only safe on a runner with the same ISA. - # The CPU model name has not been enough to guarantee that: EPYC 9V74 - # runners restored each other's objects and crashed with SIGILL. We also - # hash the flags the compiler expands -march=native to, which is exactly - # what ends up in the object files. We keep the model name in the hash too, - # so the key is never weaker than before. - - name: CPU fingerprint + # We build for x86-64-v3 (AVX2, FMA, BMI1/2, LZCNT, F16C) instead of + # -march=native. ccache hashes the literal flag string, not what + # -march=native expands to, so native objects are only safe on a runner + # with the same ISA, and EPYC 9V74 runners that restored each other's + # objects crashed with SIGILL. An explicit baseline makes the flag string + # describe the generated code, so one cache works on every runner CPU. + # Tight-Inclusion still adds -march=native for its own objects, but it + # also passes -mno-avx512f, which caps them at AVX2. coverage.yml keeps + # -march=native so CI still compiles the code users get by default. + - name: Check x86-64-v3 support run: | - CPU=$(grep -m1 '^model name' /proc/cpuinfo | cut -d: -f2-) - ISA=$(echo | c++ -march=native -E -v - 2>&1 | grep -oE -- ' -m[a-z0-9=._+-]+' | sort -u | tr -d '\n') - echo "CPU: $CPU" - echo "ISA:$ISA" - echo "CPU_KEY=$(printf '%s\n%s' "$CPU" "$ISA" | sha256sum | cut -c1-12)" >> "$GITHUB_ENV" + for f in avx2 bmi1 bmi2 f16c fma abm movbe xsave; do + grep -qw "$f" /proc/cpuinfo || { echo "::error::Runner CPU lacks $f, which x86-64-v3 requires"; exit 1; } + done + echo "CPU: $(grep -m1 '^model name' /proc/cpuinfo | cut -d: -f2-)" # The CUDA key gets its own prefix because continuous.yml uses the same - # -- names, and cache entries are immutable. Whichever - # workflow saved first would otherwise own the entry for both builds. + # - names, and cache entries are immutable. Whichever workflow + # saved first would otherwise own the entry for both builds. - name: Cache Build id: cache-build uses: actions/cache@v5 with: path: ${{ env.CACHE_PATH }} - key: cuda-${{ runner.os }}-${{ matrix.config }}-${{ env.CPU_KEY }}-cache + key: cuda-${{ runner.os }}-${{ matrix.config }}-x86-64-v3-cache - name: Prepare ccache run: | @@ -90,6 +89,7 @@ jobs: cd build cmake .. \ -DIPC_TOOLKIT_WITH_CUDA=ON \ + -DSIMD_CXX_FLAGS=-march=x86-64-v3 \ -DCMAKE_CUDA_ARCHITECTURES=75 \ -DSCALABLE_CCD_CUDA_ARCHITECTURES=75 \ -DIPC_TOOLKIT_BUILD_TESTS=ON \ diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml index 371650f53..c781facb4 100644 --- a/.github/workflows/docs.yml +++ b/.github/workflows/docs.yml @@ -91,12 +91,12 @@ jobs: cd .. # We build ipctk without ccache on purpose. It compiles with -march=native - # (from FindSIMD, plus Tight-Inclusion's own flags), and ccache hashes that - # flag string, not the CPU it expands to. A cached object could therefore - # be restored onto a runner that lacks some of its instructions, which - # crashed `import ipctk` with SIGILL when Sphinx started. The tradeoff is a - # cold build of about 5 minutes instead of about 1, which is fine because - # docs only build on pushes to main. + # (from FindSIMD, plus Tight-Inclusion's own flags), and ccache hashes + # that flag string, not the CPU it expands to. A cached object could + # therefore be restored onto a runner that lacks some of its + # instructions, which crashed `import ipctk` with SIGILL when Sphinx + # started. The tradeoff is a cold build of about 5 minutes instead of + # about 1, which is fine because docs only build on pushes to main. - name: Build ipctk run: pip install --verbose . From d0a02329ff18e764c3724d4de463339f4eddd62a Mon Sep 17 00:00:00 2001 From: Zachary Ferguson Date: Mon, 12 Oct 2026 16:54:48 -0400 Subject: [PATCH 03/11] Drop the CPU key from the continuous and CUDA CI caches Co-Authored-By: Claude Sonnet 5.5 --- .github/workflows/continuous.yml | 38 ++++++++++---------------------- .github/workflows/cuda.yml | 2 +- 2 files changed, 13 insertions(+), 27 deletions(-) diff --git a/.github/workflows/continuous.yml b/.github/workflows/continuous.yml index 2de33faf3..6adfe30c3 100644 --- a/.github/workflows/continuous.yml +++ b/.github/workflows/continuous.yml @@ -83,39 +83,25 @@ jobs: # Tight-Inclusion still adds -march=native for its own objects, but it # also passes -mno-avx512f, which caps them at AVX2. coverage.yml keeps # -march=native so CI still compiles the code users get by default. - # macOS and Windows keep a cache keyed on the CPU. - - name: CPU key (Linux/macOS) - if: runner.os != 'Windows' - run: | - if [ "$RUNNER_OS" = "Linux" ]; then - for f in avx2 bmi1 bmi2 f16c fma abm movbe xsave; do - grep -qw "$f" /proc/cpuinfo || { echo "::error::Runner CPU lacks $f, which x86-64-v3 requires"; exit 1; } - done - CPU=$(grep -m1 '^model name' /proc/cpuinfo | cut -d: -f2-) - KEY=x86-64-v3 - echo "SIMD_ARGS=-DSIMD_CXX_FLAGS=-march=x86-64-v3" >> "$GITHUB_ENV" - else - CPU=$(sysctl -n machdep.cpu.brand_string) - KEY=$(printf '%s' "$CPU" | shasum -a 256 | cut -c1-12) - fi - echo "CPU: $CPU" - echo "CPU_KEY=$KEY" >> "$GITHUB_ENV" - - - name: CPU fingerprint (Windows) - if: runner.os == 'Windows' + # Windows already passes explicit /arch flags, so its objects do not + # depend on the runner either. macOS still builds with -march=native, + # but nearly every runner is the same Apple M1, so we share one cache + # there too. + - name: Check x86-64-v3 support (Linux) + if: runner.os == 'Linux' run: | - $cpu = Get-CimInstance Win32_Processor | Select-Object -First 1 -ExpandProperty Name - Write-Host "CPU: $cpu" - $stream = [IO.MemoryStream]::new([Text.Encoding]::UTF8.GetBytes($cpu)) - $key = (Get-FileHash -InputStream $stream -Algorithm SHA256).Hash.Substring(0, 12) - "CPU_KEY=$key" | Out-File -FilePath $env:GITHUB_ENV -Append + for f in avx2 bmi1 bmi2 f16c fma abm movbe xsave; do + grep -qw "$f" /proc/cpuinfo || { echo "::error::Runner CPU lacks $f, which x86-64-v3 requires"; exit 1; } + done + echo "CPU: $(grep -m1 '^model name' /proc/cpuinfo | cut -d: -f2-)" + echo "SIMD_ARGS=-DSIMD_CXX_FLAGS=-march=x86-64-v3" >> "$GITHUB_ENV" - name: Cache Build id: cache-build uses: actions/cache@v5 with: path: ${{ env.CACHE_PATH }} - key: ${{ runner.os }}-${{ matrix.config }}-${{ env.CPU_KEY }}-cache + key: ${{ runner.os }}-${{ matrix.config }}-cache - name: Prepare ccache run: | diff --git a/.github/workflows/cuda.yml b/.github/workflows/cuda.yml index a96a5c70b..a02d2b413 100644 --- a/.github/workflows/cuda.yml +++ b/.github/workflows/cuda.yml @@ -75,7 +75,7 @@ jobs: uses: actions/cache@v5 with: path: ${{ env.CACHE_PATH }} - key: cuda-${{ runner.os }}-${{ matrix.config }}-x86-64-v3-cache + key: cuda-${{ runner.os }}-${{ matrix.config }}-cache - name: Prepare ccache run: | From eb94291e0eefa80be2f3d04002400cb98c225e70 Mon Sep 17 00:00:00 2001 From: Zachary Ferguson Date: Mon, 12 Oct 2026 17:01:04 -0400 Subject: [PATCH 04/11] Build Linux wheels for x86-64-v3 Co-Authored-By: Claude Sonnet 5.5 --- .github/workflows/pypi.yml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/.github/workflows/pypi.yml b/.github/workflows/pypi.yml index 2eb4c112a..f9ccef82b 100644 --- a/.github/workflows/pypi.yml +++ b/.github/workflows/pypi.yml @@ -50,6 +50,11 @@ jobs: CIBW_ARCHS_MACOS: ${{ matrix.arch }} CIBW_BUILD: ${{ matrix.python-version }}-* CIBW_SKIP: "cp38-* pp* *musllinux* *manylinux1* *-win32 *linux_i686 *linux_aarch64 *linux_ppc64le *linux_s390x" + # Without this the wheel takes -march=native from whichever runner built it, + # and can contain AVX-512 that users' CPUs lack. x86-64-v3 is the baseline + # the 1.6.0 wheels already required (AVX2, FMA, BMI1/2, LZCNT). The build + # runs in a manylinux container, so it has to go through cibuildwheel. + CIBW_ENVIRONMENT_LINUX: CMAKE_ARGS="-DSIMD_CXX_FLAGS=-march=x86-64-v3" - uses: actions/upload-artifact@v7 with: From 1ebe0d70d5f49c049a358e299eee33757729b656 Mon Sep 17 00:00:00 2001 From: Zachary Ferguson Date: Mon, 12 Oct 2026 17:03:16 -0400 Subject: [PATCH 05/11] Drop the CPU key from the Python CI cache Co-Authored-By: Claude Sonnet 5.5 --- .github/workflows/python.yml | 39 ++++++------------------------------ 1 file changed, 6 insertions(+), 33 deletions(-) diff --git a/.github/workflows/python.yml b/.github/workflows/python.yml index c91bab426..361769c68 100644 --- a/.github/workflows/python.yml +++ b/.github/workflows/python.yml @@ -64,45 +64,18 @@ jobs: choco install ccache "CACHE_PATH=${env:LOCALAPPDATA}\ccache" | Out-File -FilePath $env:GITHUB_ENV -Append - # FindSIMD and Tight-Inclusion both compile with -march=native, which - # bakes the building runner's ISA into every cached object file. ccache - # hashes the literal flag string, not what it expands to, so restored - # objects are only safe on a runner with the same ISA. - # The CPU model name has not been enough to guarantee that: EPYC 9V74 - # runners restored each other's objects and crashed with SIGILL. On Linux - # we also hash the flags the compiler expands -march=native to, which is - # exactly what ends up in the object files. We keep the model name in the - # hash too, so the key is never weaker than before. - - name: CPU fingerprint (Linux/macOS) - if: runner.os != 'Windows' - run: | - if [ "$RUNNER_OS" = "Linux" ]; then - CPU=$(grep -m1 '^model name' /proc/cpuinfo | cut -d: -f2-) - ISA=$(echo | c++ -march=native -E -v - 2>&1 | grep -oE -- ' -m[a-z0-9=._+-]+' | sort -u | tr -d '\n') - echo "ISA:$ISA" - KEY=$(printf '%s\n%s' "$CPU" "$ISA" | sha256sum | cut -c1-12) - else - CPU=$(sysctl -n machdep.cpu.brand_string) - KEY=$(printf '%s' "$CPU" | shasum -a 256 | cut -c1-12) - fi - echo "CPU: $CPU" - echo "CPU_KEY=$KEY" >> "$GITHUB_ENV" - - - name: CPU fingerprint (Windows) - if: runner.os == 'Windows' - run: | - $cpu = Get-CimInstance Win32_Processor | Select-Object -First 1 -ExpandProperty Name - Write-Host "CPU: $cpu" - $stream = [IO.MemoryStream]::new([Text.Encoding]::UTF8.GetBytes($cpu)) - $key = (Get-FileHash -InputStream $stream -Algorithm SHA256).Hash.Substring(0, 12) - "CPU_KEY=$key" | Out-File -FilePath $env:GITHUB_ENV -Append + # We share one cache across runner CPUs. The Linux build sets + # IPCTK_WITH_SIMD=0, so only Tight-Inclusion still uses -march=native, and + # it also passes -mno-avx512f, which caps its objects at AVX2. Windows + # passes explicit /arch flags. macOS builds with -march=native, but nearly + # every runner is the same Apple M1. - name: Cache Build id: cache-build uses: actions/cache@v5 with: path: ${{ env.CACHE_PATH }} - key: ${{ runner.os }}-Python${{ matrix.python-version }}-${{ env.CPU_KEY }}-cache + key: ${{ runner.os }}-Python${{ matrix.python-version }}-cache - name: Prepare Ccache run: | From 91dea247de41ead6b41f4fe000e654cb0f03a6e1 Mon Sep 17 00:00:00 2001 From: Zachary Ferguson Date: Mon, 12 Oct 2026 17:04:44 -0400 Subject: [PATCH 06/11] Restore the docs ccache with an x86-64-v3 baseline Co-Authored-By: Claude Sonnet 5.5 --- .github/workflows/docs.yml | 44 +++++++++++++++++++++++++++++++------- 1 file changed, 36 insertions(+), 8 deletions(-) diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml index c781facb4..58b2bba7d 100644 --- a/.github/workflows/docs.yml +++ b/.github/workflows/docs.yml @@ -29,10 +29,26 @@ jobs: - name: Requirements run: | sudo apt-get update - sudo apt-get install doxygen python3-sphinx graphviz + sudo apt-get install doxygen python3-sphinx graphviz ccache + echo 'CACHE_PATH=~/.cache/ccache' >> "$GITHUB_ENV" python -m pip install --upgrade pip pip install -r docs/requirements.txt + - name: Cache Build + id: cache-build + uses: actions/cache@v5 + with: + path: ${{ env.CACHE_PATH }} + # A new name on purpose: entries are immutable, and the old docs-cache + # holds -march=native objects. + key: docs-ccache + + - name: Prepare Ccache + run: | + ccache --max-size=1.0G + ccache -V && ccache --show-config + ccache --show-stats && ccache --zero-stats + - name: Copy release docs run: | git fetch --all --tags @@ -90,15 +106,27 @@ jobs: done cd .. - # We build ipctk without ccache on purpose. It compiles with -march=native - # (from FindSIMD, plus Tight-Inclusion's own flags), and ccache hashes - # that flag string, not the CPU it expands to. A cached object could - # therefore be restored onto a runner that lacks some of its + # We build ipctk for x86-64-v3 instead of -march=native. ccache hashes the + # literal flag string, not what -march=native expands to, so a cached + # object could be restored onto a runner that lacks some of its # instructions, which crashed `import ipctk` with SIGILL when Sphinx - # started. The tradeoff is a cold build of about 5 minutes instead of - # about 1, which is fine because docs only build on pushes to main. + # started. An explicit baseline makes the flag string describe the + # generated code, so one cache works on every runner CPU. Tight-Inclusion + # still adds -march=native for its own objects, but it also passes + # -mno-avx512f, which caps them at AVX2. + - name: Check x86-64-v3 support + run: | + for f in avx2 bmi1 bmi2 f16c fma abm movbe xsave; do + grep -qw "$f" /proc/cpuinfo || { echo "::error::Runner CPU lacks $f, which x86-64-v3 requires"; exit 1; } + done + echo "CPU: $(grep -m1 '^model name' /proc/cpuinfo | cut -d: -f2-)" + - name: Build ipctk - run: pip install --verbose . + env: + CMAKE_ARGS: -DSIMD_CXX_FLAGS=-march=x86-64-v3 + run: | + pip install --verbose . + ccache --show-stats - name: Build docs run: | From 61c8cd8a6d80cda8502fba0fca6f03172cdd8244 Mon Sep 17 00:00:00 2001 From: Zachary Ferguson Date: Mon, 12 Oct 2026 17:05:15 -0400 Subject: [PATCH 07/11] Build the coverage job for x86-64-v3 and drop its CPU key Co-Authored-By: Claude Sonnet 5.5 --- .github/workflows/continuous.yml | 3 +-- .github/workflows/coverage.yml | 38 ++++++++++++++------------------ .github/workflows/cuda.yml | 3 +-- 3 files changed, 19 insertions(+), 25 deletions(-) diff --git a/.github/workflows/continuous.yml b/.github/workflows/continuous.yml index 6adfe30c3..ca3224ed6 100644 --- a/.github/workflows/continuous.yml +++ b/.github/workflows/continuous.yml @@ -81,8 +81,7 @@ jobs: # objects crashed with SIGILL. An explicit baseline makes the flag string # describe the generated code, so one cache works on every runner CPU. # Tight-Inclusion still adds -march=native for its own objects, but it - # also passes -mno-avx512f, which caps them at AVX2. coverage.yml keeps - # -march=native so CI still compiles the code users get by default. + # also passes -mno-avx512f, which caps them at AVX2. # Windows already passes explicit /arch flags, so its objects do not # depend on the runner either. macOS still builds with -march=native, # but nearly every runner is the same Apple M1, so we share one cache diff --git a/.github/workflows/coverage.yml b/.github/workflows/coverage.yml index f4ba50ce9..5d12eba51 100644 --- a/.github/workflows/coverage.yml +++ b/.github/workflows/coverage.yml @@ -37,38 +37,33 @@ jobs: uses: SimenB/github-actions-cpu-cores@v2 id: cpu-cores - # FindSIMD and Tight-Inclusion both compile with -march=native, which - # bakes the building runner's ISA into every cached object file. ccache - # hashes the literal flag string, not what it expands to, so restored - # objects are only safe on a runner with the same ISA. - # The CPU model name has not been enough to guarantee that: EPYC 9V74 - # runners restored each other's objects and crashed with SIGILL. We also - # hash the flags the compiler expands -march=native to, which is exactly - # what ends up in the object files. We keep the model name in the hash too, - # so the key is never weaker than before. - # This job stays on -march=native on purpose, so CI still compiles the - # code users get by default. The other Linux jobs build for x86-64-v3 - # instead and share one cache across runner CPUs. - - name: CPU fingerprint + # We build for x86-64-v3 (AVX2, FMA, BMI1/2, LZCNT, F16C) instead of + # -march=native. ccache hashes the literal flag string, not what + # -march=native expands to, so native objects are only safe on a runner + # with the same ISA, and EPYC 9V74 runners that restored each other's + # objects crashed with SIGILL. An explicit baseline makes the flag string + # describe the generated code, so one cache works on every runner CPU. + # Tight-Inclusion still adds -march=native for its own objects, but it + # also passes -mno-avx512f, which caps them at AVX2. + - name: Check x86-64-v3 support run: | - CPU=$(grep -m1 '^model name' /proc/cpuinfo | cut -d: -f2-) - ISA=$(echo | c++ -march=native -E -v - 2>&1 | grep -oE -- ' -m[a-z0-9=._+-]+' | sort -u | tr -d '\n') - echo "CPU: $CPU" - echo "ISA:$ISA" - echo "CPU_KEY=$(printf '%s\n%s' "$CPU" "$ISA" | sha256sum | cut -c1-12)" >> "$GITHUB_ENV" + for f in avx2 bmi1 bmi2 f16c fma abm movbe xsave; do + grep -qw "$f" /proc/cpuinfo || { echo "::error::Runner CPU lacks $f, which x86-64-v3 requires"; exit 1; } + done + echo "CPU: $(grep -m1 '^model name' /proc/cpuinfo | cut -d: -f2-)" - name: Cache Build id: cache-build uses: actions/cache@v5 with: path: ${{ env.CACHE_PATH }} - key: coverage-${{ env.CPU_KEY }}-cache + key: coverage-cache - name: Prepare ccache run: | ccache --max-size=1.0G ccache -V && ccache --show-config - ccache --show-stats && ccache --zero-stats\ + ccache --show-stats && ccache --zero-stats - name: Configure run: | @@ -77,7 +72,8 @@ jobs: cmake .. \ -DIPC_TOOLKIT_BUILD_TESTS=ON \ -DIPC_TOOLKIT_WITH_CODE_COVERAGE=ON \ - -DCMAKE_BUILD_TYPE=Release \ + -DSIMD_CXX_FLAGS=-march=x86-64-v3 \ + -DCMAKE_BUILD_TYPE=Release - name: Build run: | diff --git a/.github/workflows/cuda.yml b/.github/workflows/cuda.yml index a02d2b413..bf21c2def 100644 --- a/.github/workflows/cuda.yml +++ b/.github/workflows/cuda.yml @@ -58,8 +58,7 @@ jobs: # objects crashed with SIGILL. An explicit baseline makes the flag string # describe the generated code, so one cache works on every runner CPU. # Tight-Inclusion still adds -march=native for its own objects, but it - # also passes -mno-avx512f, which caps them at AVX2. coverage.yml keeps - # -march=native so CI still compiles the code users get by default. + # also passes -mno-avx512f, which caps them at AVX2. - name: Check x86-64-v3 support run: | for f in avx2 bmi1 bmi2 f16c fma abm movbe xsave; do From dab3556d09272f0cf58db0db366333ab2ba30d16 Mon Sep 17 00:00:00 2001 From: Zachary Ferguson Date: Mon, 12 Oct 2026 17:25:46 -0400 Subject: [PATCH 08/11] Enable SIMD for Linux Python CI with the x86-64-v3 pin Co-Authored-By: Claude Sonnet 5.5 --- .github/workflows/python.yml | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) diff --git a/.github/workflows/python.yml b/.github/workflows/python.yml index 361769c68..6ab37f141 100644 --- a/.github/workflows/python.yml +++ b/.github/workflows/python.yml @@ -64,11 +64,15 @@ jobs: choco install ccache "CACHE_PATH=${env:LOCALAPPDATA}\ccache" | Out-File -FilePath $env:GITHUB_ENV -Append - # We share one cache across runner CPUs. The Linux build sets - # IPCTK_WITH_SIMD=0, so only Tight-Inclusion still uses -march=native, and - # it also passes -mno-avx512f, which caps its objects at AVX2. Windows - # passes explicit /arch flags. macOS builds with -march=native, but nearly - # every runner is the same Apple M1. + # On Linux, explicitly build for x86-64-v3 instead of -march=native. This + # avoids issues with ccache restoring from a runner with a different ISA. + - name: Check x86-64-v3 support (Linux) + if: runner.os == 'Linux' + run: | + for f in avx2 bmi1 bmi2 f16c fma abm movbe xsave; do + grep -qw "$f" /proc/cpuinfo || { echo "::error::Runner CPU lacks $f, which x86-64-v3 requires"; exit 1; } + done + echo "CPU: $(grep -m1 '^model name' /proc/cpuinfo | cut -d: -f2-)" - name: Cache Build id: cache-build @@ -85,7 +89,7 @@ jobs: - name: Build and Install env: - IPCTK_WITH_SIMD: ${{ runner.os == 'Linux' && '0' || '1' }} + CMAKE_ARGS: ${{ runner.os == 'Linux' && '-DSIMD_CXX_FLAGS=-march=x86-64-v3' || '' }} run: | pip install --verbose . && ccache --show-stats From 6f18f18585c43f5fe3fa364bc345966e42ad711f Mon Sep 17 00:00:00 2001 From: Zachary Ferguson Date: Mon, 12 Oct 2026 17:26:49 -0400 Subject: [PATCH 09/11] Shorten the CI cache comments --- .github/workflows/continuous.yml | 14 ++------------ .github/workflows/coverage.yml | 10 ++-------- .github/workflows/cuda.yml | 10 ++-------- .github/workflows/docs.yml | 12 ++---------- 4 files changed, 8 insertions(+), 38 deletions(-) diff --git a/.github/workflows/continuous.yml b/.github/workflows/continuous.yml index ca3224ed6..3f19bca0b 100644 --- a/.github/workflows/continuous.yml +++ b/.github/workflows/continuous.yml @@ -74,18 +74,8 @@ jobs: uses: SimenB/github-actions-cpu-cores@v2 id: cpu-cores - # On Linux we build for x86-64-v3 (AVX2, FMA, BMI1/2, LZCNT, F16C) instead - # of -march=native. ccache hashes the literal flag string, not what - # -march=native expands to, so native objects are only safe on a runner - # with the same ISA, and EPYC 9V74 runners that restored each other's - # objects crashed with SIGILL. An explicit baseline makes the flag string - # describe the generated code, so one cache works on every runner CPU. - # Tight-Inclusion still adds -march=native for its own objects, but it - # also passes -mno-avx512f, which caps them at AVX2. - # Windows already passes explicit /arch flags, so its objects do not - # depend on the runner either. macOS still builds with -march=native, - # but nearly every runner is the same Apple M1, so we share one cache - # there too. + # On Linux, explicitly build for x86-64-v3 instead of -march=native. This + # avoids issues with ccache restoring from a runner with a different ISA. - name: Check x86-64-v3 support (Linux) if: runner.os == 'Linux' run: | diff --git a/.github/workflows/coverage.yml b/.github/workflows/coverage.yml index 5d12eba51..be126ddc9 100644 --- a/.github/workflows/coverage.yml +++ b/.github/workflows/coverage.yml @@ -37,14 +37,8 @@ jobs: uses: SimenB/github-actions-cpu-cores@v2 id: cpu-cores - # We build for x86-64-v3 (AVX2, FMA, BMI1/2, LZCNT, F16C) instead of - # -march=native. ccache hashes the literal flag string, not what - # -march=native expands to, so native objects are only safe on a runner - # with the same ISA, and EPYC 9V74 runners that restored each other's - # objects crashed with SIGILL. An explicit baseline makes the flag string - # describe the generated code, so one cache works on every runner CPU. - # Tight-Inclusion still adds -march=native for its own objects, but it - # also passes -mno-avx512f, which caps them at AVX2. + # On Linux, explicitly build for x86-64-v3 instead of -march=native. This + # avoids issues with ccache restoring from a runner with a different ISA. - name: Check x86-64-v3 support run: | for f in avx2 bmi1 bmi2 f16c fma abm movbe xsave; do diff --git a/.github/workflows/cuda.yml b/.github/workflows/cuda.yml index bf21c2def..ee18e4305 100644 --- a/.github/workflows/cuda.yml +++ b/.github/workflows/cuda.yml @@ -51,14 +51,8 @@ jobs: uses: SimenB/github-actions-cpu-cores@v2 id: cpu-cores - # We build for x86-64-v3 (AVX2, FMA, BMI1/2, LZCNT, F16C) instead of - # -march=native. ccache hashes the literal flag string, not what - # -march=native expands to, so native objects are only safe on a runner - # with the same ISA, and EPYC 9V74 runners that restored each other's - # objects crashed with SIGILL. An explicit baseline makes the flag string - # describe the generated code, so one cache works on every runner CPU. - # Tight-Inclusion still adds -march=native for its own objects, but it - # also passes -mno-avx512f, which caps them at AVX2. + # On Linux, explicitly build for x86-64-v3 instead of -march=native. This + # avoids issues with ccache restoring from a runner with a different ISA. - name: Check x86-64-v3 support run: | for f in avx2 bmi1 bmi2 f16c fma abm movbe xsave; do diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml index 58b2bba7d..a9f51ddba 100644 --- a/.github/workflows/docs.yml +++ b/.github/workflows/docs.yml @@ -39,8 +39,6 @@ jobs: uses: actions/cache@v5 with: path: ${{ env.CACHE_PATH }} - # A new name on purpose: entries are immutable, and the old docs-cache - # holds -march=native objects. key: docs-ccache - name: Prepare Ccache @@ -106,14 +104,8 @@ jobs: done cd .. - # We build ipctk for x86-64-v3 instead of -march=native. ccache hashes the - # literal flag string, not what -march=native expands to, so a cached - # object could be restored onto a runner that lacks some of its - # instructions, which crashed `import ipctk` with SIGILL when Sphinx - # started. An explicit baseline makes the flag string describe the - # generated code, so one cache works on every runner CPU. Tight-Inclusion - # still adds -march=native for its own objects, but it also passes - # -mno-avx512f, which caps them at AVX2. + # On Linux, explicitly build for x86-64-v3 instead of -march=native. This + # avoids issues with ccache restoring from a runner with a different ISA. - name: Check x86-64-v3 support run: | for f in avx2 bmi1 bmi2 f16c fma abm movbe xsave; do From 0e29fc93678652f6cc8aa188ba6bdbdf09b3a7ec Mon Sep 17 00:00:00 2001 From: Zachary Ferguson Date: Mon, 12 Oct 2026 17:38:58 -0400 Subject: [PATCH 10/11] Drop -g from the Debug CI builds Co-Authored-By: Claude Sonnet 5.5 --- .github/workflows/continuous.yml | 5 +++++ .github/workflows/cuda.yml | 8 ++++++++ 2 files changed, 13 insertions(+) diff --git a/.github/workflows/continuous.yml b/.github/workflows/continuous.yml index 3f19bca0b..42e65f3c1 100644 --- a/.github/workflows/continuous.yml +++ b/.github/workflows/continuous.yml @@ -98,6 +98,9 @@ jobs: ccache -V && ccache --show-config ccache --show-stats && ccache --zero-stats + # Debug keeps the asserts (NDEBUG stays undefined) and -O0 but drops -g. + # Debug info made the Debug ccache about 20x larger than Release, and CI + # never reads it: assert and Catch2 messages carry __FILE__ and __LINE__. - name: Configure (Linux/macOS) if: runner.os != 'Windows' run: | @@ -105,6 +108,8 @@ jobs: cd build cmake .. \ -DIPC_TOOLKIT_BUILD_TESTS=ON \ + -DCMAKE_C_FLAGS_DEBUG=-O0 \ + -DCMAKE_CXX_FLAGS_DEBUG=-O0 \ $SIMD_ARGS \ -DCMAKE_BUILD_TYPE=${{ matrix.config }} diff --git a/.github/workflows/cuda.yml b/.github/workflows/cuda.yml index ee18e4305..f8a907aad 100644 --- a/.github/workflows/cuda.yml +++ b/.github/workflows/cuda.yml @@ -76,6 +76,11 @@ jobs: ccache -V && ccache --show-config ccache --show-stats && ccache --zero-stats + # Debug keeps the asserts (NDEBUG stays undefined) and -O0 but drops -g. + # Debug info made the Debug ccache about 20x larger than Release, and CI + # never reads it: assert and Catch2 messages carry __FILE__ and __LINE__. + # The CUDA Debug flags are emptied instead of set to -O0, which nvcc would + # also apply to the device code. - name: Configure run: | mkdir -p build @@ -83,6 +88,9 @@ jobs: cmake .. \ -DIPC_TOOLKIT_WITH_CUDA=ON \ -DSIMD_CXX_FLAGS=-march=x86-64-v3 \ + -DCMAKE_C_FLAGS_DEBUG=-O0 \ + -DCMAKE_CXX_FLAGS_DEBUG=-O0 \ + -DCMAKE_CUDA_FLAGS_DEBUG= \ -DCMAKE_CUDA_ARCHITECTURES=75 \ -DSCALABLE_CCD_CUDA_ARCHITECTURES=75 \ -DIPC_TOOLKIT_BUILD_TESTS=ON \ From 93b414eaec3385abc4cda0c65147063f973ff723 Mon Sep 17 00:00:00 2001 From: Zachary Ferguson Date: Mon, 12 Oct 2026 17:52:36 -0400 Subject: [PATCH 11/11] Validate SIMD_CXX_FLAGS instead of re-detecting them - Skip find_package(SIMD) when SIMD_CXX_FLAGS is already set - Fail with an error if the compiler rejects the flags, instead of silently falling back to a build without SIMD - Add ipc_toolkit_check_cxx_flags() to test a flag list as a whole Co-Authored-By: Claude Sonnet 5.5 --- CMakeLists.txt | 34 +++++++++++++------ .../ipc_toolkit_filter_flags.cmake | 23 +++++++++++++ 2 files changed, 47 insertions(+), 10 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 206196e34..7f6c06c5d 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -170,17 +170,31 @@ endif() # SIMD support if(IPC_TOOLKIT_WITH_SIMD) - # Figure out SIMD support - message(STATUS "Testing SIMD capabilities...") - find_package(SIMD) - if (SIMD_CXX_FLAGS) - message(STATUS "SIMD support found: ${SIMD_CXX_FLAGS}") + if(SIMD_CXX_FLAGS) + # The flags were given on the command line, or cached by an earlier + # configure, so we use them as they are instead of probing the host. We only + # check that the compiler accepts them. If they are rejected we stop with an + # error instead of falling back to a build without SIMD, because a silent + # fallback would let a typo in the flags pass as a working non-SIMD build. + include(ipc_toolkit_filter_flags) + ipc_toolkit_check_cxx_flags(SIMD_CXX_FLAGS SIMD_CXX_FLAGS_ARE_VALID) + if(NOT SIMD_CXX_FLAGS_ARE_VALID) + message(FATAL_ERROR "SIMD_CXX_FLAGS=\"${SIMD_CXX_FLAGS}\" is not accepted by the compiler. Unset it (-USIMD_CXX_FLAGS) to auto-detect the flags, or pass -DIPC_TOOLKIT_WITH_SIMD=OFF to build without SIMD.") + endif() + message(STATUS "Using SIMD flags: ${SIMD_CXX_FLAGS}") else() - message(WARNING "SIMD support requested but not found. Continuing without SIMD.") - # We shadow the cache entry with a normal variable instead of forcing the - # cache off. Forcing would overwrite the user's request, and the option - # would stay OFF in this build directory even after SIMD becomes available. - set(IPC_TOOLKIT_WITH_SIMD OFF) + # Figure out SIMD support + message(STATUS "Testing SIMD capabilities...") + find_package(SIMD) + if (SIMD_CXX_FLAGS) + message(STATUS "SIMD support found: ${SIMD_CXX_FLAGS}") + else() + message(WARNING "SIMD support requested but not found. Continuing without SIMD.") + # We shadow the cache entry with a normal variable instead of forcing the + # cache off. Forcing would overwrite the user's request, and the option + # would stay OFF in this build directory even after SIMD becomes available. + set(IPC_TOOLKIT_WITH_SIMD OFF) + endif() endif() endif() diff --git a/cmake/ipc_toolkit/ipc_toolkit_filter_flags.cmake b/cmake/ipc_toolkit/ipc_toolkit_filter_flags.cmake index 36583d36e..cf971b37d 100644 --- a/cmake/ipc_toolkit/ipc_toolkit_filter_flags.cmake +++ b/cmake/ipc_toolkit/ipc_toolkit_filter_flags.cmake @@ -24,6 +24,29 @@ function(ipc_toolkit_filter_flags flags) set(${flags} ${output_flags} PARENT_SCOPE) endfunction() +# Check that the C++ compiler accepts all of `flags` together, and set `result` +# to TRUE or FALSE in the caller's scope. Both arguments are variable names. +# +# Unlike ipc_toolkit_filter_flags(), which tests each flag on its own and drops +# the ones it rejects, this tests the whole list at once and reports a single +# answer. We use it when the flags were chosen by the user, where silently +# dropping a flag would hide a mistake. +# +# We only compile a trivial program and never run it, so flags that target a +# different CPU than the one doing the build still pass. The answer is cached +# under a name built from the flags, so changing the flags re-runs the check +# instead of reusing an old result. +function(ipc_toolkit_check_cxx_flags flags result) + include(CheckCXXSourceCompiles) + string(MAKE_C_IDENTIFIER "${${flags}}" flags_id) + # CMAKE_REQUIRED_FLAGS wants one space-separated string, not a CMake list. We + # set it inside this function so it does not leak into the caller's scope. + string(REPLACE ";" " " CMAKE_REQUIRED_FLAGS "${${flags}}") + check_cxx_source_compiles( + "int main() { return 0; }" IPC_TOOLKIT_CXX_FLAGS_OK${flags_id}) + set(${result} ${IPC_TOOLKIT_CXX_FLAGS_OK${flags_id}} PARENT_SCOPE) +endfunction() + # The nvcc counterpart of ipc_toolkit_filter_flags(): keep the flags of `flags` # that nvcc's host compiler accepts, checked by actually compiling with nvcc and # `-Xcompiler=`, and wrap each so it applies only to CUDA sources compiled