From c7c397191efb11d65869e4e7115c2761929ad7c4 Mon Sep 17 00:00:00 2001 From: Richard McDaniel <1130888+rmcdaniel@users.noreply.github.com> Date: Thu, 29 Oct 2026 23:26:51 +0000 Subject: [PATCH 1/3] Add exact-run native debug artifact controls --- scripts/ci/native-debug-artifact.mjs | 103 ++++++++++++++ scripts/ci/native-debug-build.sh | 51 +++++++ tests/Unit/NativeDebugArtifactTest.mjs | 184 +++++++++++++++++++++++++ 3 files changed, 338 insertions(+) create mode 100644 scripts/ci/native-debug-artifact.mjs create mode 100644 scripts/ci/native-debug-build.sh create mode 100644 tests/Unit/NativeDebugArtifactTest.mjs diff --git a/scripts/ci/native-debug-artifact.mjs b/scripts/ci/native-debug-artifact.mjs new file mode 100644 index 000000000..d2e0d02a6 --- /dev/null +++ b/scripts/ci/native-debug-artifact.mjs @@ -0,0 +1,103 @@ +import assert from 'node:assert/strict'; +import {execFileSync} from 'node:child_process'; +import {createHash} from 'node:crypto'; +import {lstatSync, readFileSync, readdirSync, writeFileSync} from 'node:fs'; +import {resolve} from 'node:path'; +import {fileURLToPath} from 'node:url'; + +const digest = value => createHash('sha256').update(value).digest('hex'); +const git = (root, ...args) => execFileSync('git', args, {cwd: root, encoding: 'utf8'}).trim(); +const required = (env, key, pattern) => { + const value = env[key]; + assert.equal(typeof value, 'string', `${key} is required`); + assert.match(value, pattern, `${key} is invalid`); + return value; +}; + +function identity(root, env, consumer) { + const source_revision = git(root, 'rev-parse', 'HEAD'); + assert.equal(required(env, 'GITHUB_SHA', /^[a-f0-9]{40}$/), source_revision, + 'native debug source does not match the workflow checkout'); + assert.equal(git(root, 'status', '--porcelain', '--untracked-files=all', '--', 'rust'), '', + 'native debug source must match the committed native tree'); + const attempt = required(env, 'GITHUB_RUN_ATTEMPT', /^[1-9][0-9]{0,9}$/); + const producerAttempt = consumer && env.NATIVE_BUILD_RUN_ATTEMPT !== undefined + ? required(env, 'NATIVE_BUILD_RUN_ATTEMPT', /^[1-9][0-9]{0,9}$/) : attempt; + assert.ok(Number(producerAttempt) <= Number(attempt), 'native debug producer attempt is in the future'); + return { + source_revision, + native_tree: git(root, 'rev-parse', 'HEAD:rust'), + cargo_lock_sha256: digest(readFileSync(resolve(root, 'rust/Cargo.lock'))), + compiler_image: required(env, 'NATIVE_RUST_IMAGE', /^mirror\.gcr\.io\/library\/rust@sha256:[a-f0-9]{64}$/), + repository: required(env, 'GITHUB_REPOSITORY', /^[A-Za-z0-9_.-]+\/[A-Za-z0-9_.-]+$/), + github_run_id: required(env, 'GITHUB_RUN_ID', /^[1-9][0-9]{0,19}$/), + github_run_attempt: producerAttempt, + runner_os: required(env, 'RUNNER_OS', /^Linux$/), + runner_arch: required(env, 'RUNNER_ARCH', /^(X64|ARM64)$/), + profile: 'dev', + locked: true, + jobs: 2, + resources: {nano_cpus: 2000000000, memory_bytes: 6442450944, memory_swap_bytes: 6442450944}, + }; +} + +function regularFile(path) { + assert.ok(lstatSync(path).isFile(), `native debug artifact requires a regular file: ${path}`); + return readFileSync(path); +} + +function checksums(directory) { + return `${digest(regularFile(resolve(directory, 'server')))} server\n` + + `${digest(regularFile(resolve(directory, 'build.json')))} build.json\n`; +} + +export function createArtifact(directory, compiler, seconds, {root = process.cwd(), env = process.env} = {}) { + assert.match(compiler, /^rustc [0-9]+\.[0-9]+\.[0-9]+[^\r\n]{0,200}$/, + 'native debug artifact needs the actual compiler version'); + assert.ok(Number.isSafeInteger(seconds) && seconds >= 0, 'invalid native debug build duration'); + const record = { + ...identity(root, env, false), + compiler, + binary_sha256: digest(regularFile(resolve(directory, 'server'))), + build_duration_seconds: seconds, + }; + writeFileSync(resolve(directory, 'build.json'), `${JSON.stringify(record, null, 2)}\n`); + writeFileSync(resolve(directory, 'SHA256SUMS'), checksums(directory)); + return record; +} + +export function verifyArtifact(directory, {root = process.cwd(), env = process.env} = {}) { + assert.ok(lstatSync(directory).isDirectory(), 'native debug artifact must be a directory'); + assert.deepEqual(readdirSync(directory).sort(), ['SHA256SUMS', 'build.json', 'server'], + 'native debug artifact must contain only the binary and its identity/checksums'); + const record = JSON.parse(regularFile(resolve(directory, 'build.json')).toString('utf8')); + const expected = identity(root, env, true); + for (const [key, value] of Object.entries(expected)) { + assert.deepEqual(record[key], value, `native debug artifact ${key} does not match this job`); + } + assert.match(record.compiler, /^rustc [0-9]+\.[0-9]+\.[0-9]+[^\r\n]{0,200}$/, + 'invalid native debug compiler version'); + assert.ok(Number.isSafeInteger(record.build_duration_seconds) && record.build_duration_seconds >= 0, + 'invalid native debug build duration'); + assert.equal(record.binary_sha256, digest(regularFile(resolve(directory, 'server'))), + 'native debug binary checksum mismatch'); + assert.equal(regularFile(resolve(directory, 'SHA256SUMS')).toString('utf8'), checksums(directory), + 'native debug artifact checksum list mismatch'); + return record; +} + +if (process.argv[1] && resolve(process.argv[1]) === fileURLToPath(import.meta.url)) { + try { + const [action, directory, compiler, seconds, ...extra] = process.argv.slice(2); + assert.ok(extra.length === 0 && (directory || action === 'identity'), + 'usage: native-debug-artifact.mjs identity | create DIR COMPILER SECONDS | verify DIR'); + const record = action === 'identity' && directory === undefined ? identity(process.cwd(), process.env, false) + : action === 'create' ? createArtifact(directory, compiler, Number(seconds)) + : action === 'verify' && compiler === undefined ? verifyArtifact(directory) : null; + assert.ok(record, 'unknown native debug artifact operation'); + process.stdout.write(`${JSON.stringify(record)}\n`); + } catch (error) { + process.stderr.write(`Native debug artifact refused: ${error.message}\n`); + process.exitCode = 1; + } +} diff --git a/scripts/ci/native-debug-build.sh b/scripts/ci/native-debug-build.sh new file mode 100644 index 000000000..227839f45 --- /dev/null +++ b/scripts/ci/native-debug-build.sh @@ -0,0 +1,51 @@ +#!/usr/bin/env bash +# Share the current run's debug server only. Native test builds keep their own path. +set -euo pipefail +test "$#" = 1 || exit 2 +root=$(git rev-parse --show-toplevel) +cd "$root" +image=${NATIVE_RUST_IMAGE:?NATIVE_RUST_IMAGE must be the pinned workflow compiler} +node scripts/ci/native-debug-artifact.mjs identity >/dev/null +scope="dw-native-debug-${GITHUB_RUN_ID:?}-${GITHUB_RUN_ATTEMPT:?}-${GITHUB_JOB:?}" +output=$(realpath -m "$1") +case "$output" in "$root"/*) ;; *) echo 'native debug output must be inside this checkout' >&2; exit 2 ;; esac +test ! -e "$output" +clean_cid="$output.clean.cid" +build_cid="$output.build.cid" +test ! -e "$clean_cid" +test ! -e "$build_cid" +mkdir -p "$output" cargo-home rust-target +cleanup() { + for receipt in "$clean_cid" "$build_cid"; do + if test -s "$receipt"; then + read -r container < "$receipt" || true + if [[ "$container" =~ ^[a-f0-9]{64}$ ]]; then + docker rm -f "$container" >/dev/null 2>&1 || true + fi + fi + rm -f -- "$receipt" + done +} +trap cleanup EXIT +trap 'exit 130' INT +trap 'exit 143' TERM +started=$SECONDS +docker run --rm --init --name "$scope-clean" --cidfile "$clean_cid" \ + --label "dw.task=server-native-debug-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}" \ + --user="$(id -u):$(id -g)" --network=none --cpus=2 --memory=2g --memory-swap=2g \ + -e CARGO_HOME=/cargo -e CARGO_TARGET_DIR=/target \ + -v "$root:/source:ro" -v "$root/cargo-home:/cargo" -v "$root/rust-target:/target" -w /source/rust \ + "$image" cargo clean --locked --offline --package durable-workflow-server +if ! docker run --rm --init --name "$scope-build" --cidfile "$build_cid" \ + --label "dw.task=server-native-debug-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}" \ + --user="$(id -u):$(id -g)" --network=none --cpus=2 --memory=6g --memory-swap=6g \ + -e CARGO_HOME=/cargo -e CARGO_TARGET_DIR=/target -e CARGO_BUILD_JOBS=2 \ + -v "$root:/source:ro" -v "$root/cargo-home:/cargo" -v "$root/rust-target:/target" -w /source/rust \ + "$image" bash -c 'set -e; rustc --version; cargo build --locked --offline --bin durable-workflow-server' \ + > "$output.build.txt" 2>&1; then + tail -n 40 "$output.build.txt" >&2 + exit 1 +fi +cp rust-target/debug/durable-workflow-server "$output/server" +compiler=$(sed -n '/^rustc [0-9]/{p;q;}' "$output.build.txt") +node scripts/ci/native-debug-artifact.mjs create "$output" "$compiler" "$((SECONDS-started))" diff --git a/tests/Unit/NativeDebugArtifactTest.mjs b/tests/Unit/NativeDebugArtifactTest.mjs new file mode 100644 index 000000000..3395dedde --- /dev/null +++ b/tests/Unit/NativeDebugArtifactTest.mjs @@ -0,0 +1,184 @@ +import assert from 'node:assert/strict'; +import {execFileSync, spawnSync} from 'node:child_process'; +import {createHash} from 'node:crypto'; +import {copyFileSync, existsSync, mkdirSync, mkdtempSync, readFileSync, rmSync, symlinkSync, writeFileSync} from 'node:fs'; +import {tmpdir} from 'node:os'; +import {dirname, join, resolve} from 'node:path'; +import test from 'node:test'; +import {fileURLToPath} from 'node:url'; +import {createArtifact, verifyArtifact} from '../../scripts/ci/native-debug-artifact.mjs'; + +const repository = resolve(dirname(fileURLToPath(import.meta.url)), '../..'); +const compiler = 'rustc 1.99.0 (test fixture)'; +const digest = path => createHash('sha256').update(readFileSync(path)).digest('hex'); + +function fixture(t) { + const root = mkdtempSync(join(tmpdir(), 'native-debug-artifact-')); + t.after(() => rmSync(root, {recursive: true, force: true})); + mkdirSync(join(root, 'rust/src'), {recursive: true}); + writeFileSync(join(root, 'rust/Cargo.lock'), '# fixture lock\n'); + writeFileSync(join(root, 'rust/Cargo.toml'), '[package]\nname="fixture"\nversion="0.0.0"\n'); + writeFileSync(join(root, 'rust/src/main.rs'), 'fn main() {}\n'); + mkdirSync(join(root, 'scripts/ci'), {recursive: true}); + for (const name of ['native-debug-artifact.mjs', 'native-debug-build.sh']) { + copyFileSync(join(repository, 'scripts/ci', name), join(root, 'scripts/ci', name)); + } + const git = (...args) => execFileSync('git', args, {cwd: root, encoding: 'utf8'}).trim(); + git('init', '-q'); + git('add', '.'); + git('-c', 'user.name=Fixture', '-c', 'user.email=fixture@example.invalid', 'commit', '-qm', 'fixture'); + const env = {...process.env, GITHUB_SHA: git('rev-parse', 'HEAD'), GITHUB_REPOSITORY: 'fixture/server', + GITHUB_RUN_ID: '12345', GITHUB_RUN_ATTEMPT: '1', GITHUB_JOB: 'native-debug-build', + NATIVE_RUST_IMAGE: `mirror.gcr.io/library/rust@sha256:${'a'.repeat(64)}`, RUNNER_OS: 'Linux', RUNNER_ARCH: 'X64'}; + delete env.NATIVE_BUILD_RUN_ATTEMPT; + const directory = join(root, 'artifact'); + mkdirSync(directory); + writeFileSync(join(directory, 'server'), 'not an executable: artifact-integrity test fixture\n'); + const options = {root, env}; + createArtifact(directory, compiler, 7, options); + return {root, directory, env, options}; +} + +test('a consumer accepts only the bound binary and identity files', t => { + const {directory, options} = fixture(t); + const actual = verifyArtifact(directory, options); + assert.equal(actual.build_duration_seconds, 7); + assert.equal(actual.profile, 'dev'); + assert.equal(actual.locked, true); +}); + +for (const [name, key, value] of [ + ['another workflow run', 'GITHUB_RUN_ID', '54321'], + ['another repository', 'GITHUB_REPOSITORY', 'other/server'], + ['another compiler', 'NATIVE_RUST_IMAGE', `mirror.gcr.io/library/rust@sha256:${'b'.repeat(64)}`], + ['another architecture', 'RUNNER_ARCH', 'ARM64'], + ['another producer attempt', 'GITHUB_RUN_ATTEMPT', '2'], + ['another checkout', 'GITHUB_SHA', '0'.repeat(40)], +]) { + test(`refuses ${name}`, t => { + const {directory, root, env} = fixture(t); + assert.throws(() => verifyArtifact(directory, {root, env: {...env, [key]: value}})); + }); +} + +test('a failed-job retry can use its successful producer output from this same run', t => { + const {directory, root, env} = fixture(t); + const retry = {...env, GITHUB_RUN_ATTEMPT: '2', NATIVE_BUILD_RUN_ATTEMPT: '1'}; + assert.equal(verifyArtifact(directory, {root, env: retry}).github_run_attempt, '1'); + assert.throws(() => verifyArtifact(directory, {root, env: {...retry, NATIVE_BUILD_RUN_ATTEMPT: '3'}})); +}); + +for (const [key, value] of [['native_tree', '0'.repeat(40)], ['cargo_lock_sha256', '0'.repeat(64)], + ['profile', 'release'], ['locked', false], ['jobs', 1], ['compiler', ''], + ['resources', {nano_cpus: 2000000000, memory_bytes: 128, memory_swap_bytes: 128}], + ['build_duration_seconds', -1]]) { + test(`refuses a changed ${key} even with a recomputed metadata checksum`, t => { + const {directory, options} = fixture(t); + const path = join(directory, 'build.json'); + const record = JSON.parse(readFileSync(path)); + record[key] = value; + writeFileSync(path, `${JSON.stringify(record)}\n`); + writeFileSync(join(directory, 'SHA256SUMS'), + `${digest(join(directory, 'server'))} server\n${digest(path)} build.json\n`); + const reason = key === 'build_duration_seconds' ? 'duration' : key; + assert.throws(() => verifyArtifact(directory, options), new RegExp(reason)); + }); +} + +test('refuses an edited native working tree before creating or accepting an artifact', t => { + const {directory, root, options} = fixture(t); + writeFileSync(join(root, 'rust/src/main.rs'), 'fn main() { panic!() }\n'); + assert.throws(() => verifyArtifact(directory, options), /committed native tree/); + assert.throws(() => createArtifact(directory, compiler, 1, options), /committed native tree/); +}); + +test('refuses binary tampering, checksum-list tampering and compiler-cache payloads', t => { + const {directory, options} = fixture(t); + writeFileSync(join(directory, 'server'), 'substituted binary\n'); + assert.throws(() => verifyArtifact(directory, options), /binary checksum/); + createArtifact(directory, compiler, 7, options); + writeFileSync(join(directory, 'SHA256SUMS'), 'untrusted list\n'); + assert.throws(() => verifyArtifact(directory, options), /checksum list/); + createArtifact(directory, compiler, 7, options); + mkdirSync(join(directory, 'target')); + assert.throws(() => verifyArtifact(directory, options), /only the binary/); +}); + +test('refuses symlinked binary and metadata files', t => { + const {root, directory, options} = fixture(t); + for (const name of ['server', 'build.json', 'SHA256SUMS']) { + const path = join(directory, name); + const outside = join(root, `${name}.outside`); + copyFileSync(path, outside); + rmSync(path); + symlinkSync(outside, path); + assert.throws(() => verifyArtifact(directory, options), /regular file/); + rmSync(path); + copyFileSync(outside, path); + } +}); + +test('identity preflight and existing-output guard refuse before any container starts', t => { + const {root, env, directory} = fixture(t); + const bin = fakeDocker(root); + const log = join(root, 'docker-calls.jsonl'); + const commandEnv = {...env, PATH: `${bin}:${env.PATH}`, FAKE_DOCKER_LOG: log}; + const args = ['scripts/ci/native-debug-build.sh', 'artifact']; + const existing = readFileSync(join(directory, 'server')); + const run = spawnSync('bash', args, {cwd: root, env: commandEnv, encoding: 'utf8'}); + assert.notEqual(run.status, 0); + assert.deepEqual(readFileSync(join(directory, 'server')), existing); + assert.equal(existsSync(log), false); + const badIdentity = spawnSync('bash', ['scripts/ci/native-debug-build.sh', 'other'], + {cwd: root, env: {...commandEnv, GITHUB_SHA: '0'.repeat(40)}, encoding: 'utf8'}); + assert.notEqual(badIdentity.status, 0); + assert.match(badIdentity.stderr, /workflow checkout/); + assert.equal(existsSync(log), false); +}); + +function fakeDocker(root) { + const bin = join(root, 'fake-bin'); + mkdirSync(bin); + writeFileSync(join(bin, 'docker'), `#!/usr/bin/env node +import('node:fs').then(async fs => { + const {createHash} = await import('node:crypto'); + const args = process.argv.slice(2); + fs.appendFileSync(process.env.FAKE_DOCKER_LOG, JSON.stringify(args) + '\\n'); + if (args[0] !== 'run') return; + const cid = createHash('sha256').update(args.join(' ')).digest('hex'); + fs.writeFileSync(args[args.indexOf('--cidfile') + 1], cid + '\\n'); + if (!args.some(arg => arg.includes('cargo build'))) return; + if (process.env.FAKE_BUILD_FAIL) process.exit(23); + const target = args.find(arg => arg.endsWith(':/target')).slice(0, -8); + fs.mkdirSync(target + '/debug', {recursive: true}); + fs.writeFileSync(target + '/debug/durable-workflow-server', 'fake compiled server\\n'); + process.stdout.write('rustc 1.99.0 (fixture)\\n'); +}); +`, {mode: 0o755}); + return bin; +} + +for (const failure of [false, true]) { + test(`the build helper ${failure ? 'cleans its own failed build' : 'exports only the thin artifact'}`, t => { + const {root, env} = fixture(t); + const bin = fakeDocker(root); + const log = join(root, 'docker-calls.jsonl'); + const commandEnv = {...env, PATH: `${bin}:${env.PATH}`, FAKE_DOCKER_LOG: log}; + if (failure) commandEnv.FAKE_BUILD_FAIL = '1'; + const run = spawnSync('bash', ['scripts/ci/native-debug-build.sh', 'bundle'], {cwd: root, env: commandEnv, encoding: 'utf8'}); + assert.equal(run.status === 0, !failure, run.stderr); + const calls = readFileSync(log, 'utf8').trim().split('\n').map(JSON.parse); + const builds = calls.filter(args => args[0] === 'run'); + assert.equal(builds.length, 2); + for (const args of builds) { + assert.ok(args.includes('--init') && args.includes('--network=none')); + assert.ok(args.includes('--cpus=2') && args.includes(env.NATIVE_RUST_IMAGE)); + assert.ok(args.includes(`--user=${process.getuid()}:${process.getgid()}`)); + } + assert.deepEqual(builds[0].slice(-6), ['cargo', 'clean', '--locked', '--offline', '--package', 'durable-workflow-server']); + assert.ok(builds[1].some(arg => arg.includes('cargo build --locked --offline --bin durable-workflow-server'))); + assert.ok(builds[1].includes('--memory=6g') && builds[1].includes('--memory-swap=6g')); + assert.equal(calls.filter(args => args[0] === 'rm' && /^[a-f0-9]{64}$/.test(args.at(-1))).length, 2); + if (!failure) verifyArtifact(join(root, 'bundle'), {root, env}); + }); +} From 2b11d99353259a437de9e4e79d414f9734198705 Mon Sep 17 00:00:00 2001 From: Richard McDaniel <1130888+rmcdaniel@users.noreply.github.com> Date: Thu, 29 Oct 2026 23:45:17 +0000 Subject: [PATCH 2/3] Share verified debug server across non-test parity jobs --- .github/workflows/server-parity.yml | 175 +++++++++++++++++++++-- scripts/ci/native-debug-build.sh | 1 + tests/Unit/NativeDebugArtifactTest.mjs | 135 +++++++++++++++++ tests/Unit/ServerParityAggregateTest.mjs | 23 ++- 4 files changed, 312 insertions(+), 22 deletions(-) diff --git a/.github/workflows/server-parity.yml b/.github/workflows/server-parity.yml index 7be89f569..911204849 100644 --- a/.github/workflows/server-parity.yml +++ b/.github/workflows/server-parity.yml @@ -21,6 +21,7 @@ on: - scripts/ci/pull-images.sh - scripts/ci/patched-pip.sh - scripts/ci/sdk-adapter-cache-key.sh + - tests/Unit/NativeDebugArtifactTest.mjs - config/dw-contract.php - tests/Fixtures/ActivityMetadataProjection/** - tests/Fixtures/CodecRegression/** @@ -155,6 +156,72 @@ jobs: MERGE_GROUP_HEAD_SHA: ${{ github.event.merge_group.head_sha }} run: bash scripts/conformance/server-parity/changes.sh >> "$GITHUB_OUTPUT" + # Only the server executable is shared. Native test parts retain their + # Cargo builds, full test commands and the trusted-main dependency writer. + native-debug-build: + needs: changes + if: ${{ needs.changes.outputs.run == 'true' }} + runs-on: ubuntu-latest + timeout-minutes: 25 + outputs: + artifact_name: ${{ steps.artifact.outputs.name }} + producer_attempt: ${{ steps.artifact.outputs.attempt }} + steps: + - uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 # v6 + with: + persist-credentials: false + - name: Pull the pinned compiler, retrying registry errors + run: bash scripts/ci/pull-images.sh "${NATIVE_RUST_IMAGE}" + - name: Validate native debug artifact and cleanup controls + run: node --test tests/Unit/NativeDebugArtifactTest.mjs + - name: Bind the artifact name to this producer attempt + id: artifact + run: | + printf 'name=native-debug-build-%s-%s\nattempt=%s\n' \ + "$GITHUB_RUN_ID" "$GITHUB_RUN_ATTEMPT" "$GITHUB_RUN_ATTEMPT" >> "$GITHUB_OUTPUT" + - name: Restore native Cargo dependencies from trusted main + uses: actions/cache/restore@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 + with: + path: | + cargo-home/registry + cargo-home/git + rust-target + key: native-cargo-deps-v2-${{ runner.os }}-${{ runner.arch }}-${{ env.NATIVE_RUST_IMAGE }}-${{ hashFiles('rust/Cargo.lock', 'rust/Cargo.toml', 'rust/vendor/**', 'rust/.cargo/**') }} + - name: Fetch locked Rust dependencies + run: | + set -euo pipefail + mkdir -p cargo-home + docker run --rm --init --user="$(id -u):$(id -g)" --cpus=2 --memory=2g --memory-swap=2g \ + -e CARGO_HOME=/cargo -e CARGO_NET_RETRY=10 \ + -v "$PWD:/source:ro" -v "$PWD/cargo-home:/cargo" -w /source/rust \ + "${NATIVE_RUST_IMAGE}" cargo fetch --locked + - name: Build one run-bound native debug server + run: bash scripts/ci/native-debug-build.sh native-debug-build + - name: Verify the thin artifact before sharing + run: node scripts/ci/native-debug-artifact.mjs verify native-debug-build + - name: Share this attempt's server without compiler caches + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 + with: + name: ${{ steps.artifact.outputs.name }} + path: | + native-debug-build/server + native-debug-build/build.json + native-debug-build/SHA256SUMS + retention-days: 1 + if-no-files-found: error + - name: Retire the owned debug build containers + if: always() + run: | + for receipt in native-debug-build.clean.cid native-debug-build.build.cid; do + if test -s "$receipt"; then + read -r container < "$receipt" || true + if [[ "$container" =~ ^[a-f0-9]{64}$ ]]; then + docker rm -f "$container" >/dev/null 2>&1 || true + fi + fi + rm -f -- "$receipt" + done + published-sdk-adapters: needs: changes if: ${{ needs.changes.outputs.run == 'true' }} @@ -379,7 +446,8 @@ jobs: # Named explicitly so the added native-tests-b combinations read like the # others instead of listing every matrix value. name: mysql-execution (${{ matrix.label }}, ${{ matrix.part }}) - needs: published-sdk-adapters + needs: [published-sdk-adapters, native-debug-build] + if: ${{ !cancelled() && needs.published-sdk-adapters.result == 'success' }} runs-on: ubuntu-latest # Each part takes about 20 to 35 minutes; the fixture step runs every # later profile before failing, and scenario budgets stay fixed. @@ -534,6 +602,34 @@ jobs: -e CARGO_HOME=/cargo -e CARGO_TARGET_DIR=/target \ -v "$PWD:/source:ro" -v "$PWD/cargo-home:/cargo" -v "$PWD/rust-target:/target" -w /source/rust \ "${NATIVE_RUST_IMAGE}" cargo clean --locked --offline --package durable-workflow-server + - name: Require a successful native debug producer + if: ${{ matrix.part != 'native-tests' && matrix.part != 'native-tests-b' }} + env: + NATIVE_BUILD_RESULT: ${{ needs.native-debug-build.result }} + NATIVE_BUILD_ARTIFACT: ${{ needs.native-debug-build.outputs.artifact_name }} + NATIVE_BUILD_RUN_ATTEMPT: ${{ needs.native-debug-build.outputs.producer_attempt }} + run: | + set -euo pipefail + test "$NATIVE_BUILD_RESULT" = success + [[ "$NATIVE_BUILD_RUN_ATTEMPT" =~ ^[1-9][0-9]{0,9}$ ]] + (( NATIVE_BUILD_RUN_ATTEMPT <= GITHUB_RUN_ATTEMPT )) + test "$NATIVE_BUILD_ARTIFACT" = "native-debug-build-$GITHUB_RUN_ID-$NATIVE_BUILD_RUN_ATTEMPT" + - name: Fetch this run's native debug server + if: ${{ matrix.part != 'native-tests' && matrix.part != 'native-tests-b' }} + uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8 + with: + name: ${{ needs.native-debug-build.outputs.artifact_name }} + path: native-debug-build + digest-mismatch: error + - name: Verify and install this run's native debug server + if: ${{ matrix.part != 'native-tests' && matrix.part != 'native-tests-b' }} + env: + NATIVE_BUILD_RUN_ATTEMPT: ${{ needs.native-debug-build.outputs.producer_attempt }} + run: | + set -euo pipefail + mkdir -p parity-evidence rust-target/debug + node scripts/ci/native-debug-artifact.mjs verify native-debug-build > parity-evidence/native-debug-build.json + install -m 0755 native-debug-build/server rust-target/debug/durable-workflow-server - name: Bootstrap published PHP and native MySQL databases and run the native tests run: | set -euo pipefail @@ -583,7 +679,7 @@ jobs: -v "$PWD:/source:ro" -v "$PWD/cargo-home:/cargo" -v "$PWD/rust-target:/target" \ -v "$PWD/parity-evidence:/evidence" -w /source/rust --entrypoint bash \ "${NATIVE_RUST_IMAGE}" -c \ - 'case "$DW_PARITY_PART" in native-tests) shard=1 suffix= ;; native-tests-b) shard=2 suffix=-b ;; *) shard= ;; esac; if [ "$DW_PARITY_PART" = native-tests ]; then cargo test --locked --offline --lib runtime::mysql:: -- --ignored --skip runtime::mysql::tests::bootstrap_child --nocapture --test-threads=2 > /evidence/mysql-tests.txt || exit 1; fi; if [ -n "$shard" ]; then cargo test --locked --offline --test execution -- --list --format terse | sed -n "s/: test\$//p" | LC_ALL=C sort | awk -v count="$DW_EXECUTION_SHARDS" -v shard="$shard" "(NR - 1) % count == shard - 1" > /evidence/execution-shard$suffix.txt && test -s /evidence/execution-shard$suffix.txt && cargo test --locked --offline --test execution -- --exact --nocapture --test-threads=2 $(cat /evidence/execution-shard$suffix.txt) > /evidence/mysql-execution-tests$suffix.txt || exit 1; fi; cargo build --locked --offline' + 'case "$DW_PARITY_PART" in native-tests) shard=1 suffix= ;; native-tests-b) shard=2 suffix=-b ;; *) shard= ;; esac; if [ "$DW_PARITY_PART" = native-tests ]; then cargo test --locked --offline --lib runtime::mysql:: -- --ignored --skip runtime::mysql::tests::bootstrap_child --nocapture --test-threads=2 > /evidence/mysql-tests.txt || exit 1; fi; if [ -n "$shard" ]; then cargo test --locked --offline --test execution -- --list --format terse | sed -n "s/: test\$//p" | LC_ALL=C sort | awk -v count="$DW_EXECUTION_SHARDS" -v shard="$shard" "(NR - 1) % count == shard - 1" > /evidence/execution-shard$suffix.txt && test -s /evidence/execution-shard$suffix.txt && cargo test --locked --offline --test execution -- --exact --nocapture --test-threads=2 $(cat /evidence/execution-shard$suffix.txt) > /evidence/mysql-execution-tests$suffix.txt || exit 1; fi; case "$DW_PARITY_PART" in native-tests|native-tests-b) cargo build --locked --offline ;; *) test -x /target/debug/durable-workflow-server ;; esac' docker run --rm --user="$(id -u):$(id -g)" --network "$RESOURCE_SCOPE" \ -e DW_RUST_EXPERIMENTAL=1 -e DB_CONNECTION=mysql -e DB_HOST=parity-mysql -e DB_DATABASE=native_ref \ -e DB_USERNAME=root -e DB_PASSWORD=parity-disposable-password \ @@ -1114,7 +1210,8 @@ jobs: retention-days: 1 postgres-storage: - needs: published-sdk-adapters + needs: [published-sdk-adapters, native-debug-build] + if: ${{ !cancelled() && needs.published-sdk-adapters.result == 'success' }} runs-on: ubuntu-latest # Each part takes about 20 to 35 minutes; the fixture step runs every # later profile before failing, and scenario budgets stay fixed. @@ -1215,6 +1312,34 @@ jobs: -e CARGO_HOME=/cargo -e CARGO_TARGET_DIR=/target \ -v "$PWD:/source:ro" -v "$PWD/cargo-home:/cargo" -v "$PWD/rust-target:/target" -w /source/rust \ "${NATIVE_RUST_IMAGE}" cargo clean --locked --offline --package durable-workflow-server + - name: Require a successful native debug producer + if: ${{ matrix.part != 'native-tests' && matrix.part != 'native-tests-b' }} + env: + NATIVE_BUILD_RESULT: ${{ needs.native-debug-build.result }} + NATIVE_BUILD_ARTIFACT: ${{ needs.native-debug-build.outputs.artifact_name }} + NATIVE_BUILD_RUN_ATTEMPT: ${{ needs.native-debug-build.outputs.producer_attempt }} + run: | + set -euo pipefail + test "$NATIVE_BUILD_RESULT" = success + [[ "$NATIVE_BUILD_RUN_ATTEMPT" =~ ^[1-9][0-9]{0,9}$ ]] + (( NATIVE_BUILD_RUN_ATTEMPT <= GITHUB_RUN_ATTEMPT )) + test "$NATIVE_BUILD_ARTIFACT" = "native-debug-build-$GITHUB_RUN_ID-$NATIVE_BUILD_RUN_ATTEMPT" + - name: Fetch this run's native debug server + if: ${{ matrix.part != 'native-tests' && matrix.part != 'native-tests-b' }} + uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8 + with: + name: ${{ needs.native-debug-build.outputs.artifact_name }} + path: native-debug-build + digest-mismatch: error + - name: Verify and install this run's native debug server + if: ${{ matrix.part != 'native-tests' && matrix.part != 'native-tests-b' }} + env: + NATIVE_BUILD_RUN_ATTEMPT: ${{ needs.native-debug-build.outputs.producer_attempt }} + run: | + set -euo pipefail + mkdir -p parity-evidence rust-target/debug + node scripts/ci/native-debug-artifact.mjs verify native-debug-build > parity-evidence/native-debug-build.json + install -m 0755 native-debug-build/server rust-target/debug/durable-workflow-server - name: Bootstrap published PHP and native PostgreSQL databases and run the native tests env: POSTGRES_IMAGE: ${{ matrix.image }} @@ -1263,7 +1388,7 @@ jobs: -v "$PWD:/source:ro" -v "$PWD/cargo-home:/cargo" -v "$PWD/rust-target:/target" \ -v "$PWD/parity-evidence:/evidence" -w /source/rust --entrypoint bash \ "${NATIVE_RUST_IMAGE}" -c \ - 'if [ "$DW_PARITY_PART" = native-tests ]; then cargo test --locked --offline --lib runtime::postgres::tests::qualification_ -- --ignored --nocapture > /evidence/postgres-tests.txt && cargo test --locked --offline --test execution -- --nocapture > /evidence/postgres-execution-tests.txt; fi && cargo build --locked --offline' + 'if [ "$DW_PARITY_PART" = native-tests ]; then cargo test --locked --offline --lib runtime::postgres::tests::qualification_ -- --ignored --nocapture > /evidence/postgres-tests.txt && cargo test --locked --offline --test execution -- --nocapture > /evidence/postgres-execution-tests.txt; fi && if [ "$DW_PARITY_PART" = native-tests ]; then cargo build --locked --offline; else test -x /target/debug/durable-workflow-server; fi' docker run --rm --user="$(id -u):$(id -g)" --network "$RESOURCE_SCOPE" \ -e DW_RUST_EXPERIMENTAL=1 -e DB_CONNECTION=pgsql -e DB_HOST=parity-pg -e DB_DATABASE=native_ref \ -e DB_USERNAME=parity -e DB_PASSWORD=parity-disposable-password \ @@ -1842,7 +1967,8 @@ jobs: retention-days: 1 execution-and-codec: - needs: published-sdk-adapters + needs: [published-sdk-adapters, native-debug-build] + if: ${{ !cancelled() && needs.published-sdk-adapters.result == 'success' }} runs-on: ubuntu-latest # Each part takes about 20 to 35 minutes; the fixture step runs every # later profile before failing, and scenario budgets stay fixed. @@ -1852,8 +1978,8 @@ jobs: # Six parts on separate runners: the native suites and codec, the # shared corpus, its profiles and restart cases, the published-SDK # direction profiles, the corrected marker cohort, and the other - # cohorts. Every part builds the same native server; no part depends on - # another. + # cohorts. Non-test parts use this run's verified debug server artifact; + # native-tests still builds and tests from source. matrix: part: [native-tests, corpus, restarts, cohorts, sdk-directions, corrected-markers] env: @@ -1934,16 +2060,34 @@ jobs: -e CARGO_HOME=/cargo -e CARGO_TARGET_DIR=/target \ -v "$PWD:/source:ro" -v "$PWD/cargo-home:/cargo" -v "$PWD/rust-target:/target" -w /source/rust \ "${NATIVE_RUST_IMAGE}" cargo clean --locked --offline --package durable-workflow-server - - name: Build the native server - if: ${{ matrix.part != 'native-tests' }} + - name: Require a successful native debug producer + if: ${{ matrix.part != 'native-tests' && matrix.part != 'native-tests-b' }} + env: + NATIVE_BUILD_RESULT: ${{ needs.native-debug-build.result }} + NATIVE_BUILD_ARTIFACT: ${{ needs.native-debug-build.outputs.artifact_name }} + NATIVE_BUILD_RUN_ATTEMPT: ${{ needs.native-debug-build.outputs.producer_attempt }} run: | set -euo pipefail - mkdir -p cargo-home rust-target parity-evidence - docker run --rm --user="$(id -u):$(id -g)" --cpus=2 --memory=6g --memory-swap=6g \ - -e CARGO_HOME=/cargo -e CARGO_TARGET_DIR=/target -e CARGO_BUILD_JOBS=2 \ - -v "$PWD:/source:ro" -v "$PWD/cargo-home:/cargo" -v "$PWD/rust-target:/target" -w /source/rust \ - "${NATIVE_RUST_IMAGE}" \ - cargo build --locked --offline + test "$NATIVE_BUILD_RESULT" = success + [[ "$NATIVE_BUILD_RUN_ATTEMPT" =~ ^[1-9][0-9]{0,9}$ ]] + (( NATIVE_BUILD_RUN_ATTEMPT <= GITHUB_RUN_ATTEMPT )) + test "$NATIVE_BUILD_ARTIFACT" = "native-debug-build-$GITHUB_RUN_ID-$NATIVE_BUILD_RUN_ATTEMPT" + - name: Fetch this run's native debug server + if: ${{ matrix.part != 'native-tests' && matrix.part != 'native-tests-b' }} + uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8 + with: + name: ${{ needs.native-debug-build.outputs.artifact_name }} + path: native-debug-build + digest-mismatch: error + - name: Verify and install this run's native debug server + if: ${{ matrix.part != 'native-tests' && matrix.part != 'native-tests-b' }} + env: + NATIVE_BUILD_RUN_ATTEMPT: ${{ needs.native-debug-build.outputs.producer_attempt }} + run: | + set -euo pipefail + mkdir -p parity-evidence rust-target/debug + node scripts/ci/native-debug-artifact.mjs verify native-debug-build > parity-evidence/native-debug-build.json + install -m 0755 native-debug-build/server rust-target/debug/durable-workflow-server - name: Execute official Rust codec and cross-decode with published PHP if: ${{ matrix.part == 'native-tests' }} run: | @@ -2524,6 +2668,7 @@ jobs: - changes - published-sdk-adapters - codec-native-build + - native-debug-build - codec-deadline-health - mysql-execution - postgres-storage diff --git a/scripts/ci/native-debug-build.sh b/scripts/ci/native-debug-build.sh index 227839f45..fee3495f5 100644 --- a/scripts/ci/native-debug-build.sh +++ b/scripts/ci/native-debug-build.sh @@ -47,5 +47,6 @@ if ! docker run --rm --init --name "$scope-build" --cidfile "$build_cid" \ exit 1 fi cp rust-target/debug/durable-workflow-server "$output/server" +cat "$output.build.txt" compiler=$(sed -n '/^rustc [0-9]/{p;q;}' "$output.build.txt") node scripts/ci/native-debug-artifact.mjs create "$output" "$compiler" "$((SECONDS-started))" diff --git a/tests/Unit/NativeDebugArtifactTest.mjs b/tests/Unit/NativeDebugArtifactTest.mjs index 3395dedde..1e2ecc165 100644 --- a/tests/Unit/NativeDebugArtifactTest.mjs +++ b/tests/Unit/NativeDebugArtifactTest.mjs @@ -11,6 +11,12 @@ import {createArtifact, verifyArtifact} from '../../scripts/ci/native-debug-arti const repository = resolve(dirname(fileURLToPath(import.meta.url)), '../..'); const compiler = 'rustc 1.99.0 (test fixture)'; const digest = path => createHash('sha256').update(readFileSync(path)).digest('hex'); +const workflow = readFileSync(join(repository, '.github/workflows/server-parity.yml'), 'utf8'); +function workflowJob(id) { + const match = new RegExp(`\\n ${id}:\\n([\\s\\S]*?)(?=\\n [a-z0-9-]+:|$)`).exec(workflow); + assert.ok(match, `missing workflow job ${id}`); + return match[1]; +} function fixture(t) { const root = mkdtempSync(join(tmpdir(), 'native-debug-artifact-')); @@ -182,3 +188,132 @@ for (const failure of [false, true]) { if (!failure) verifyArtifact(join(root, 'bundle'), {root, env}); }); } + +test('the producer shares only three files with pinned actions and bounded retention', () => { + const producer = workflowJob('native-debug-build'); + assert.match(producer, /needs: changes\n {4}if: \$\{\{ needs\.changes\.outputs\.run == 'true' \}\}/); + assert.match(producer, /persist-credentials: false/); + assert.match(producer, /artifact_name: \$\{\{ steps\.artifact\.outputs\.name \}\}/); + assert.match(producer, /producer_attempt: \$\{\{ steps\.artifact\.outputs\.attempt \}\}/); + assert.match(producer, /native-debug-build-%s-%s/); + assert.match(producer, /bash scripts\/ci\/native-debug-build\.sh native-debug-build/); + assert.match(producer, /node scripts\/ci\/native-debug-artifact\.mjs verify native-debug-build/); + assert.match(producer, /actions\/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a/); + const paths = producer.match(/ path: \|\n((?: {12}.+\n)+)/g).at(-1); + assert.equal(paths, ' path: |\n native-debug-build/server\n' + + ' native-debug-build/build.json\n native-debug-build/SHA256SUMS\n'); + assert.match(producer, /retention-days: 1\n {10}if-no-files-found: error/); + assert.doesNotMatch(producer, /cache\/save|contents: write|packages: write/); +}); + +test('all non-test consumers verify the same-run artifact before installing the executable', () => { + for (const id of ['mysql-execution', 'postgres-storage', 'execution-and-codec']) { + const block = workflowJob(id); + assert.match(block, /needs: \[published-sdk-adapters, native-debug-build\]/); + assert.match(block, /if: \$\{\{ !cancelled\(\) && needs\.published-sdk-adapters\.result == 'success' \}\}/); + const download = block.slice(block.indexOf(" - name: Fetch this run's native debug server"), + block.indexOf(" - name: Verify and install this run's native debug server")); + assert.match(download, /matrix\.part != 'native-tests' && matrix\.part != 'native-tests-b'/); + assert.match(download, /actions\/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c/); + assert.match(download, /name: \$\{\{ needs\.native-debug-build\.outputs\.artifact_name \}\}/); + assert.match(download, /digest-mismatch: error/); + assert.doesNotMatch(download, /continue-on-error|run-id:|repository:/); + assert.match(block, /NATIVE_BUILD_RUN_ATTEMPT: \$\{\{ needs\.native-debug-build\.outputs\.producer_attempt \}\}/); + const verify = block.indexOf('node scripts/ci/native-debug-artifact.mjs verify native-debug-build'); + const install = block.indexOf('install -m 0755 native-debug-build/server rust-target/debug/durable-workflow-server'); + assert.ok(verify > 0 && install > verify, id); + } +}); + +test('consumer preflight refuses a failed producer or missing identity before download', () => { + for (const id of ['mysql-execution', 'postgres-storage', 'execution-and-codec']) { + const block = workflowJob(id); + const guard = block.slice(block.indexOf(' - name: Require a successful native debug producer'), + block.indexOf(" - name: Fetch this run's native debug server")); + assert.match(guard, /matrix\.part != 'native-tests' && matrix\.part != 'native-tests-b'/); + assert.match(guard, /NATIVE_BUILD_RESULT: \$\{\{ needs\.native-debug-build\.result \}\}/); + const body = guard.match(/ run: \|\n((?: {10}[^\n]*\n)+)/)?.[1] + .split('\n').map(line => line.slice(10)).join('\n'); + assert.ok(body); + const env = {...process.env, GITHUB_RUN_ID: '12345', GITHUB_RUN_ATTEMPT: '2', + NATIVE_BUILD_RESULT: 'success', NATIVE_BUILD_ARTIFACT: 'native-debug-build-12345-1', + NATIVE_BUILD_RUN_ATTEMPT: '1'}; + const run = changes => spawnSync('bash', ['-c', body], {env: {...env, ...changes}, encoding: 'utf8'}); + assert.equal(run({}).status, 0); + for (const changes of [{NATIVE_BUILD_RESULT: 'failure'}, {NATIVE_BUILD_RESULT: 'skipped'}, + {NATIVE_BUILD_RESULT: 'cancelled'}, {NATIVE_BUILD_ARTIFACT: ''}, + {NATIVE_BUILD_ARTIFACT: 'native-debug-build-99999-1'}, {NATIVE_BUILD_RUN_ATTEMPT: ''}, + {NATIVE_BUILD_RUN_ATTEMPT: '3'}, {NATIVE_BUILD_RUN_ATTEMPT: '01'}, + {NATIVE_BUILD_RUN_ATTEMPT: '10000000000'}]) { + assert.notEqual(run(changes).status, 0, `${id}: ${JSON.stringify(changes)}`); + } + } +}); + +function databaseCommand(id) { + const line = workflowJob(id).split('\n').find(value => value.trimStart().startsWith( + id === 'mysql-execution' ? "'case \"$DW_PARITY_PART\"" : "'if [ \"$DW_PARITY_PART\"")); + assert.ok(line?.trim().endsWith("'")); + return line.trim().slice(1, -1); +} + +function runDatabaseCommand(t, id, part, binary = true) { + const root = mkdtempSync(join(tmpdir(), 'native-debug-cargo-')); + t.after(() => rmSync(root, {recursive: true, force: true})); + mkdirSync(join(root, 'bin')); + mkdirSync(join(root, 'evidence')); + mkdirSync(join(root, 'target/debug'), {recursive: true}); + if (binary) writeFileSync(join(root, 'target/debug/durable-workflow-server'), 'fixture\n', {mode: 0o755}); + const log = join(root, 'cargo.jsonl'); + writeFileSync(log, ''); + writeFileSync(join(root, 'bin/cargo'), `#!/usr/bin/env node +const fs = require('node:fs'); +const args = process.argv.slice(2); +fs.appendFileSync(process.env.CARGO_CALL_LOG, JSON.stringify(args) + '\\n'); +if(args.includes('--list')) console.log('case_a: test\\ncase_b: test'); +`, {mode: 0o755}); + const script = databaseCommand(id).replaceAll('/evidence/', `${root}/evidence/`).replaceAll('/target/', `${root}/target/`); + const run = spawnSync('bash', ['-c', script], {encoding: 'utf8', env: {...process.env, + PATH: `${root}/bin:${process.env.PATH}`, CARGO_CALL_LOG: log, DW_PARITY_PART: part, DW_EXECUTION_SHARDS: '2'}}); + const calls = readFileSync(log, 'utf8').trim().split('\n').filter(Boolean).map(JSON.parse); + return {...run, calls}; +} + +for (const part of ['native-tests', 'native-tests-b']) { + test(`MySQL ${part} retains the original Cargo tests, sharding and build`, t => { + const run = runDatabaseCommand(t, 'mysql-execution', part); + assert.equal(run.status, 0, run.stderr); + const flags = ['--locked', '--offline']; + const expected = part === 'native-tests' ? [ + ['test', ...flags, '--lib', 'runtime::mysql::', '--', '--ignored', '--skip', + 'runtime::mysql::tests::bootstrap_child', '--nocapture', '--test-threads=2'], + ] : []; + expected.push(['test', ...flags, '--test', 'execution', '--', '--list', '--format', 'terse'], + ['test', ...flags, '--test', 'execution', '--', '--exact', '--nocapture', '--test-threads=2', + part === 'native-tests' ? 'case_a' : 'case_b'], ['build', ...flags]); + assert.deepEqual(run.calls, expected); + }); +} + +test('PostgreSQL native tests retain qualification, execution and build commands', t => { + const run = runDatabaseCommand(t, 'postgres-storage', 'native-tests'); + assert.equal(run.status, 0, run.stderr); + const flags = ['--locked', '--offline']; + assert.deepEqual(run.calls, [ + ['test', ...flags, '--lib', 'runtime::postgres::tests::qualification_', '--', '--ignored', '--nocapture'], + ['test', ...flags, '--test', 'execution', '--', '--nocapture'], ['build', ...flags], + ]); +}); + +test('database corpus/profile parts never invoke Cargo and fail without their executable', t => { + for (const id of ['mysql-execution', 'postgres-storage']) { + for (const part of ['corpus', 'restarts', 'cohorts', 'sdk-directions', 'corrected-markers']) { + const present = runDatabaseCommand(t, id, part); + assert.equal(present.status, 0, `${id}/${part}: ${present.stderr}`); + assert.deepEqual(present.calls, []); + const missing = runDatabaseCommand(t, id, part, false); + assert.notEqual(missing.status, 0, `${id}/${part} must fail without the verified executable`); + assert.deepEqual(missing.calls, [], 'there must be no fallback build'); + } + } +}); diff --git a/tests/Unit/ServerParityAggregateTest.mjs b/tests/Unit/ServerParityAggregateTest.mjs index 02f8b0e10..fe21c2d61 100644 --- a/tests/Unit/ServerParityAggregateTest.mjs +++ b/tests/Unit/ServerParityAggregateTest.mjs @@ -131,22 +131,29 @@ test('the fixture jobs run only through the classifier and the result waits for assert.match(blocks.changes, /run: \$\{\{ steps\.classify\.outputs\.run \}\}/); assert.match(blocks.changes, /run: bash scripts\/conformance\/server-parity\/changes\.sh >> "\$GITHUB_OUTPUT"$/m); // The root fixture jobs run only through the classifier. - const roots = ['published-sdk-adapters', 'codec-native-build']; + const roots = ['published-sdk-adapters', 'codec-native-build', 'native-debug-build']; for (const id of roots) { assert.match(blocks[id], /^ {4}needs: changes\n {4}if: \$\{\{ needs\.changes\.outputs\.run == 'true' \}\}$/m, id); } const fixtureJobs = ids.filter(id => id !== 'changes' && id !== 'result'); for (const id of fixtureJobs.filter(id => !roots.includes(id))) { // A skipped root skips these jobs; no job-level condition may override - // that. Every one waits for the SDK adapters, and only deadline health - // also waits for the optimized native build. - const expected = id === 'codec-deadline-health' ? '[published-sdk-adapters, codec-native-build]' : 'published-sdk-adapters'; + // that. Deadline health uses the optimized artifact; matrix jobs also + // wait for this run's verified debug server. + const expected = id === 'codec-deadline-health' + ? '[published-sdk-adapters, codec-native-build]' : '[published-sdk-adapters, native-debug-build]'; assert.ok(blocks[id].split('\n').includes(` needs: ${expected}`), `${id} needs ${expected}`); - assert.doesNotMatch(blocks[id], /^ {4}if:/m, id); + if (id === 'codec-deadline-health') { + assert.doesNotMatch(blocks[id], /^ {4}if:/m, id); + } else { + // A failed debug producer cannot suppress independent native tests. + // SDK success still requires a successful, run=true classifier. + assert.match(blocks[id], /^ {4}if: \$\{\{ !cancelled\(\) && needs\.published-sdk-adapters\.result == 'success' \}\}$/m, id); + } } // Every job other than the classifier and the result is a gated fixture job. - assert.deepEqual(fixtureJobs.sort(), ['codec-deadline-health', 'codec-native-build', 'execution-and-codec', - 'mysql-execution', 'postgres-storage', 'published-sdk-adapters']); + assert.deepEqual(fixtureJobs.sort(), ['codec-deadline-health', 'codec-native-build', 'native-debug-build', 'execution-and-codec', + 'mysql-execution', 'postgres-storage', 'published-sdk-adapters'].sort()); const result = blocks.result; assert.equal(ids.at(-1), 'result'); assert.match(result, /^ {4}name: Shared Server fixtures result$/m); @@ -168,6 +175,7 @@ const fixture = (result, run) => ({ changes: {result: 'success', outputs: run === undefined ? {} : {run}}, 'published-sdk-adapters': {result, outputs: {}}, 'codec-native-build': {result, outputs: {}}, + 'native-debug-build': {result, outputs: {}}, 'codec-deadline-health': {result, outputs: {}}, 'mysql-execution': {result, outputs: {}}, 'postgres-storage': {result, outputs: {}}, @@ -187,6 +195,7 @@ test('the result passes when no input changed and every fixture job was skipped' }); for (const [label, needs, reason] of [ + ['a failed debug artifact producer', {...fixture('success', 'true'), 'native-debug-build': {result: 'failure', outputs: {}}}, /must be success when run=true: native-debug-build=failure\./], ['a failed fixture job', {...fixture('success', 'true'), 'mysql-execution': {result: 'failure', outputs: {}}}, /must be success when run=true: mysql-execution=failure\./], ['a cancelled fixture job', {...fixture('success', 'true'), 'execution-and-codec': {result: 'cancelled', outputs: {}}}, /must be success when run=true: execution-and-codec=cancelled\./], ['a skipped fixture job when inputs changed', {...fixture('success', 'true'), 'postgres-storage': {result: 'skipped', outputs: {}}}, /must be success when run=true: postgres-storage=skipped\./], From 5e911ad47644a63b4b6ddb035e8b6f10f5132b6f Mon Sep 17 00:00:00 2001 From: Richard McDaniel <1130888+rmcdaniel@users.noreply.github.com> Date: Fri, 30 Oct 2026 00:02:44 +0000 Subject: [PATCH 3/3] Use scoped workflow cleanup and preserve parity part contracts --- .github/workflows/server-parity.yml | 12 +++--------- scripts/ci/native-debug-build.sh | 25 ++++--------------------- tests/Unit/NativeDebugArtifactTest.mjs | 18 ++++++++++++------ tests/Unit/ServerParityPartsTest.mjs | 22 ++++++++++++++++------ 4 files changed, 35 insertions(+), 42 deletions(-) diff --git a/.github/workflows/server-parity.yml b/.github/workflows/server-parity.yml index 911204849..c07e6c35b 100644 --- a/.github/workflows/server-parity.yml +++ b/.github/workflows/server-parity.yml @@ -163,6 +163,8 @@ jobs: if: ${{ needs.changes.outputs.run == 'true' }} runs-on: ubuntu-latest timeout-minutes: 25 + env: + RESOURCE_SCOPE: dw-native-debug-${{ github.run_id }}-${{ github.run_attempt }}-${{ github.job }} outputs: artifact_name: ${{ steps.artifact.outputs.name }} producer_attempt: ${{ steps.artifact.outputs.attempt }} @@ -212,15 +214,7 @@ jobs: - name: Retire the owned debug build containers if: always() run: | - for receipt in native-debug-build.clean.cid native-debug-build.build.cid; do - if test -s "$receipt"; then - read -r container < "$receipt" || true - if [[ "$container" =~ ^[a-f0-9]{64}$ ]]; then - docker rm -f "$container" >/dev/null 2>&1 || true - fi - fi - rm -f -- "$receipt" - done + docker rm -f "$RESOURCE_SCOPE-clean" "$RESOURCE_SCOPE-build" >/dev/null 2>&1 || true published-sdk-adapters: needs: changes diff --git a/scripts/ci/native-debug-build.sh b/scripts/ci/native-debug-build.sh index fee3495f5..b9d29997f 100644 --- a/scripts/ci/native-debug-build.sh +++ b/scripts/ci/native-debug-build.sh @@ -6,37 +6,20 @@ root=$(git rev-parse --show-toplevel) cd "$root" image=${NATIVE_RUST_IMAGE:?NATIVE_RUST_IMAGE must be the pinned workflow compiler} node scripts/ci/native-debug-artifact.mjs identity >/dev/null -scope="dw-native-debug-${GITHUB_RUN_ID:?}-${GITHUB_RUN_ATTEMPT:?}-${GITHUB_JOB:?}" +: "${RESOURCE_SCOPE:?RESOURCE_SCOPE must be unique to this workflow job}" +test "$RESOURCE_SCOPE" = "dw-native-debug-${GITHUB_RUN_ID:?}-${GITHUB_RUN_ATTEMPT:?}-${GITHUB_JOB:?}" output=$(realpath -m "$1") case "$output" in "$root"/*) ;; *) echo 'native debug output must be inside this checkout' >&2; exit 2 ;; esac test ! -e "$output" -clean_cid="$output.clean.cid" -build_cid="$output.build.cid" -test ! -e "$clean_cid" -test ! -e "$build_cid" mkdir -p "$output" cargo-home rust-target -cleanup() { - for receipt in "$clean_cid" "$build_cid"; do - if test -s "$receipt"; then - read -r container < "$receipt" || true - if [[ "$container" =~ ^[a-f0-9]{64}$ ]]; then - docker rm -f "$container" >/dev/null 2>&1 || true - fi - fi - rm -f -- "$receipt" - done -} -trap cleanup EXIT -trap 'exit 130' INT -trap 'exit 143' TERM started=$SECONDS -docker run --rm --init --name "$scope-clean" --cidfile "$clean_cid" \ +docker run --rm --init --name "$RESOURCE_SCOPE-clean" \ --label "dw.task=server-native-debug-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}" \ --user="$(id -u):$(id -g)" --network=none --cpus=2 --memory=2g --memory-swap=2g \ -e CARGO_HOME=/cargo -e CARGO_TARGET_DIR=/target \ -v "$root:/source:ro" -v "$root/cargo-home:/cargo" -v "$root/rust-target:/target" -w /source/rust \ "$image" cargo clean --locked --offline --package durable-workflow-server -if ! docker run --rm --init --name "$scope-build" --cidfile "$build_cid" \ +if ! docker run --rm --init --name "$RESOURCE_SCOPE-build" \ --label "dw.task=server-native-debug-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}" \ --user="$(id -u):$(id -g)" --network=none --cpus=2 --memory=6g --memory-swap=6g \ -e CARGO_HOME=/cargo -e CARGO_TARGET_DIR=/target -e CARGO_BUILD_JOBS=2 \ diff --git a/tests/Unit/NativeDebugArtifactTest.mjs b/tests/Unit/NativeDebugArtifactTest.mjs index 1e2ecc165..5612dccee 100644 --- a/tests/Unit/NativeDebugArtifactTest.mjs +++ b/tests/Unit/NativeDebugArtifactTest.mjs @@ -35,6 +35,7 @@ function fixture(t) { git('-c', 'user.name=Fixture', '-c', 'user.email=fixture@example.invalid', 'commit', '-qm', 'fixture'); const env = {...process.env, GITHUB_SHA: git('rev-parse', 'HEAD'), GITHUB_REPOSITORY: 'fixture/server', GITHUB_RUN_ID: '12345', GITHUB_RUN_ATTEMPT: '1', GITHUB_JOB: 'native-debug-build', + RESOURCE_SCOPE: 'dw-native-debug-12345-1-native-debug-build', NATIVE_RUST_IMAGE: `mirror.gcr.io/library/rust@sha256:${'a'.repeat(64)}`, RUNNER_OS: 'Linux', RUNNER_ARCH: 'X64'}; delete env.NATIVE_BUILD_RUN_ATTEMPT; const directory = join(root, 'artifact'); @@ -140,6 +141,10 @@ test('identity preflight and existing-output guard refuse before any container s assert.notEqual(badIdentity.status, 0); assert.match(badIdentity.stderr, /workflow checkout/); assert.equal(existsSync(log), false); + const badScope = spawnSync('bash', ['scripts/ci/native-debug-build.sh', 'other'], + {cwd: root, env: {...commandEnv, RESOURCE_SCOPE: 'another-job-scope'}, encoding: 'utf8'}); + assert.notEqual(badScope.status, 0); + assert.equal(existsSync(log), false); }); function fakeDocker(root) { @@ -147,12 +152,9 @@ function fakeDocker(root) { mkdirSync(bin); writeFileSync(join(bin, 'docker'), `#!/usr/bin/env node import('node:fs').then(async fs => { - const {createHash} = await import('node:crypto'); const args = process.argv.slice(2); fs.appendFileSync(process.env.FAKE_DOCKER_LOG, JSON.stringify(args) + '\\n'); if (args[0] !== 'run') return; - const cid = createHash('sha256').update(args.join(' ')).digest('hex'); - fs.writeFileSync(args[args.indexOf('--cidfile') + 1], cid + '\\n'); if (!args.some(arg => arg.includes('cargo build'))) return; if (process.env.FAKE_BUILD_FAIL) process.exit(23); const target = args.find(arg => arg.endsWith(':/target')).slice(0, -8); @@ -165,7 +167,7 @@ import('node:fs').then(async fs => { } for (const failure of [false, true]) { - test(`the build helper ${failure ? 'cleans its own failed build' : 'exports only the thin artifact'}`, t => { + test(`the build helper ${failure ? 'uses scoped ephemeral containers on failure' : 'exports only the thin artifact'}`, t => { const {root, env} = fixture(t); const bin = fakeDocker(root); const log = join(root, 'docker-calls.jsonl'); @@ -177,14 +179,16 @@ for (const failure of [false, true]) { const builds = calls.filter(args => args[0] === 'run'); assert.equal(builds.length, 2); for (const args of builds) { - assert.ok(args.includes('--init') && args.includes('--network=none')); + assert.ok(args.includes('--rm') && args.includes('--init') && args.includes('--network=none')); assert.ok(args.includes('--cpus=2') && args.includes(env.NATIVE_RUST_IMAGE)); assert.ok(args.includes(`--user=${process.getuid()}:${process.getgid()}`)); } assert.deepEqual(builds[0].slice(-6), ['cargo', 'clean', '--locked', '--offline', '--package', 'durable-workflow-server']); assert.ok(builds[1].some(arg => arg.includes('cargo build --locked --offline --bin durable-workflow-server'))); assert.ok(builds[1].includes('--memory=6g') && builds[1].includes('--memory-swap=6g')); - assert.equal(calls.filter(args => args[0] === 'rm' && /^[a-f0-9]{64}$/.test(args.at(-1))).length, 2); + assert.deepEqual(builds.map(args => args[args.indexOf('--name') + 1]), + [`${env.RESOURCE_SCOPE}-clean`, `${env.RESOURCE_SCOPE}-build`]); + assert.equal(calls.some(args => args[0] === 'rm'), false, 'cleanup belongs to the workflow always-step'); if (!failure) verifyArtifact(join(root, 'bundle'), {root, env}); }); } @@ -193,6 +197,7 @@ test('the producer shares only three files with pinned actions and bounded reten const producer = workflowJob('native-debug-build'); assert.match(producer, /needs: changes\n {4}if: \$\{\{ needs\.changes\.outputs\.run == 'true' \}\}/); assert.match(producer, /persist-credentials: false/); + assert.match(producer, /RESOURCE_SCOPE: dw-native-debug-\$\{\{ github\.run_id \}\}-\$\{\{ github\.run_attempt \}\}-\$\{\{ github\.job \}\}/); assert.match(producer, /artifact_name: \$\{\{ steps\.artifact\.outputs\.name \}\}/); assert.match(producer, /producer_attempt: \$\{\{ steps\.artifact\.outputs\.attempt \}\}/); assert.match(producer, /native-debug-build-%s-%s/); @@ -204,6 +209,7 @@ test('the producer shares only three files with pinned actions and bounded reten + ' native-debug-build/build.json\n native-debug-build/SHA256SUMS\n'); assert.match(producer, /retention-days: 1\n {10}if-no-files-found: error/); assert.doesNotMatch(producer, /cache\/save|contents: write|packages: write/); + assert.match(producer, /- name: Retire the owned debug build containers\n {8}if: always\(\)\n {8}run: \|\n {10}docker rm -f "\$RESOURCE_SCOPE-clean" "\$RESOURCE_SCOPE-build"/); }); test('all non-test consumers verify the same-run artifact before installing the executable', () => { diff --git a/tests/Unit/ServerParityPartsTest.mjs b/tests/Unit/ServerParityPartsTest.mjs index 4ffcdbf94..14c087d54 100644 --- a/tests/Unit/ServerParityPartsTest.mjs +++ b/tests/Unit/ServerParityPartsTest.mjs @@ -1,8 +1,8 @@ // Each Shared Server fixtures database job runs in six parts on separate // runners. Every step either prepares the part (and so runs in every part) or -// belongs to exactly one part, so no check runs twice or not at all. The one -// exception starts the published PHP and native nodes that the corpus and -// restarts parts each need, and runs in exactly those two. +// belongs to exactly one part, so no check runs twice or not at all. Shared +// node preparation belongs to corpus/restarts; debug artifact preparation +// belongs to the five non-native-test parts. import test from 'node:test'; import assert from 'node:assert/strict'; import {readFileSync} from 'node:fs'; @@ -11,6 +11,9 @@ import {spawnSync} from 'node:child_process'; const workflow = readFileSync('.github/workflows/server-parity.yml', 'utf8'); const PARTS = ['native-tests', 'corpus', 'restarts', 'cohorts', 'sdk-directions', 'corrected-markers']; const SHARED_NODES = 'Start the published PHP and native nodes the corpus parts share'; +const NON_NATIVE_PARTS = PARTS.filter(part => part !== 'native-tests'); +const NATIVE_DEBUG_PREP = new Set(['Require a successful native debug producer', + "Fetch this run's native debug server", "Verify and install this run's native debug server"]); const EVERY_PART = [/^actions\/checkout@/, /^Pull the pinned images, retrying registry errors$/, /^Fetch /, /^Restore native Cargo dependencies from trusted main$/, /^Discard restored native workspace outputs before rebuilding$/, /^Create TLS fixture certificates$/, /^Bootstrap published /, /^Collect /, /^Summarize /, /^Retain /]; @@ -34,6 +37,10 @@ function steps(block) { // The parts a step runs in, read from its condition. function partsOf(step) { + if (NATIVE_DEBUG_PREP.has(step.name)) { + assert.equal(step.condition, "${{ matrix.part != 'native-tests' && matrix.part != 'native-tests-b' }}", step.name); + return NON_NATIVE_PARTS; + } if (EVERY_PART.some(pattern => pattern.test(step.name))) return PARTS; const either = /^\$\{\{ matrix\.part == '([a-z-]+)' \|\| matrix\.part == '([a-z-]+)' \}\}$/.exec(step.condition ?? ''); if (either) return [either[1], either[2]]; @@ -58,7 +65,10 @@ for (const [name, key] of [['mysql-execution', 'Compare three runtimes'], ['post assert.ok(EVERY_PART.some(pattern => pattern.test(step.name)), `${name}: "${step.name}" must belong to one part`); } else if (step.name === SHARED_NODES) { assert.deepEqual(parts, ['corpus', 'restarts'], `${name}: the shared nodes start for exactly the corpus parts`); - } else if (step.name !== 'Build the native server') { + } else if (NATIVE_DEBUG_PREP.has(step.name)) { + assert.deepEqual(parts, ['corpus', 'restarts', 'cohorts', 'sdk-directions', 'corrected-markers'], + `${name}: debug preparation runs in exactly the five non-native-test parts`); + } else { assert.equal(parts.length, 1, `${name}: "${step.name}"`); } } @@ -121,12 +131,12 @@ for (const [name, key] of [['mysql-execution', 'Compare three runtimes'], ['post test('the database bootstraps run the native suites only in the native-tests parts', () => { const postgres = steps(job('postgres-storage')).find(step => step.name.startsWith('Bootstrap published ')); assert.match(postgres.text, /-e DW_PARITY_PART \\\n/); - assert.match(postgres.text, /'if \[ "\$DW_PARITY_PART" = native-tests \]; then cargo test [^']* --test execution [^']*; fi && cargo build --locked --offline'/); + assert.match(postgres.text, /'if \[ "\$DW_PARITY_PART" = native-tests \]; then cargo test [^']* --test execution [^']*; fi && if \[ "\$DW_PARITY_PART" = native-tests \]; then cargo build --locked --offline; else test -x \/target\/debug\/durable-workflow-server; fi'/); const mysql = steps(job('mysql-execution')).find(step => step.name.startsWith('Bootstrap published ')); assert.match(mysql.text, /-e DW_PARITY_PART -e DW_EXECUTION_SHARDS \\\n/); // native-tests runs the database suite and the first half of the execution // tests; native-tests-b, where a label has it, runs the second half. - assert.match(mysql.text, /'case "\$DW_PARITY_PART" in native-tests\) shard=1 suffix= ;; native-tests-b\) shard=2 suffix=-b ;; \*\) shard= ;; esac; if \[ "\$DW_PARITY_PART" = native-tests \]; then cargo test [^']*runtime::mysql::[^']* > \/evidence\/mysql-tests\.txt \|\| exit 1; fi; if \[ -n "\$shard" \]; then [^']*cargo test --locked --offline --test execution -- --exact --nocapture --test-threads=2 \$\(cat \/evidence\/execution-shard\$suffix\.txt\) > \/evidence\/mysql-execution-tests\$suffix\.txt \|\| exit 1; fi; cargo build --locked --offline'/); + assert.match(mysql.text, /'case "\$DW_PARITY_PART" in native-tests\) shard=1 suffix= ;; native-tests-b\) shard=2 suffix=-b ;; \*\) shard= ;; esac; if \[ "\$DW_PARITY_PART" = native-tests \]; then cargo test [^']*runtime::mysql::[^']* > \/evidence\/mysql-tests\.txt \|\| exit 1; fi; if \[ -n "\$shard" \]; then [^']*cargo test --locked --offline --test execution -- --exact --nocapture --test-threads=2 \$\(cat \/evidence\/execution-shard\$suffix\.txt\) > \/evidence\/mysql-execution-tests\$suffix\.txt \|\| exit 1; fi; case "\$DW_PARITY_PART" in native-tests\|native-tests-b\) cargo build --locked --offline ;; \*\) test -x \/target\/debug\/durable-workflow-server ;; esac'/); }); test('the MySQL 8 labels split their execution tests into two parts, every test in exactly one', () => {