diff --git a/.github/workflows/dr8_test_execution.yml b/.github/workflows/dr8_test_execution.yml new file mode 100644 index 00000000000..c9722d95acc --- /dev/null +++ b/.github/workflows/dr8_test_execution.yml @@ -0,0 +1,253 @@ +# ******************************************************************************* +# Copyright (c) 2025 Contributors to the Eclipse Foundation +# +# See the NOTICE file(s) distributed with this work for additional +# information regarding copyright ownership. +# +# This program and the accompanying materials are made available under the +# terms of the Apache License Version 2.0 which is available at +# https://www.apache.org/licenses/LICENSE-2.0 +# +# SPDX-License-Identifier: Apache-2.0 +# ******************************************************************************* +name: DR-008 Test Execution (Stage 1 & 2) +permissions: + contents: write + pull-requests: write +on: + # Not pull_request_target: no step here needs a fork's secrets (the release upload is + # tag-gated), and it would run this file from the base branch, never the PR's own version. + pull_request: + types: [opened, reopened, synchronize] + push: + branches: + - main + merge_group: + types: [checks_requested] + release: + types: [created] +# Do not flood CI with unneeded previous runs in PR +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: ${{ github.event_name == 'pull_request' }} +env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} +jobs: + stage1_integration: + name: "Stage 1 — Platform Build & Feature Integration Tests" + runs-on: ubuntu-latest + permissions: + contents: write + pull-requests: write + steps: + - name: Clean disk space + uses: eclipse-score/more-disk-space@v1.1 + with: + level: 4 + - name: Setup Bazel + uses: bazel-contrib/setup-bazel@0.18.0 + with: + bazelisk-cache: true + disk-cache: ${{ github.workflow }}-stage1 + repository-cache: true + cache-save: ${{ github.event_name == 'push' }} + - name: Checkout repository + uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + - name: Execute Feature Integration Tests + run: | + bazel test --lockfile_mode=error --config=linux-x86_64 //feature_integration_tests/test_cases:fit + - name: Export resolved dependency manifest + if: always() + run: | + mkdir -p artifacts/stage1-resolved-deps + # Merge the resolved registry versions with ref_int's own override directives into the + # single Stage 1 -> Stage 2 handoff manifest. The script stores the graph alongside it, + # which Stage 2 needs to pin each module's full transitive closure. + # --verbose populates originalVersion; without it every pin report verdict is "unknown". + bazel mod graph --verbose --output=json --lockfile_mode=error > resolved_graph.json + bazel run //scripts/known_good:resolve_deps -- \ + --mod-graph resolved_graph.json \ + --export artifacts/stage1-resolved-deps/resolved_versions.json + cp MODULE.bazel.lock artifacts/stage1-resolved-deps/ # evidence of full resolution + - name: Upload resolved dependency set artifact + if: always() + uses: actions/upload-artifact@v4.4.0 + with: + name: stage1-resolved-deps + path: artifacts/stage1-resolved-deps/ + retention-days: 14 + if-no-files-found: warn + # --------------------------------------------------------------------------- + # Stage 2 matrix, derived from known_good.json's target_sw group and never hardcoded here. + # Each entry carries {name, repo, slug, commit, branch} so Stage 2 can check the module out. + # --------------------------------------------------------------------------- + prepare_matrix: + name: "Prepare Stage 2 module matrix" + needs: stage1_integration + if: ${{ !cancelled() }} + runs-on: ubuntu-latest + outputs: + modules: ${{ steps.list.outputs.modules }} + steps: + - name: Checkout repository + uses: actions/checkout@v4 + - name: Setup Bazel + uses: bazel-contrib/setup-bazel@0.18.0 + with: + bazelisk-cache: true + repository-cache: true + cache-save: ${{ github.event_name == 'push' }} + - name: List target_sw modules from known_good.json + id: list + run: | + echo "modules=$(bazel run --ui_event_filters=-info,-stdout --noshow_progress \ + //scripts/known_good:list_modules -- --group target_sw)" >> "$GITHUB_OUTPUT" + # --------------------------------------------------------------------------- + # Stage 2 — Module-Scoped (DR-008 Option 4). Per module: check it out at its known_good commit, + # pin its MODULE.bazel to the Stage-1 resolved set, and run its own unit tests + coverage inside + # the module (bazel root //...), not through ref_int's graph. Injection is ephemeral (CI checkout + # only). fail-fast: false so one module's failure does not hide the others' results. + # --------------------------------------------------------------------------- + stage2_module_validation: + name: "Stage 2 — Module UT & Coverage (${{ matrix.module.name }})" + needs: [stage1_integration, prepare_matrix] + if: ${{ !cancelled() }} + strategy: + fail-fast: false + matrix: + module: ${{ fromJSON(needs.prepare_matrix.outputs.modules) }} + runs-on: ubuntu-latest + steps: + - name: Clean disk space + uses: eclipse-score/more-disk-space@v1.1 + with: + level: 4 + - name: Setup Bazel + uses: bazel-contrib/setup-bazel@0.18.0 + with: + bazelisk-cache: true + disk-cache: ${{ github.workflow }}-stage2-${{ matrix.module.name }} + repository-cache: true + cache-save: ${{ github.event_name == 'push' }} + - name: Install lcov + run: | + sudo apt-get update + sudo apt-get install -y lcov + # ref_int checkout — provides the scripts (quality_runners.py, ResolvedDependencies). + - name: Checkout reference_integration + uses: actions/checkout@v4 + # The module under test, checked out at its Stage-1 known_good commit (R4). + - name: Checkout module under test + uses: actions/checkout@v4 + with: + repository: ${{ matrix.module.slug }} + ref: ${{ matrix.module.commit }} + path: _module + # Consume the Stage-1 resolved dependency set (R2). + - name: Download Stage 1 resolved dependency set + uses: actions/download-artifact@v4.1.8 + with: + name: stage1-resolved-deps + path: _resolved_deps/ + - name: Execute Unit Tests with Coverage Analysis (in module context) + run: | + bazel run //scripts:quality_runners -- \ + --modules-to-test ${{ matrix.module.name }} \ + --module-dir _module \ + --resolved-deps _resolved_deps + # DR-008's claim is that the module was validated against ref_int's resolved versions. + # Prove it from the module's own post-MVS graph rather than assuming the injection took. + - name: Verify module resolved to ref_int's dependency versions + if: always() + run: | + # The resolution gate already captured this graph, before the tests ran and under the + # resolution they were pinned to. Reuse it rather than recomputing a second one. + if [ ! -s _module/module_graph.json ]; then + echo "::warning::no module graph captured for ${{ matrix.module.name }}"; exit 0 + fi + bazel run //scripts/known_good:verify_stage2_resolution -- \ + --mod-graph _module/module_graph.json \ + --resolved _resolved_deps/resolved_versions.json \ + --module-bazel _module/MODULE.bazel \ + --module ${{ matrix.module.name }} + - name: Upload module quality report + if: always() + uses: actions/upload-artifact@v4.4.0 + with: + name: stage2-report-${{ matrix.module.name }} + path: docs/verification_report/ + retention-days: 14 + if-no-files-found: warn + - name: Upload module test logs and coverage + if: always() + uses: actions/upload-artifact@v4.4.0 + with: + name: stage2-testlogs-${{ matrix.module.name }} + path: | + _module/bazel-testlogs/ + artifacts/coverage/ + retention-days: 14 + if-no-files-found: warn + # MODULE.bazel.lock as the resolution gate wrote it, after injection and before any test ran; + # selection_digest then asserts the test run did not move any selected version. + # module_graph.json is the module-rooted post-MVS graph -- the only artifact carrying a + # module's dev-dependency closure, since Stage 1's graph is rooted at ref_int where those + # edges are inactive. + - name: Upload regenerated module lockfile and resolved graph + if: always() + uses: actions/upload-artifact@v4.4.0 + with: + name: stage2-resolved-lock-${{ matrix.module.name }} + path: | + _module/MODULE.bazel.lock + _module/module_graph.json + retention-days: 14 + if-no-files-found: warn + # --------------------------------------------------------------------------- + # Aggregate — consolidate Stage 1 + Stage 2 results into one quality report. + # Also handles the release-tag test-report ZIP. + # --------------------------------------------------------------------------- + aggregate: + name: "Aggregate Quality Report" + needs: [stage1_integration, stage2_module_validation] + if: always() + runs-on: ubuntu-latest + steps: + - name: Checkout repository + uses: actions/checkout@v4 + - name: Setup Bazel + uses: bazel-contrib/setup-bazel@0.18.0 + with: + bazelisk-cache: true + repository-cache: true + cache-save: ${{ github.event_name == 'push' }} + - name: Download Stage 2 quality reports + uses: actions/download-artifact@v4.1.8 + with: + pattern: stage2-report-* + path: _stage2_reports/ + # Distinct name from test_and_docs's release asset -- avoids a tag-event upload race. + - name: Create archive of test reports + if: github.ref_type == 'tag' + run: | + mkdir -p artifacts/test-reports + find _stage2_reports -name 'test.xml' -print0 | \ + xargs -0 -I{} cp --parents {} artifacts/test-reports/ 2>/dev/null || true + zip -r ${{ github.event.repository.name }}_test_reports_stage2.zip artifacts/test-reports/ + shell: bash + - name: Upload release asset (attach ZIP to GitHub Release) + uses: softprops/action-gh-release@v2.5.0 + if: github.ref_type == 'tag' + with: + files: ${{ github.event.repository.name }}_test_reports_stage2.zip + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + - name: Publish consolidated quality report + if: always() + run: | + bazel run --ui_event_filters=-info,-stdout --noshow_progress \ + //scripts:aggregate_quality_report -- \ + --stage1-result "${{ needs.stage1_integration.result }}" \ + --stage2-result "${{ needs.stage2_module_validation.result }}" \ + --stage2-dir "_stage2_reports/" \ + >> "$GITHUB_STEP_SUMMARY" diff --git a/.github/workflows/internal_tests.yml b/.github/workflows/internal_tests.yml index ec62f248aa9..4bc693e770d 100644 --- a/.github/workflows/internal_tests.yml +++ b/.github/workflows/internal_tests.yml @@ -21,4 +21,6 @@ jobs: internal_tests: uses: eclipse-score/cicd-workflows/.github/workflows/tests.yml@main with: - bazel-target: "test //scripts/tooling:tooling_tests //scripts/known_good:known_good_tests" + # Bundles tooling_tests with known_good_tests and quality_scripts_tests; previously only + # tooling_tests ran, so the DR-008 unit tests were never executed in CI. + bazel-target: "test //scripts:all_python_unit_tests" diff --git a/MODULE.bazel.lock b/MODULE.bazel.lock index 8b3ee03d415..736962e7846 100644 --- a/MODULE.bazel.lock +++ b/MODULE.bazel.lock @@ -1006,8 +1006,8 @@ "https://raw.githubusercontent.com/eclipse-score/bazel_registry/main/modules/score_rust_policies/0.0.3/MODULE.bazel": "9e8310a75c13ccebc49fb9cbf7acc6c1b75654292b2ca907fb5d513133dbf6f3", "https://raw.githubusercontent.com/eclipse-score/bazel_registry/main/modules/score_rust_policies/0.0.5/MODULE.bazel": "7de02547bdf121d3dedf5141b97f0fd9a545bd255ff5c7b699056b35816ffad9", "https://raw.githubusercontent.com/eclipse-score/bazel_registry/main/modules/score_rust_policies/0.0.5/source.json": "22c8bf0a5cbf7c7b06f774f3f66498e0bc14346a8b2208f7427a8fbb78a42547", - "https://raw.githubusercontent.com/eclipse-score/bazel_registry/main/modules/score_toolchains_rust/0.8.0/MODULE.bazel": "ea57a9a4dcb8ad49f4556f824500eb559365f413ccbb39d70d0b363685aacec5", - "https://raw.githubusercontent.com/eclipse-score/bazel_registry/main/modules/score_toolchains_rust/0.8.0/source.json": "394a615e03ad722bc27bd4a6f098c6ff2fe7120b69cdf3925d47e39d30ada8a4", + "https://raw.githubusercontent.com/eclipse-score/bazel_registry/main/modules/score_toolchains_rust/0.9.1/MODULE.bazel": "40cab3f733d11fa7ebfa00667148c8da7c4c4168f0010cc8fff90d577f4f28f9", + "https://raw.githubusercontent.com/eclipse-score/bazel_registry/main/modules/score_toolchains_rust/0.9.1/source.json": "af8b25d7a21b2f60678f31fef4ad767c6e0ad8935386d51e0e3180bf5989e8d9", "https://raw.githubusercontent.com/eclipse-score/bazel_registry/main/modules/stardoc/0.5.0/MODULE.bazel": "not found", "https://raw.githubusercontent.com/eclipse-score/bazel_registry/main/modules/stardoc/0.5.1/MODULE.bazel": "not found", "https://raw.githubusercontent.com/eclipse-score/bazel_registry/main/modules/stardoc/0.5.3/MODULE.bazel": "not found", @@ -10125,10 +10125,86 @@ ] } }, + "@@score_toolchains_rust+//extensions:ferrocene_toolchain_ext.bzl%ferrocene_rules_rust_miri_toolchain_ext": { + "general": { + "bzlTransitiveDigest": "JTR6scmBZuukTYspnbvaNFU27CfBHiQyu6OywcI1QPE=", + "usagesDigest": "B0VP7yPsZUtmcwtuYiF7WqfWQv6X8v5yPartmPS6dyg=", + "recordedFileInputs": {}, + "recordedDirentsInputs": {}, + "envVariables": {}, + "generatedRepoSpecs": { + "ferrocene_x86_64_unknown_linux_gnu_rules_rust_miri": { + "repoRuleId": "@@score_toolchains_rust+//extensions:ferrocene_toolchain_ext.bzl%ferrocene_rules_rust_miri_toolchain_repo", + "attributes": { + "ferrocene_repo_name": "ferrocene_x86_64_unknown_linux_gnu", + "toolchain_name": "rust_ferrocene", + "env": {}, + "exec_compatible_with": [ + "@platforms//cpu:x86_64", + "@platforms//os:linux" + ], + "target_compatible_with": [ + "@platforms//cpu:x86_64", + "@platforms//os:linux" + ] + } + }, + "ferrocene_aarch64_unknown_linux_gnu_rules_rust_miri": { + "repoRuleId": "@@score_toolchains_rust+//extensions:ferrocene_toolchain_ext.bzl%ferrocene_rules_rust_miri_toolchain_repo", + "attributes": { + "ferrocene_repo_name": "ferrocene_aarch64_unknown_linux_gnu", + "toolchain_name": "rust_ferrocene", + "env": {}, + "exec_compatible_with": [ + "@platforms//cpu:x86_64", + "@platforms//os:linux" + ], + "target_compatible_with": [ + "@platforms//cpu:aarch64", + "@platforms//os:linux" + ] + } + }, + "ferrocene_x86_64_pc_nto_qnx800_rules_rust_miri": { + "repoRuleId": "@@score_toolchains_rust+//extensions:ferrocene_toolchain_ext.bzl%ferrocene_rules_rust_miri_toolchain_repo", + "attributes": { + "ferrocene_repo_name": "ferrocene_x86_64_pc_nto_qnx800", + "toolchain_name": "rust_ferrocene", + "env": {}, + "exec_compatible_with": [ + "@platforms//cpu:x86_64", + "@platforms//os:linux" + ], + "target_compatible_with": [ + "@platforms//cpu:x86_64", + "@platforms//os:qnx" + ] + } + }, + "ferrocene_aarch64_unknown_nto_qnx800_rules_rust_miri": { + "repoRuleId": "@@score_toolchains_rust+//extensions:ferrocene_toolchain_ext.bzl%ferrocene_rules_rust_miri_toolchain_repo", + "attributes": { + "ferrocene_repo_name": "ferrocene_aarch64_unknown_nto_qnx800", + "toolchain_name": "rust_ferrocene", + "env": {}, + "exec_compatible_with": [ + "@platforms//cpu:x86_64", + "@platforms//os:linux" + ], + "target_compatible_with": [ + "@platforms//cpu:aarch64", + "@platforms//os:qnx" + ] + } + } + }, + "recordedRepoMappingEntries": [] + } + }, "@@score_toolchains_rust+//extensions:ferrocene_toolchain_ext.bzl%ferrocene_toolchain_ext": { "general": { - "bzlTransitiveDigest": "XEgifqjEEdSGw80RccoJ/aUy9smsRXQJ9jO4RDOf2vk=", - "usagesDigest": "IlxwUERhbOjfZJ1PLnLDjAAkSMdSsMwxXS0NFZ7I+Rw=", + "bzlTransitiveDigest": "JTR6scmBZuukTYspnbvaNFU27CfBHiQyu6OywcI1QPE=", + "usagesDigest": "SYlMBiaVjqmn1xAOp7YjU0tmU/mTruZJe2gp+bHZQBM=", "recordedFileInputs": {}, "recordedDirentsInputs": {}, "envVariables": {}, @@ -10168,14 +10244,17 @@ ], "coverage_tools_url": "https://github.com/eclipse-score/ferrocene_toolchain_builder/releases/download/1.0.1/coverage-tools-779fbed05ae9e9fe2a04137929d99cc9b3d516fd-x86_64-unknown-linux-gnu.tar.gz", "coverage_tools_sha256": "497958e925bc94833ea226d68f6d5ba38bd890f571c73e230141d2923e30dd94", - "coverage_tools_strip_prefix": "779fbed05ae9e9fe2a04137929d99cc9b3d516fd/x86_64-unknown-linux-gnu" + "coverage_tools_strip_prefix": "779fbed05ae9e9fe2a04137929d99cc9b3d516fd/x86_64-unknown-linux-gnu", + "miri_sysroot_url": "", + "miri_sysroot_sha256": "", + "miri_sysroot_strip_prefix": "" } }, "ferrocene_x86_64_unknown_linux_gnu": { "repoRuleId": "@@score_toolchains_rust+//extensions:ferrocene_toolchain_ext.bzl%ferrocene_toolchain_repo", "attributes": { - "url": "https://github.com/eclipse-score/ferrocene_toolchain_builder/releases/download/1.0.1/ferrocene-779fbed05ae9e9fe2a04137929d99cc9b3d516fd-x86_64-unknown-linux-gnu.tar.gz", - "sha256": "4c08b41eaafd39cff66333ca4d4646a5331c780050b8b9a8447353fcd301dddc", + "url": "https://github.com/eclipse-score/ferrocene_toolchain_builder/releases/download/1.2.0/ferrocene-779fbed05ae9e9fe2a04137929d99cc9b3d516fd-x86_64-unknown-linux-gnu.tar.gz", + "sha256": "4082058e4d054b1e26261e7ec99f01bf807f87b4ea580d246e48d9ccd487a591", "strip_prefix": "", "toolchain_name": "rust_ferrocene", "target_triple": "x86_64-unknown-linux-gnu", @@ -10201,16 +10280,19 @@ "@platforms//cpu:x86_64", "@platforms//os:linux" ], - "coverage_tools_url": "https://github.com/eclipse-score/ferrocene_toolchain_builder/releases/download/1.0.1/coverage-tools-779fbed05ae9e9fe2a04137929d99cc9b3d516fd-x86_64-unknown-linux-gnu.tar.gz", - "coverage_tools_sha256": "497958e925bc94833ea226d68f6d5ba38bd890f571c73e230141d2923e30dd94", - "coverage_tools_strip_prefix": "779fbed05ae9e9fe2a04137929d99cc9b3d516fd/x86_64-unknown-linux-gnu" + "coverage_tools_url": "https://github.com/eclipse-score/ferrocene_toolchain_builder/releases/download/1.2.0/coverage-tools-779fbed05ae9e9fe2a04137929d99cc9b3d516fd-x86_64-unknown-linux-gnu.tar.gz", + "coverage_tools_sha256": "841172d34b2fc0a8bed2756cf16f38d29ac18c13ee29fbb87af3ae047aa2a6a0", + "coverage_tools_strip_prefix": "779fbed05ae9e9fe2a04137929d99cc9b3d516fd/x86_64-unknown-linux-gnu", + "miri_sysroot_url": "https://github.com/eclipse-score/ferrocene_toolchain_builder/releases/download/1.2.0/miri-sysroot-779fbed05ae9e9fe2a04137929d99cc9b3d516fd-x86_64-unknown-linux-gnu.tar.gz", + "miri_sysroot_sha256": "8b745cc64fe4d9d27081196cc565ea3cd198b24fce0ef7e2f014a11d85629745", + "miri_sysroot_strip_prefix": "x86_64-unknown-linux-gnu" } }, "ferrocene_aarch64_unknown_linux_gnu": { "repoRuleId": "@@score_toolchains_rust+//extensions:ferrocene_toolchain_ext.bzl%ferrocene_toolchain_repo", "attributes": { - "url": "https://github.com/eclipse-score/ferrocene_toolchain_builder/releases/download/1.0.1/ferrocene-779fbed05ae9e9fe2a04137929d99cc9b3d516fd-aarch64-unknown-linux-gnu.tar.gz", - "sha256": "b1f1eb1146bf595fe1f4a65d5793b7039b37d2cb6d395d1c3100fa7d0377b6c9", + "url": "https://github.com/eclipse-score/ferrocene_toolchain_builder/releases/download/1.2.0/ferrocene-779fbed05ae9e9fe2a04137929d99cc9b3d516fd-aarch64-unknown-linux-gnu.tar.gz", + "sha256": "3fd5fe5da4836eb6d554731e7899d378a6992106ce6275b136279dec29598383", "strip_prefix": "", "toolchain_name": "rust_ferrocene", "target_triple": "aarch64-unknown-linux-gnu", @@ -10236,16 +10318,19 @@ "@platforms//cpu:aarch64", "@platforms//os:linux" ], - "coverage_tools_url": "https://github.com/eclipse-score/ferrocene_toolchain_builder/releases/download/1.0.1/coverage-tools-779fbed05ae9e9fe2a04137929d99cc9b3d516fd-x86_64-unknown-linux-gnu.tar.gz", - "coverage_tools_sha256": "497958e925bc94833ea226d68f6d5ba38bd890f571c73e230141d2923e30dd94", - "coverage_tools_strip_prefix": "779fbed05ae9e9fe2a04137929d99cc9b3d516fd/x86_64-unknown-linux-gnu" + "coverage_tools_url": "https://github.com/eclipse-score/ferrocene_toolchain_builder/releases/download/1.2.0/coverage-tools-779fbed05ae9e9fe2a04137929d99cc9b3d516fd-x86_64-unknown-linux-gnu.tar.gz", + "coverage_tools_sha256": "841172d34b2fc0a8bed2756cf16f38d29ac18c13ee29fbb87af3ae047aa2a6a0", + "coverage_tools_strip_prefix": "779fbed05ae9e9fe2a04137929d99cc9b3d516fd/x86_64-unknown-linux-gnu", + "miri_sysroot_url": "https://github.com/eclipse-score/ferrocene_toolchain_builder/releases/download/1.2.0/miri-sysroot-779fbed05ae9e9fe2a04137929d99cc9b3d516fd-aarch64-unknown-linux-gnu.tar.gz", + "miri_sysroot_sha256": "74f90eabcb34809e44300535016f25eb0cf4a500763c0d18e7f587583b5b9908", + "miri_sysroot_strip_prefix": "aarch64-unknown-linux-gnu" } }, "ferrocene_x86_64_pc_nto_qnx800": { "repoRuleId": "@@score_toolchains_rust+//extensions:ferrocene_toolchain_ext.bzl%ferrocene_toolchain_repo", "attributes": { - "url": "https://github.com/eclipse-score/ferrocene_toolchain_builder/releases/download/1.0.1/ferrocene-779fbed05ae9e9fe2a04137929d99cc9b3d516fd-x86_64-pc-nto-qnx800.tar.gz", - "sha256": "6daabbe20c0b06551335f83c2490326ce447759628dea04cd1c90d297c3a0bd3", + "url": "https://github.com/eclipse-score/ferrocene_toolchain_builder/releases/download/1.2.0/ferrocene-779fbed05ae9e9fe2a04137929d99cc9b3d516fd-x86_64-pc-nto-qnx800.tar.gz", + "sha256": "3fede22a89d7431668d4bc2810147a957d2b334ee8cb7097ad9c56b546f805cc", "strip_prefix": "", "toolchain_name": "rust_ferrocene", "target_triple": "x86_64-pc-nto-qnx800", @@ -10271,16 +10356,19 @@ "@platforms//cpu:x86_64", "@platforms//os:qnx" ], - "coverage_tools_url": "https://github.com/eclipse-score/ferrocene_toolchain_builder/releases/download/1.0.1/coverage-tools-779fbed05ae9e9fe2a04137929d99cc9b3d516fd-x86_64-unknown-linux-gnu.tar.gz", - "coverage_tools_sha256": "497958e925bc94833ea226d68f6d5ba38bd890f571c73e230141d2923e30dd94", - "coverage_tools_strip_prefix": "779fbed05ae9e9fe2a04137929d99cc9b3d516fd/x86_64-unknown-linux-gnu" + "coverage_tools_url": "https://github.com/eclipse-score/ferrocene_toolchain_builder/releases/download/1.2.0/coverage-tools-779fbed05ae9e9fe2a04137929d99cc9b3d516fd-x86_64-unknown-linux-gnu.tar.gz", + "coverage_tools_sha256": "841172d34b2fc0a8bed2756cf16f38d29ac18c13ee29fbb87af3ae047aa2a6a0", + "coverage_tools_strip_prefix": "779fbed05ae9e9fe2a04137929d99cc9b3d516fd/x86_64-unknown-linux-gnu", + "miri_sysroot_url": "https://github.com/eclipse-score/ferrocene_toolchain_builder/releases/download/1.2.0/miri-sysroot-779fbed05ae9e9fe2a04137929d99cc9b3d516fd-x86_64-pc-nto-qnx800.tar.gz", + "miri_sysroot_sha256": "ac434b7dc3cc3d67d31f73513a027aea50cca355c189c3a3f8c3162b1fccbca0", + "miri_sysroot_strip_prefix": "x86_64-pc-nto-qnx800" } }, "ferrocene_aarch64_unknown_nto_qnx800": { "repoRuleId": "@@score_toolchains_rust+//extensions:ferrocene_toolchain_ext.bzl%ferrocene_toolchain_repo", "attributes": { - "url": "https://github.com/eclipse-score/ferrocene_toolchain_builder/releases/download/1.0.1/ferrocene-779fbed05ae9e9fe2a04137929d99cc9b3d516fd-aarch64-unknown-nto-qnx800.tar.gz", - "sha256": "563a2438324ee1c6fdcfd13fbe352bedf1cf3f0756d07bb7ba7bdca334df92bf", + "url": "https://github.com/eclipse-score/ferrocene_toolchain_builder/releases/download/1.2.0/ferrocene-779fbed05ae9e9fe2a04137929d99cc9b3d516fd-aarch64-unknown-nto-qnx800.tar.gz", + "sha256": "d5ccceb0e3118a5e6bfdf1a3f894054db3c2cd346f927b39a57a69faf688849d", "strip_prefix": "", "toolchain_name": "rust_ferrocene", "target_triple": "aarch64-unknown-nto-qnx800", @@ -10306,9 +10394,12 @@ "@platforms//cpu:aarch64", "@platforms//os:qnx" ], - "coverage_tools_url": "https://github.com/eclipse-score/ferrocene_toolchain_builder/releases/download/1.0.1/coverage-tools-779fbed05ae9e9fe2a04137929d99cc9b3d516fd-x86_64-unknown-linux-gnu.tar.gz", - "coverage_tools_sha256": "497958e925bc94833ea226d68f6d5ba38bd890f571c73e230141d2923e30dd94", - "coverage_tools_strip_prefix": "779fbed05ae9e9fe2a04137929d99cc9b3d516fd/x86_64-unknown-linux-gnu" + "coverage_tools_url": "https://github.com/eclipse-score/ferrocene_toolchain_builder/releases/download/1.2.0/coverage-tools-779fbed05ae9e9fe2a04137929d99cc9b3d516fd-x86_64-unknown-linux-gnu.tar.gz", + "coverage_tools_sha256": "841172d34b2fc0a8bed2756cf16f38d29ac18c13ee29fbb87af3ae047aa2a6a0", + "coverage_tools_strip_prefix": "779fbed05ae9e9fe2a04137929d99cc9b3d516fd/x86_64-unknown-linux-gnu", + "miri_sysroot_url": "https://github.com/eclipse-score/ferrocene_toolchain_builder/releases/download/1.2.0/miri-sysroot-779fbed05ae9e9fe2a04137929d99cc9b3d516fd-aarch64-unknown-nto-qnx800.tar.gz", + "miri_sysroot_sha256": "8fc8f406c33a7dc31362133b8a2ffbb66b44f62354bfc98a3bc21a1fcbc9a7e6", + "miri_sysroot_strip_prefix": "aarch64-unknown-nto-qnx800" } } }, diff --git a/bazel_common/score_modules_target_sw.MODULE.bazel b/bazel_common/score_modules_target_sw.MODULE.bazel index 70634a40c99..9985b3fbffa 100644 --- a/bazel_common/score_modules_target_sw.MODULE.bazel +++ b/bazel_common/score_modules_target_sw.MODULE.bazel @@ -93,6 +93,7 @@ git_override( patch_strip = 1, patches = [ "//patches/time:001-memory-shared-moved-to-communication.patch", + "//patches/time:002-lifecycle-client-moved-to-launch-manager.patch", ], remote = "https://github.com/eclipse-score/time.git", ) diff --git a/bazel_common/score_rust_toolchains.MODULE.bazel b/bazel_common/score_rust_toolchains.MODULE.bazel index 44e64b55712..2ddc1ccc1f1 100644 --- a/bazel_common/score_rust_toolchains.MODULE.bazel +++ b/bazel_common/score_rust_toolchains.MODULE.bazel @@ -12,7 +12,7 @@ # ******************************************************************************* bazel_dep(name = "rules_rust", version = "0.68.1-score") -bazel_dep(name = "score_toolchains_rust", version = "0.8.0", dev_dependency = True) +bazel_dep(name = "score_toolchains_rust", version = "0.9.1", dev_dependency = True) ferrocene = use_extension( "@score_toolchains_rust//extensions:ferrocene_toolchain_ext.bzl", diff --git a/bazel_common/score_test_artifact_versions.MODULE.bazel b/bazel_common/score_test_artifact_versions.MODULE.bazel index e860cb7c186..889c936902c 100644 --- a/bazel_common/score_test_artifact_versions.MODULE.bazel +++ b/bazel_common/score_test_artifact_versions.MODULE.bazel @@ -34,8 +34,9 @@ single_version_override( version = "0.68.2-score", ) -# Ferrocene coverage tooling behind those .profraw files. +# Ferrocene coverage tooling behind those .profraw files. Must move in the same commit as the +# bazel_dep in score_rust_toolchains.MODULE.bazel, or the bump is overruled back to this value. single_version_override( module_name = "score_toolchains_rust", - version = "0.8.0", + version = "0.9.1", ) diff --git a/ci/README.md b/ci/README.md new file mode 100644 index 00000000000..93180d8ba3b --- /dev/null +++ b/ci/README.md @@ -0,0 +1,168 @@ + + +# DR-008 two-stage test execution — running it locally + +Every step of [`.github/workflows/dr8_test_execution.yml`](../.github/workflows/dr8_test_execution.yml) +reproduced on a workstation. The commands are the ones the workflow runs, so a local failure is +the same failure CI reports. + +**Prerequisites:** `bazel` (or `bazelisk`), `git`, and `lcov` for Stage 2 +(`sudo apt-get install -y lcov`, which provides `genhtml`). The scripts run as `bazel run` +targets, so their interpreter comes from Bazel's toolchain rather than the machine — a system +`python3` is needed only by the shell snippet below that picks a module out of the JSON. + +## What the two stages are + +| Stage | Bazel root | Question it answers | +|---|---|---| +| 1 — integration | `reference_integration` | Do the pinned modules integrate, and which versions does MVS actually select? | +| 2 — per module | the module under test | Does each module's own test suite pass against *those* versions? | + +Stage 1's output is the `stage1-resolved-deps` artifact. Stage 2 consumes it, so Stage 1 must be +run first — there is no way to reproduce Stage 2 alone. + +## Stage 1 — platform build and feature integration tests + +```bash +bazel test --lockfile_mode=error --config=linux-x86_64 //feature_integration_tests/test_cases:fit +``` + +Then export the resolved dependency set that Stage 2 pins against: + +```bash +mkdir -p artifacts/stage1-resolved-deps + +# --verbose populates originalVersion. Without it every pin report verdict is "unknown" and +# no consumer version difference can be detected. +bazel mod graph --verbose --output=json --lockfile_mode=error > resolved_graph.json + +bazel run //scripts/known_good:resolve_deps -- \ + --mod-graph resolved_graph.json \ + --export artifacts/stage1-resolved-deps/resolved_versions.json + +cp MODULE.bazel.lock artifacts/stage1-resolved-deps/ +``` + +`artifacts/stage1-resolved-deps/` then holds what CI uploads: + +| File | Purpose | +|---|---| +| `resolved_versions.json` | The manifest: every dependency's resolved version or commit, plus the patches ref_int applies to it. | +| `graph.json` | The post-MVS graph, so Stage 2 can pin a module's whole transitive closure rather than only its direct deps. | +| `patches/` | The patch files themselves. Stage 2 re-hosts them inside the module checkout, so an injected dependency is built exactly as Stage 1 built it. | +| `resolved_pins_report.json` | Where each pin came from, which consumers asked for something else, and which overrides the manifest could not carry. Read this first when a Stage 2 module fails on a version. | +| `MODULE.bazel.lock` | Evidence of full resolution; not read by Stage 2. | + +## Stage 2 — one module's unit tests and coverage + +CI derives the module list from `known_good.json` rather than hardcoding it: + +```bash +bazel run --ui_event_filters=-info,-stdout --noshow_progress \ + //scripts/known_good:list_modules -- --group target_sw +``` + +Pick one module and check it out at its `known_good` commit: + +```bash +MODULE=score_logging +SLUG=$(bazel run --ui_event_filters=-info,-stdout --noshow_progress \ + //scripts/known_good:list_modules -- --group target_sw \ + | python3 -c "import json,sys;print(next(m['slug'] for m in json.load(sys.stdin) if m['name']=='$MODULE'))") +COMMIT=$(bazel run --ui_event_filters=-info,-stdout --noshow_progress \ + //scripts/known_good:list_modules -- --group target_sw \ + | python3 -c "import json,sys;print(next(m['commit'] for m in json.load(sys.stdin) if m['name']=='$MODULE'))") + +rm -rf _module +git clone "https://github.com/${SLUG}.git" _module +git -C _module checkout "$COMMIT" +``` + +Run the module's tests against Stage 1's resolved set: + +```bash +bazel run //scripts:quality_runners -- \ + --modules-to-test "$MODULE" \ + --module-dir _module \ + --resolved-deps artifacts/stage1-resolved-deps +``` + +In order, this applies the module's own `bazel_patches`, overwrites `_module/MODULE.bazel` with +ref_int's resolved overrides, re-hosts each pinned dependency's patches under +`_module/ref_int_patches/`, deletes the now-stale `_module/MODULE.bazel.lock`, pins +`.bazelversion` to ref_int's, then runs an analysis-only gate (`build --nobuild`) before the +tests and coverage. + +**It also writes outside `_module/`**, into your ref_int checkout: +`docs/verification_report/{unit_test_summary.md,coverage_summary.md,failure_attribution.json}` +and `artifacts/coverage/`. The first two are tracked files, so a local run leaves them modified — +`git checkout -- docs/verification_report/` afterwards. + +The module's own `.bazelrc` is not read (`--noworkspace_rc`), so +[`ci/stage2/module.bazelrc`](stage2/module.bazelrc) is the single source of common config. +Two exceptions, both listed at the top of `scripts/quality_runners.py`: `score_communication` +also gets `ci/stage2/score_communication.bazelrc`, and `score_config_management` keeps its own +`.bazelrc` (`MODULES_WITH_OWN_RC`) because ref_int cannot yet replace its libclang registration. + +## Verify the module really built against ref_int's pins + +```bash +bazel run //scripts/known_good:verify_stage2_resolution -- \ + --mod-graph _module/module_graph.json \ + --resolved artifacts/stage1-resolved-deps/resolved_versions.json \ + --module-bazel _module/MODULE.bazel \ + --module "$MODULE" +``` + +Fails when ref_int injected an override that did not take effect, or when a dependency ref_int +patches was pinned without its patches. Warns for anything ref_int never pinned — a +`dev_dependency`, or a dep behind an `archive_override` the manifest cannot express. + +## Aggregate the report + +The aggregator reads one `stage2-report-/` directory per module, the layout CI gets from +downloading the per-module artifacts. Locally, stage the run you just did into that shape: + +```bash +mkdir -p "_stage2_reports/stage2-report-${MODULE}" +cp docs/verification_report/* "_stage2_reports/stage2-report-${MODULE}/" + +bazel run --ui_event_filters=-info,-stdout --noshow_progress \ + //scripts:aggregate_quality_report -- \ + --stage1-result success \ + --stage2-result success \ + --stage2-dir _stage2_reports/ +``` + +## The Python unit tests behind all of this + +```bash +bazel test //scripts:all_python_unit_tests +``` + +## When a Stage 2 module fails + +Attribute the failure before debugging it — the harness records who owns it: + +1. `resolved_pins_report.json` — did ref_int impose a version the module never asked for? + `uncarried` lists overrides ref_int declares but cannot carry into Stage 2. +2. The `verify_stage2_resolution` output — an `::error::` there means ref_int's injection failed + (a ref_int defect); a `::warning::` means the module resolved that dependency itself. +3. `quality_runners`' own warnings — a dependency pinned without its patches, or a module that + resolves dependencies in ways ref_int's pins cannot reach (its own `*_override`, or an + `http_archive`-style fetch outside bzlmod). +4. `docs/verification_report/failure_attribution.json` — written when the resolution gate fails, + recording whether the cause was an integration conflict or a ref_int harness defect. diff --git a/ci/stage2/module.bazelrc b/ci/stage2/module.bazelrc new file mode 100644 index 00000000000..ad8f27cce7c --- /dev/null +++ b/ci/stage2/module.bazelrc @@ -0,0 +1,98 @@ +# ******************************************************************************* +# Copyright (c) 2026 Contributors to the Eclipse Foundation +# +# See the NOTICE file(s) distributed with this work for additional +# information regarding copyright ownership. +# +# This program and the accompanying materials are made available under the +# terms of the Apache License Version 2.0 which is available at +# https://www.apache.org/licenses/LICENSE-2.0 +# +# SPDX-License-Identifier: Apache-2.0 +# ******************************************************************************* +# +# DR-008 Stage 2 configuration, owned by reference_integration. Sole source of common config: +# quality_runners.py passes --noworkspace_rc, so the module's own .bazelrc is not read. + +# Registries must be unconditional -- repo mapping is computed before command configs apply. +common --registry=https://raw.githubusercontent.com/eclipse-score/bazel_registry/main/ +common --registry=https://bcr.bazel.build + +# Download resilience. Lost under --noworkspace_rc since each module's own .bazelrc carried it. +common --experimental_repository_downloader_retries=10 +common --experimental_scale_timeouts=2.0 +common --http_connector_attempts=10 +common --http_connector_retry_max_timeout=1s +common --http_timeout_scaling=2.0 + +# ─── Repo-scoped build settings, ported from ref_int's root .bazelrc ───────── +# Opt-in per module, because a build setting can only be loaded by a module whose graph contains +# the repo that defines it. Set unconditionally, they abort option loading with "No repository +# visible as '@'" for every module that does not depend on it (score_kyron and +# score_orchestrator depend on neither). +build:stage2-baselibs-flags --@score_baselibs//score/log_rust:safety_level=qm +build:stage2-communication-flags --@score_communication//score/memory/shared/flags:use_typedshmd=False + +# rules_android is pulled in transitively (grpc-java -> rules_jvm_external) and evaluates +# android_sdk_repository, which fails when ANDROID_HOME points at an incomplete SDK, as on +# CI runners after the disk-cleanup step. Only score_baselibs guards against this itself. +common --repo_env=ANDROID_HOME= + +# ─── stage2-linux-x86_64: emitted unconditionally by quality_runners.py ────── +build:stage2-linux-x86_64 --host_platform=@score_bazel_platforms//:x86_64-linux-gcc_12.2.0-posix +build:stage2-linux-x86_64 --platforms=@score_bazel_platforms//:x86_64-linux-gcc_12.2.0-posix + +# Test selection and coverage policy. `coverage` inherits `test` inherits `build`. +# -miri: ref_int registers no miri toolchain. -no-coverage: gcov-instrumenting a TSAN +# binary reports false races on the non-atomic __gcov* counters. +test:stage2-linux-x86_64 --build_tests_only +test:stage2-linux-x86_64 --test_tag_filters=-manual,-miri,-no-coverage +test:stage2-linux-x86_64 --test_output=errors +test:stage2-linux-x86_64 --test_summary=testcase +test:stage2-linux-x86_64 --test_verbose_timeout_warnings +test:stage2-linux-x86_64 --test_timeout=1200 +test:stage2-linux-x86_64 --nocache_test_results + +coverage:stage2-linux-x86_64 --features=coverage +coverage:stage2-linux-x86_64 --combined_report=lcov +# Make gcov counter updates atomic so a multithreaded coverage test is race-free. +coverage:stage2-linux-x86_64 --copt=-fprofile-update=atomic +coverage:stage2-linux-x86_64 --linkopt=-fprofile-update=atomic + +# ─── stage2-gcc: score's gcc x86_64 toolchain ──────────────────────────────── +# Opt-in because the target name is generated by each module's own gcc.toolchain() call: +# score_communication passes use_base_constraints_only = True, which yields :x86_64-linux +# instead, and it registers its own cc toolchain unconditionally — so it omits this. +build:stage2-gcc --extra_toolchains=@score_gcc_x86_64_toolchain//:x86_64-linux-gcc_12.2.0 + +# ─── stage2-rust: ferrocene Rust toolchain ─────────────────────────────────── +# Opt-in because score_time declares no score_toolchains_rust, so the apparent repo name +# does not resolve in its graph. Folds into the base once Phase 1 injects a bazel_dep stub +# for every module in the resolved set (PR #278). +build:stage2-rust --extra_toolchains=@score_toolchains_rust//toolchains/ferrocene:ferrocene_x86_64_unknown_linux_gnu + +# ─── ferrocene-coverage: Rust coverage instrumentation ─────────────────────── +# Added in code by stage2_config_flags, never opted into via known_good.json: rustc must emit +# .profraw during the same run ferrocene_report later reads. kyron/persistency/lifecycle_health +# define this name identically themselves (layering repeats the same values, a no-op); +# score_logging has no Rust instrumentation config, so this is its only source. +build:ferrocene-coverage --@rules_rust//rust/settings:extra_rustc_flag=-Cinstrument-coverage +build:ferrocene-coverage --@rules_rust//rust/settings:extra_rustc_flag=-Clink-dead-code +build:ferrocene-coverage --@rules_rust//rust/settings:extra_rustc_flag=-Ccodegen-units=1 +build:ferrocene-coverage --@rules_rust//rust/settings:extra_rustc_flag=-Cdebuginfo=2 +build:ferrocene-coverage --@rules_rust//rust/settings:extra_exec_rustc_flag=-Cinstrument-coverage +build:ferrocene-coverage --@rules_rust//rust/settings:extra_exec_rustc_flag=-Clink-dead-code +build:ferrocene-coverage --@rules_rust//rust/settings:extra_exec_rustc_flag=-Ccodegen-units=1 +build:ferrocene-coverage --@rules_rust//rust/settings:extra_exec_rustc_flag=-Cdebuginfo=2 +test:ferrocene-coverage --run_under=@score_tooling//coverage:llvm_profile_wrapper + +# Coverage needs to have all intermediate .rlibs to be able to proceed +build:ferrocene-coverage --remote_download_all + +# score_persistency's rust_coverage_config; its own .bazelrc does not define this name, so this +# is the sole source. Only the two score_baselibs settings are ported -- deps persistency itself +# declares, so they resolve in its checkout. ref_int's root .bazelrc has a third, +# @score_logging-relative one, which persistency's `extra_test_config` already passes directly. +build:ferrocene-coverage-per --config=ferrocene-coverage +build:ferrocene-coverage-per --@score_baselibs//score/log_rust:safety_level=qm +build:ferrocene-coverage-per --@score_baselibs//score/json:base_library=nlohmann diff --git a/ci/stage2/score_communication.bazelrc b/ci/stage2/score_communication.bazelrc new file mode 100644 index 00000000000..af43fbef3d9 --- /dev/null +++ b/ci/stage2/score_communication.bazelrc @@ -0,0 +1,34 @@ +# ******************************************************************************* +# Copyright (c) 2026 Contributors to the Eclipse Foundation +# +# See the NOTICE file(s) distributed with this work for additional +# information regarding copyright ownership. +# +# This program and the accompanying materials are made available under the +# terms of the Apache License Version 2.0 which is available at +# https://www.apache.org/licenses/LICENSE-2.0 +# +# SPDX-License-Identifier: Apache-2.0 +# ******************************************************************************* +# +# DR-008 Stage 2 — score_communication only, read after ci/stage2/module.bazelrc under +# --noworkspace_rc. Separate file because these are `//`-relative labels, valid only when this +# module is Bazel root. + +# libclang for this module's code generation. ref_int can't share one instance across modules +# (needs @llvm_toolchain_llvm, root-only under bzlmod), so this registers the module's own. +build --extra_toolchains=//bazel/toolchains:score_communication_libclang_toolchain + +# cc toolchain: this module's gcc.toolchain() call names its target differently, so it can't use +# stage2-gcc. Both lines required, or the gate passes on the wrong (auto-detected) toolchain. +build --extra_toolchains=@gcc_toolchain_x86_64//:cc_toolchain +build --action_env=BAZEL_DO_NOT_DETECT_CPP_TOOLCHAIN=1 + +build --@score_baselibs//score/json:base_library=nlohmann +build --@score_baselibs//score/analysis/tracing/generic_trace_library:implementation=@score_baselibs//score/analysis/tracing/generic_trace_library/stub_implementation +build --incompatible_strict_action_env +build --experimental_retain_test_configuration_across_testonly # bazelbuild/bazel#6842 + +test --sandbox_tmpfs_path=/dev/shm +test --sandbox_tmpfs_path=/tmp +test --nosandbox_default_allow_network diff --git a/known_good.json b/known_good.json index 9389a21a057..6f71f52edc0 100644 --- a/known_good.json +++ b/known_good.json @@ -12,14 +12,25 @@ "@score_baselibs//score/json:base_library=nlohmann" ], "exclude_test_targets": [ + "//score/language/safecpp/aborts_upon_exception:abortsuponexception_toolchain_test" + ], + "langs": [ + "cpp" + ], + "bazel_config": [ + "stage2-gcc", + "stage2-rust", + "stage2-baselibs-flags" + ], + "exclude_test_target_reasons": { + "//score/language/safecpp/aborts_upon_exception:abortsuponexception_toolchain_test": "Stage 2's toolchain does not implement the aborts_upon_exception feature." + }, + "legacy_exclude_test_targets": [ "//score/language/safecpp/aborts_upon_exception:abortsuponexception_toolchain_test", "//score/containers:dynamic_array_test", "//score/mw/log/configuration:*", "//score/json/examples:*", "//score/flatbuffers:version_reader_test" - ], - "langs": [ - "cpp" ] } }, @@ -36,7 +47,12 @@ "extra_test_config": [ "@score_communication//score/memory/shared/flags:use_typedshmd=False" ], - "exclude_test_targets": [ + "exclude_test_targets": [], + "bazel_config": [ + "stage2-rust", + "stage2-baselibs-flags" + ], + "legacy_exclude_test_targets": [ "//score/mw/com/impl:unit_test_runtime_single_exec", "//score/mw/com/impl:runtime_test", "//score/mw/com/impl/configuration:config_parser_test", @@ -66,7 +82,19 @@ "exclude_test_targets": [ "//src/cpp/tests:bm_kvs_cpp" ], - "rust_coverage_config": "ferrocene-coverage-per" + "rust_coverage_config": "ferrocene-coverage-per", + "bazel_config": [ + "stage2-gcc", + "stage2-rust", + "stage2-communication-flags", + "stage2-baselibs-flags" + ], + "exclude_test_target_reasons": { + "//src/cpp/tests:bm_kvs_cpp": "google_benchmark benchmark declared as a cc_test; a timing measurement, not a correctness test, so it is not meaningful in a UT/coverage run." + }, + "legacy_exclude_test_targets": [ + "//src/cpp/tests:bm_kvs_cpp" + ] } }, "score_orchestrator": { @@ -76,6 +104,10 @@ "code_root_path": "//src/...", "langs": [ "rust" + ], + "bazel_config": [ + "stage2-gcc", + "stage2-rust" ] } }, @@ -86,6 +118,10 @@ "code_root_path": "//src/...", "langs": [ "rust" + ], + "bazel_config": [ + "stage2-gcc", + "stage2-rust" ] } }, @@ -96,6 +132,19 @@ "metadata": { "code_root_path": "//score/...", "exclude_test_targets": [ + "//score/health_monitor/src/rust:miri_tests", + "//score/launch_manager/src/daemon/src/common/concurrency:mpmc_concurrent_queue_tsan_test" + ], + "bazel_config": [ + "stage2-gcc", + "stage2-rust", + "stage2-baselibs-flags" + ], + "exclude_test_target_reasons": { + "//score/health_monitor/src/rust:miri_tests": "miri_test rule (tags=[\"miri\"]); needs the miri interpreter toolchain, which Stage 2's configs do not provide.", + "//score/launch_manager/src/daemon/src/common/concurrency:mpmc_concurrent_queue_tsan_test": "built with -fsanitize=thread (tags=[\"tsan\"]); needs the TSan runtime and a sanitizer build, incompatible with the coverage configuration." + }, + "legacy_exclude_test_targets": [ "//score/health_monitor/src/rust:miri_tests", "//score/launch_manager/src/daemon/src/common/concurrency:mpmc_concurrent_queue_tsan_test", "//score/launch_manager/src/daemon/src/common/concurrency:mpsc_bounded_queue_tsan_test" @@ -121,7 +170,14 @@ "@score_logging//score/datarouter/build_configuration_flags:file_transfer=False", "@score_logging//score/datarouter/build_configuration_flags:use_local_vlan=True" ], - "exclude_test_targets": [ + "exclude_test_targets": [], + "bazel_config": [ + "stage2-gcc", + "stage2-rust", + "stage2-communication-flags", + "stage2-baselibs-flags" + ], + "legacy_exclude_test_targets": [ "//score/mw/log/legacy_non_verbose_api:unit_test" ] } @@ -130,11 +186,17 @@ "repo": "https://github.com/eclipse-score/time.git", "hash": "8c42d34698535dabeaa4782d5ec123256151fb30", "bazel_patches": [ - "//patches/time:001-memory-shared-moved-to-communication.patch" + "//patches/time:001-memory-shared-moved-to-communication.patch", + "//patches/time:002-lifecycle-client-moved-to-launch-manager.patch" ], "metadata": { "langs": [ "cpp" + ], + "bazel_config": [ + "stage2-gcc", + "stage2-communication-flags", + "stage2-baselibs-flags" ] } }, @@ -158,6 +220,24 @@ ], "langs": [ "cpp" + ], + "exclude_test_target_reasons": { + "//score/config_management/config_daemon/code/factory/details:unit_test_mw_com": "Excluded when the module joined ref_int (#299): depends on mw_com/SOCAL targets not present in the OSS build, which ref_int's patches/config_management/001-adapt-proxy-api-and-remove-internal-targets.patch removes. Carried over unchanged; not yet re-audited for Stage 2.", + "//score/config_management/config_daemon/code/services/details/mw_com:unit_test": "Excluded when the module joined ref_int (#299): depends on mw_com/SOCAL targets not present in the OSS build, which ref_int's patches/config_management/001-adapt-proxy-api-and-remove-internal-targets.patch removes. Carried over unchanged; not yet re-audited for Stage 2.", + "//score/config_management/config_provider/code/config_provider/factory:unit_tests_mw_com": "Excluded when the module joined ref_int (#299): depends on mw_com/SOCAL targets not present in the OSS build, which ref_int's patches/config_management/001-adapt-proxy-api-and-remove-internal-targets.patch removes. Carried over unchanged; not yet re-audited for Stage 2.", + "//score/config_management/config_provider/code/proxies/details:unit_test_mw": "Excluded when the module joined ref_int (#299): depends on mw_com/SOCAL targets not present in the OSS build, which ref_int's patches/config_management/001-adapt-proxy-api-and-remove-internal-targets.patch removes. Carried over unchanged; not yet re-audited for Stage 2.", + "//score/config_management/dependability/...": "Excluded when the module joined ref_int (#299): depends on mw_com/SOCAL targets not present in the OSS build, which ref_int's patches/config_management/001-adapt-proxy-api-and-remove-internal-targets.patch removes. Carried over unchanged; not yet re-audited for Stage 2." + }, + "legacy_exclude_test_targets": [ + "//score/config_management/config_daemon/code/factory/details:unit_test_mw_com", + "//score/config_management/config_daemon/code/services/details/mw_com:unit_test", + "//score/config_management/config_provider/code/config_provider/factory:unit_tests_mw_com", + "//score/config_management/config_provider/code/proxies/details:unit_test_mw", + "//score/config_management/dependability/..." + ], + "bazel_config": [ + "stage2-communication-flags", + "stage2-baselibs-flags" ] } } diff --git a/patches/logging/005-memory-shared-moved-to-communication.patch b/patches/logging/005-memory-shared-moved-to-communication.patch index 3ff1f5b1a12..b7799fe8393 100644 --- a/patches/logging/005-memory-shared-moved-to-communication.patch +++ b/patches/logging/005-memory-shared-moved-to-communication.patch @@ -3,6 +3,8 @@ b90e0695 ("Migrate score/memory/shared to score_communication"). score_logging still points at the old @score_baselibs labels; score_communication is already a bazel_dep here, so only the labels need redirecting. +Source labels only; the module's .bazelrc has the same stale labels but ref_int no longer patches it. + diff --git a/score/mw/log/backend/BUILD b/score/mw/log/backend/BUILD index 846c781f..de1997f8 100644 --- a/score/mw/log/backend/BUILD diff --git a/patches/time/001-memory-shared-moved-to-communication.patch b/patches/time/001-memory-shared-moved-to-communication.patch index 8ab2b311057..dbde756ff5c 100644 --- a/patches/time/001-memory-shared-moved-to-communication.patch +++ b/patches/time/001-memory-shared-moved-to-communication.patch @@ -3,6 +3,8 @@ b90e0695 ("Migrate score/memory/shared to score_communication"). score_time HEAD still points at the old @score_baselibs labels, so redirect them and declare the score_communication dependency they now require. +Source labels only; the module's .bazelrc has the same stale labels but ref_int no longer patches it. + diff --git a/MODULE.bazel b/MODULE.bazel index 19aa03e7..839777b7 100644 --- a/MODULE.bazel diff --git a/patches/time/002-lifecycle-client-moved-to-launch-manager.patch b/patches/time/002-lifecycle-client-moved-to-launch-manager.patch new file mode 100644 index 00000000000..e6d3c1c05be --- /dev/null +++ b/patches/time/002-lifecycle-client-moved-to-launch-manager.patch @@ -0,0 +1,30 @@ +Upstream lifecycle moved src/lifecycle_client_lib to +score/launch_manager/src/lifecycle_client, re-exported publicly as +//score/launch_manager:lifecycle_cc. score_time at its pin still names the old path. +Upstream time main already carries this redirect; retire on the next time pin bump. + +diff --git a/score/time_daemon/src/application/BUILD b/score/time_daemon/src/application/BUILD +index e8019b4..57ade29 100644 +--- a/score/time_daemon/src/application/BUILD ++++ b/score/time_daemon/src/application/BUILD +@@ -42,7 +42,7 @@ cc_binary( + "//score/time_daemon/src/application/svt:svt_handler", + "//score/time_daemon/src/common:logging_contexts", + "@score_baselibs//score/concurrency", +- "@score_lifecycle_health//src/lifecycle_client_lib", ++ "@score_lifecycle_health//score/launch_manager:lifecycle_cc", + # "@score_logging//score/mw/log", + "@score_baselibs//score/mw/log:console_only_backend", + ], +diff --git a/score/time_slave/src/application/BUILD b/score/time_slave/src/application/BUILD +index 81d686f..34ed6ef 100644 +--- a/score/time_slave/src/application/BUILD ++++ b/score/time_slave/src/application/BUILD +@@ -28,6 +28,6 @@ cc_binary( + "//score/time_slave/src/gptp:gptp_engine", + "//score/ts_client/src:gptp_ipc", + "@score_baselibs//score/mw/log:console_only_backend", +- "@score_lifecycle_health//src/lifecycle_client_lib", ++ "@score_lifecycle_health//score/launch_manager:lifecycle_cc", + ], + ) diff --git a/rust_coverage/BUILD b/rust_coverage/BUILD index 3cdb06861b8..5144f162c2c 100644 --- a/rust_coverage/BUILD +++ b/rust_coverage/BUILD @@ -22,7 +22,7 @@ rust_coverage_report( "linux-x86_64", "ferrocene-coverage", ], - query = 'kind("rust_test", @score_communication//score/mw/com/impl/...) -@score_communication//score/mw/com/impl:unit_test_runtime_single_exec -@score_communication//score/mw/com/impl:runtime_test -@score_communication//score/mw/com/impl/configuration:config_parser_test -@score_communication//score/mw/com/impl/configuration:configuration_test -@score_communication//score/mw/com/impl/configuration:configuration_json_parsing_strategy_test -@score_communication//score/mw/com/impl/tracing/configuration:tracing_filter_config_parser_test -@score_communication//score/mw/com/impl/tracing:tracing_runtime_test -@score_communication//score/mw/com/impl/bindings/lola/tracing:tracing_runtime_test', + query = 'kind("rust_test", @score_communication//score/mw/com/impl/...)', visibility = ["//visibility:public"], ) @@ -62,7 +62,7 @@ rust_coverage_report( "linux-x86_64", "ferrocene-coverage", ], - query = 'kind("rust_test", @score_lifecycle_health//score/...) -@score_lifecycle_health//score/health_monitor/src/rust:miri_tests -@score_lifecycle_health//score/launch_manager/src/daemon/src/common/concurrency:mpmc_concurrent_queue_tsan_test -@score_lifecycle_health//score/launch_manager/src/daemon/src/common/concurrency:mpsc_bounded_queue_tsan_test', + query = 'kind("rust_test", @score_lifecycle_health//score/...) -@score_lifecycle_health//score/health_monitor/src/rust:miri_tests -@score_lifecycle_health//score/launch_manager/src/daemon/src/common/concurrency:mpmc_concurrent_queue_tsan_test', visibility = ["//visibility:public"], ) @@ -72,6 +72,6 @@ rust_coverage_report( "linux-x86_64", "ferrocene-coverage", ], - query = 'kind("rust_test", @score_logging//score/mw/log/...) -@score_logging//score/mw/log/legacy_non_verbose_api:unit_test', + query = 'kind("rust_test", @score_logging//score/mw/log/...)', visibility = ["//visibility:public"], ) diff --git a/scripts/BUILD b/scripts/BUILD new file mode 100644 index 00000000000..ae3ec7cac8c --- /dev/null +++ b/scripts/BUILD @@ -0,0 +1,64 @@ +# ******************************************************************************* +# Copyright (c) 2026 Contributors to the Eclipse Foundation +# +# See the NOTICE file(s) distributed with this work for additional +# information regarding copyright ownership. +# +# This program and the accompanying materials are made available under the +# terms of the Apache License Version 2.0 which is available at +# https://www.apache.org/licenses/LICENSE-2.0 +# +# SPDX-License-Identifier: Apache-2.0 +# ******************************************************************************* +load("@rules_python//python:defs.bzl", "py_binary", "py_library") +load("@score_tooling//python_basics:defs.bzl", "score_py_pytest") + +# The Stage-2 driver and the report aggregator. +py_library( + name = "quality_scripts", + srcs = [ + "aggregate_quality_report.py", + "quality_runners.py", + ], + visibility = ["//visibility:public"], + deps = ["//scripts/known_good"], +) + +# No `data` needed: the tests point STAGE2_RC at a temp file rather than reading +# ci/stage2/module.bazelrc, so they do not depend on runfiles layout. +score_py_pytest( + name = "quality_scripts_tests", + srcs = glob(["tests/**/*.py"]), + pytest_config = "//:pyproject.toml", + deps = [":quality_scripts"], +) + +# Stage 2's driver: checks a module out against ref_int's resolved set and runs its tests. +py_binary( + name = "quality_runners", + srcs = ["quality_runners.py"], + data = ["//:known_good.json"], + main = "quality_runners.py", + visibility = ["//visibility:public"], + deps = [":quality_scripts"], +) + +# The consolidated Stage 1 + Stage 2 report written to the job summary. +py_binary( + name = "aggregate_quality_report", + srcs = ["aggregate_quality_report.py"], + main = "aggregate_quality_report.py", + visibility = ["//visibility:public"], + deps = [":quality_scripts"], +) + +# One label for every Python unit test, so CI runs all of them by naming a single target. +test_suite( + name = "all_python_unit_tests", + tests = [ + ":quality_scripts_tests", + "//scripts/known_good:known_good_tests", + "//scripts/tooling:tooling_tests", + ], + visibility = ["//visibility:public"], +) diff --git a/scripts/aggregate_quality_report.py b/scripts/aggregate_quality_report.py new file mode 100644 index 00000000000..7a1dcde1b6e --- /dev/null +++ b/scripts/aggregate_quality_report.py @@ -0,0 +1,358 @@ +#!/usr/bin/env python3 +# ******************************************************************************* +# Copyright (c) 2026 Contributors to the Eclipse Foundation +# +# See the NOTICE file(s) distributed with this work for additional +# information regarding copyright ownership. +# +# This program and the accompanying materials are made available under the +# terms of the Apache License Version 2.0 which is available at +# https://www.apache.org/licenses/LICENSE-2.0 +# +# SPDX-License-Identifier: Apache-2.0 +# ******************************************************************************* +"""Aggregate Stage 1 and Stage 2 quality reports into a single consolidated report. + +The upstream aggregation step of DR-008 Option 4. ``--stage2-dir`` holds one +``stage2-report-/`` per module, each with the ``unit_test_summary.md`` and +``coverage_summary.md`` quality_runners.py produced. + +Usage: + python3 scripts/aggregate_quality_report.py \\ + --stage1-result success \\ + --stage2-result success \\ + --stage2-dir _stage2_reports/ \\ + >> "$GITHUB_STEP_SUMMARY" +""" + +import argparse +import json +import sys +from pathlib import Path + +_HERE = Path(__file__).resolve().parent +try: + from known_good.resolved_dependencies import workspace_path +except ImportError: + if str(_HERE) not in sys.path: + sys.path.insert(0, str(_HERE)) + from known_good.resolved_dependencies import workspace_path # noqa: E402 + +_STATUS_MAP = { + "success": "✅ Success", + "failure": "❌ Failure", + "cancelled": "⚪ Cancelled", + "skipped": "⚪ Skipped", + "": "⚪ Unknown", +} + +# Printed when an exclusion carries no recorded reason, so the gap is visible in the report +# rather than papered over with a generic justification. +_NO_EXCLUSION_REASON = "⚠️ no reason recorded" + +# Written by quality_runners.py into the same report directory; keep in sync with the constant of +# the same name there. Carries the owner of a failure, which the count columns cannot: zero tests +# looks identical for a harness defect and an integration conflict. +ATTRIBUTION_NAME = "failure_attribution.json" + +_OWNER_REF_INT = "ref_int (harness defect)" +_OWNER_MODULE = "module team (integration finding)" +_OWNER_JOINT = "integration conflict (ref_int pin ↔ module sources)" + + +def _format_status(result: str) -> str: + return _STATUS_MAP.get(result.lower().strip(), "⚪ Unknown") + + +def _read_attributions(artifact_dir: Path) -> dict[str, dict]: + """Return ``{module: {"owner", "conflicting"}}`` from one report directory. + + Missing or unparseable is ``{}``, so :func:`_classify` falls back to its count-based default + rather than the report failing on a corrupt sidecar. + """ + path = artifact_dir / ATTRIBUTION_NAME + if not path.is_file(): + return {} + try: + data = json.loads(path.read_text(encoding="utf-8")) + except ValueError: + return {} + return data if isinstance(data, dict) else {} + + +def _extract_table_data_rows(md_path: Path) -> list[str]: + """Return the data rows of the first markdown table found in md_path. + + Skips the title line (starts with #), the header row, and the separator + row (contains ---), then collects all remaining pipe-delimited lines. + """ + if not md_path.exists(): + return [] + + lines = md_path.read_text(encoding="utf-8").splitlines() + data_rows: list[str] = [] + header_seen = False + separator_seen = False + + for line in lines: + stripped = line.strip() + if not stripped.startswith("|"): + continue + if not header_seen: + header_seen = True + continue + if not separator_seen: + separator_seen = True + continue + if stripped: + data_rows.append(stripped) + + return data_rows + + +def _parse_ut_rows(rows: list[str]) -> list[tuple[str, int, int, int, int]]: + """Parse ``| module | passed | failed | skipped | total |`` rows into typed tuples. + + Rows whose numeric cells do not parse are skipped rather than crashing the report. + """ + parsed: list[tuple[str, int, int, int, int]] = [] + for row in rows: + cells = [c.strip() for c in row.strip().strip("|").split("|")] + if len(cells) < 5: + continue + try: + parsed.append((cells[0], int(cells[1]), int(cells[2]), int(cells[3]), int(cells[4]))) + except ValueError: + continue + return parsed + + +def _classify(total: int, failed: int, attribution: dict | None = None) -> tuple[str, str]: + """Return (verdict, owner) for one module's unit-test result. + + Zero tests validated nothing and always fails, but *who must act* does not follow from the + count: only ``quality_runners.classify_gate_failure`` sees which repositories the failure + named, and ``attribution`` is that verdict carried through :data:`ATTRIBUTION_NAME`. + + Tests that ran and failed are the module team's regardless of any earlier attribution; an + absent attribution keeps ref_int as the conservative default. + """ + if total == 0: + owner = (attribution or {}).get("owner", "") + conflicting = (attribution or {}).get("conflicting") or [] + if owner == "integration conflict": + over = f" over {', '.join(conflicting)}" if conflicting else "" + return f"❌ no tests executed — integration conflict{over}", _OWNER_JOINT + return "❌ no tests executed", _OWNER_REF_INT + if failed > 0: + return f"❌ {failed} failing", _OWNER_MODULE + return "✅ passed", "—" + + +def _excluded_test_targets(known_good_path: Path) -> list[tuple[str, list[tuple[str, str]]]]: + """Return [(module, [(excluded target, reason)])] for target_sw modules in known_good.json. + + These targets never run in Stage 2 and so are absent from the counts above; they remain + covered by each module's own CI. Surfacing them keeps the report honest about completeness. + + The reason is printed rather than inferred. Stage 2 runs each module as the Bazel *root*, so + the old blanket explanation -- "depends on dev_dependency-only deps invisible from the + resolved graph" -- describes a build scope that no longer exists: a root module's dev edges + are active. An exclusion that survives that change has a specific, scope-independent reason + (a benchmark, a sanitizer or miri target), and it belongs in ``metadata.exclude_test_target_reasons`` + next to the label. An entry with no recorded reason is flagged here instead of being dressed + up in a justification nobody checked. + """ + if not known_good_path.exists(): + return [] + + data = json.loads(known_good_path.read_text(encoding="utf-8")) + target_sw = data.get("modules", {}).get("target_sw", {}) + + excluded: list[tuple[str, list[tuple[str, str]]]] = [] + for name in sorted(target_sw): + metadata = target_sw[name].get("metadata", {}) + targets = metadata.get("exclude_test_targets", []) + reasons = metadata.get("exclude_test_target_reasons", {}) + if targets: + excluded.append((name, [(t, reasons.get(t, _NO_EXCLUSION_REASON)) for t in targets])) + return excluded + + +def main() -> int: + parser = argparse.ArgumentParser( + description="Aggregate Stage 1 + Stage 2 quality reports (DR-008 Option 4).", + formatter_class=argparse.RawDescriptionHelpFormatter, + epilog=( + "Examples:\n" + " python3 scripts/aggregate_quality_report.py \\\n" + " --stage1-result success \\\n" + " --stage2-result failure \\\n" + " --stage2-dir _stage2_reports/ \\\n" + " >> $GITHUB_STEP_SUMMARY\n" + ), + ) + parser.add_argument( + "--stage1-result", + default="", + help="GitHub Actions result of the stage1_integration job (success/failure/cancelled/skipped).", + ) + parser.add_argument( + "--stage2-result", + default="", + help="GitHub Actions result of the stage2_module_validation job.", + ) + parser.add_argument( + "--stage2-dir", + type=Path, + default=Path("_stage2_reports"), + help="Directory containing downloaded stage2-report-* artifact subdirectories.", + ) + parser.add_argument( + "--known-good-path", + type=Path, + default=Path("known_good.json"), + help="Path to known_good.json (used to list test targets excluded from Stage 2).", + ) + args = parser.parse_args() + # Under 'bazel run' the cwd is the runfiles tree, so relative paths need anchoring. + args.stage2_dir = workspace_path(args.stage2_dir) + args.known_good_path = workspace_path(args.known_good_path) + + out = sys.stdout + + out.write("# S-CORE Quality Report — DR-008 Option 4\n\n") + + # ------------------------------------------------------------------ + # Stage 1 summary + # ------------------------------------------------------------------ + out.write("## Stage 1 — Integration Results\n\n") + out.write("| Check | Status |\n") + out.write("|-------|--------|\n") + out.write(f"| Platform Build + Feature Integration Tests (linux-x86_64) | {_format_status(args.stage1_result)} |\n") + out.write("\n") + + # ------------------------------------------------------------------ + # Stage 2 summary — read per-module reports + # ------------------------------------------------------------------ + out.write("## Stage 2 — Module Validation Results\n\n") + + stage2_dir: Path = args.stage2_dir + ut_rows: list[str] = [] + cov_rows: list[str] = [] + attributions: dict[str, dict] = {} + + if stage2_dir.exists(): + for artifact_dir in sorted(stage2_dir.iterdir()): + if not artifact_dir.is_dir(): + continue + if not artifact_dir.name.startswith("stage2-report-"): + continue + ut_rows.extend(_extract_table_data_rows(artifact_dir / "unit_test_summary.md")) + cov_rows.extend(_extract_table_data_rows(artifact_dir / "coverage_summary.md")) + attributions.update(_read_attributions(artifact_dir)) + else: + out.write(f"*Stage 2 reports directory not found: `{stage2_dir}`*\n\n") + + if ut_rows: + out.write("### Unit Test Summary\n\n") + out.write("| module | passed | failed | skipped | total |\n") + out.write("|--------|--------|--------|---------|-------|\n") + for row in ut_rows: + out.write(f"{row}\n") + out.write("\n") + else: + out.write("*No Stage 2 unit test reports found.*\n\n") + + # Failure ownership — a Stage-2 job that ran no tests validated nothing and always fails, but + # the owner comes from the attribution Stage 2 recorded, never from the count (see _classify). + parsed = _parse_ut_rows(ut_rows) + no_tests = [name for name, _p, _f, _s, total in parsed if total == 0] + if parsed: + out.write("### Failure Ownership\n\n") + out.write("| module | tests run | verdict | owner |\n") + out.write("|--------|-----------|---------|-------|\n") + for name, _passed, failed, _skipped, total in parsed: + verdict, owner = _classify(total, failed, attributions.get(name)) + out.write(f"| {name} | {total} | {verdict} | {owner} |\n") + out.write("\n") + + if cov_rows: + out.write("### Coverage Summary\n\n") + out.write("| module | lines | functions | branches |\n") + out.write("|--------|-------|-----------|----------|\n") + for row in cov_rows: + out.write(f"{row}\n") + out.write("\n") + + # score_communication and score_orchestrator have known-broken rust coverage extraction + # (mostly proc_macro) and are excluded from the *_rust rows above in both modes — see + # DISABLED_RUST_COVERAGE in quality_runners.py. Stated explicitly so their absent row + # reads as "not measured for this module", not "not measured at all". + out.write( + "> Rust coverage is not measured for `score_communication` or `score_orchestrator` " + "(known extraction issues, mostly proc_macro). Rust *tests* do run for both; every " + "other Rust module's coverage is measured in Stage 2 the same as in the old workflow.\n\n" + ) + + # ------------------------------------------------------------------ + # Excluded test targets — completeness disclosure (DR-008 Q4) + # ------------------------------------------------------------------ + excluded = _excluded_test_targets(args.known_good_path) + if excluded: + out.write("### Test Targets Excluded from Stage 2\n\n") + out.write( + "These targets do not run in Stage 2, so they are not counted above. They are still " + "validated by each module's own CI. Stage 2 runs each module as the Bazel root, so an " + "exclusion has to justify itself on its own terms — the reason is recorded per target " + "in `known_good.json`.\n\n" + ) + out.write("| module | excluded test target | reason |\n") + out.write("|--------|----------------------|--------|\n") + for module_name, targets in excluded: + for target, reason in targets: + out.write(f"| {module_name} | `{target}` | {reason} |\n") + out.write("\n") + + # ------------------------------------------------------------------ + # Overall status + # ------------------------------------------------------------------ + out.write("## Overall Status\n\n") + stage1_ok = args.stage1_result == "success" + stage2_ok = args.stage2_result in ("success", "skipped") + # A module that configured but executed no tests is a failure: Stage 2's purpose is to + # run the module's tests against the resolved set, and zero tests validates nothing. + tests_ran = not no_tests + + if stage1_ok and stage2_ok and tests_ran: + out.write("✅ All quality checks passed.\n") + else: + out.write("❌ One or more quality checks failed — see details above.\n\n") + out.write("| Stage | Result |\n") + out.write("|-------|--------|\n") + out.write(f"| Stage 1 (integration) | {_format_status(args.stage1_result)} |\n") + out.write(f"| Stage 2 (module validation) | {_format_status(args.stage2_result)} |\n") + # Both are failures, but they go to different people, so they cannot share one heading. + conflicts = [n for n in no_tests if (attributions.get(n) or {}).get("owner") == "integration conflict"] + harness = [n for n in no_tests if n not in conflicts] + if harness: + out.write( + f"\n**ref_int harness defect** — no tests executed for: " + f"{', '.join(f'`{n}`' for n in harness)}. " + "These did not validate the resolved dependency set.\n" + ) + for name in conflicts: + over = ", ".join(f"`{c}`" for c in (attributions.get(name) or {}).get("conflicting") or []) + out.write( + f"\n**Integration conflict** — `{name}` did not run: ref_int's resolved set and the " + f"module's sources are each self-consistent but mutually incompatible" + f"{f' over {over}' if over else ''}. " + "Resolve by moving ref_int's pin or the module's `known_good` commit — not by " + "changing the harness.\n" + ) + + return 0 if (stage1_ok and stage2_ok and tests_ran) else 1 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/scripts/integration_test.py b/scripts/integration_test.py index 219905689c7..4f735bfa6cb 100755 --- a/scripts/integration_test.py +++ b/scripts/integration_test.py @@ -17,7 +17,6 @@ """ import argparse -import json import os import re import subprocess @@ -27,7 +26,6 @@ from pathlib import Path from typing import Dict, Optional, Tuple -from models.build_config import BuildModuleConfig, load_build_config from known_good.models import Module from known_good.models.known_good import load_known_good @@ -266,12 +264,6 @@ def main(): default=None, help="Path to known_good.json file (default: known_good.json in repo root)", ) - parser.add_argument( - "--build-config", - type=Path, - default=None, - help="Path to build_config.json file (default: build_config.json in repo root)", - ) parser.add_argument( "--config", default=os.environ.get("CONFIG", "x86_64-linux"), @@ -289,13 +281,6 @@ def main(): if not known_good_file: known_good_file = repo_root / "known_good.json" - build_config_file = args.build_config - if not build_config_file: - build_config_file = repo_root / "build_config.json" - - # Load build configuration - BUILD_TARGET_GROUPS = load_build_config(build_config_file) - # Create log directory log_dir.mkdir(parents=True, exist_ok=True) summary_file.parent.mkdir(parents=True, exist_ok=True) @@ -333,11 +318,16 @@ def main(): overall_depr_total = 0 any_failed = False - # Build each group - for group_name, module_config in BUILD_TARGET_GROUPS.items(): + # Derive build targets from known_good.json (build_config.json was removed in #101; + # known_good.json is the single source of truth for module locations). + all_modules = {name: module for group in new_modules.values() for name, module in group.items()} + + # Build each module + for group_name, module in all_modules.items(): + build_targets = f"@{module.name}{module.metadata.code_root_path}" log_file = log_dir / f"{group_name}-{config}.log" - exit_code, duration = build_group(group_name, module_config.build_targets, config, log_file) + exit_code, duration = build_group(group_name, build_targets, config, log_file) if exit_code != 0: any_failed = True diff --git a/scripts/known_good/BUILD b/scripts/known_good/BUILD index cfdff652c00..92ffc01a303 100644 --- a/scripts/known_good/BUILD +++ b/scripts/known_good/BUILD @@ -22,6 +22,9 @@ py_library( ["**/*.py"], exclude = ["tests/**"], ), + # scripts/ on sys.path, so "from known_good...." resolves under 'bazel run' as it does for + # "python3 scripts/...". Propagates to every consumer of this library. + imports = [".."], visibility = ["//visibility:public"], ) @@ -61,3 +64,22 @@ py_binary( visibility = ["//visibility:public"], deps = [":known_good"], ) + +# The Stage-2 matrix, derived from known_good.json rather than hardcoded in the workflow. +py_binary( + name = "list_modules", + srcs = ["list_modules.py"], + data = ["//:known_good.json"], + main = "list_modules.py", + visibility = ["//visibility:public"], + deps = [":known_good"], +) + +# The Stage-2 resolution gate: did the module really build against ref_int's pins and patches. +py_binary( + name = "verify_stage2_resolution", + srcs = ["verify_stage2_resolution.py"], + main = "verify_stage2_resolution.py", + visibility = ["//visibility:public"], + deps = [":known_good"], +) diff --git a/scripts/known_good/list_modules.py b/scripts/known_good/list_modules.py new file mode 100644 index 00000000000..bbb319346bb --- /dev/null +++ b/scripts/known_good/list_modules.py @@ -0,0 +1,88 @@ +#!/usr/bin/env python3 +# ******************************************************************************* +# Copyright (c) 2026 Contributors to the Eclipse Foundation +# +# See the NOTICE file(s) distributed with this work for additional +# information regarding copyright ownership. +# +# This program and the accompanying materials are made available under the +# terms of the Apache License Version 2.0 which is available at +# https://www.apache.org/licenses/LICENSE-2.0 +# +# SPDX-License-Identifier: Apache-2.0 +# ******************************************************************************* +"""Emit the modules of a known_good.json group as a JSON array. + +Builds the Stage-2 module matrix in test_and_docs.yml so it is always sourced from +known_good.json and never hardcoded. Each entry is + {"name": , "repo": , "slug": , + "commit": , "branch": } +where "slug" is what actions/checkout expects as `repository:`, derived from the git URL because +the repo name often differs from the bazel module name (score_lifecycle_health -> +eclipse-score/lifecycle). + +Usage: + python scripts/known_good/list_modules.py --group target_sw +""" + +from __future__ import annotations + +import argparse +import json +import re +import sys +from pathlib import Path + + +def repo_slug(repo_url: str) -> str: + """Derive the 'owner/name' slug actions/checkout expects from a git URL.""" + match = re.search(r"[:/]([^/:]+/[^/:]+?)(?:\.git)?/?$", repo_url or "") + return match.group(1) if match else "" + + +_HERE = Path(__file__).resolve().parent +try: + from known_good.models.known_good import load_known_good + from known_good.resolved_dependencies import repo_root, workspace_path +except ImportError: + if str(_HERE) not in sys.path: + sys.path.insert(0, str(_HERE)) + from models.known_good import load_known_good # noqa: E402 + from resolved_dependencies import repo_root, workspace_path # noqa: E402 + + +def main() -> None: + parser = argparse.ArgumentParser(description="List known_good.json modules of a group as JSON (for CI matrices).") + parser.add_argument( + "--known-good-path", + type=Path, + default=repo_root() / "known_good.json", + help="Path to known_good.json (default: repo-root known_good.json).", + ) + parser.add_argument("--group", default="target_sw", help="Module group to list (default: target_sw).") + args = parser.parse_args() + + kg = load_known_good(workspace_path(args.known_good_path).resolve()) + if args.group not in kg.modules: + raise SystemExit(f"Group '{args.group}' not found in {args.known_good_path}. Groups: {sorted(kg.modules)}") + + modules = [kg.modules[args.group][name] for name in sorted(kg.modules[args.group])] + + print( + json.dumps( + [ + { + "name": m.name, + "repo": m.repo, + "slug": repo_slug(m.repo), + "commit": m.hash, + "branch": m.branch, + } + for m in modules + ] + ) + ) + + +if __name__ == "__main__": + main() diff --git a/scripts/known_good/models/module.py b/scripts/known_good/models/module.py index 72cae75c678..fac3ba0d19c 100644 --- a/scripts/known_good/models/module.py +++ b/scripts/known_good/models/module.py @@ -28,14 +28,24 @@ class Metadata: code_root_path: Root path to the code directory extra_test_config: List of extra test configuration flags exclude_test_targets: List of test targets to exclude + exclude_test_target_reasons: Why each excluded target is excluded, keyed by + the label as it appears in exclude_test_targets. Every exclusion needs + one: without it nobody can tell a scope-independent exclusion (a + benchmark, a sanitizer target) from a stale workaround for a build + scope that no longer exists. + legacy_exclude_test_targets: central-mode-only exclusions. Predate the audit, so + exempt from the wildcard and reason checks; deleted with that runner. langs: List of languages supported (e.g., ["cpp", "rust"]) """ code_root_path: str = "//score/..." extra_test_config: list[str] = field(default_factory=lambda: []) exclude_test_targets: list[str] = field(default_factory=lambda: []) + exclude_test_target_reasons: dict[str, str] = field(default_factory=lambda: {}) + legacy_exclude_test_targets: list[str] = field(default_factory=lambda: []) langs: list[str] = field(default_factory=lambda: ["cpp", "rust"]) - rust_coverage_config: str | None = "ferrocene-coverage" # Optional field for Rust coverage configuration + rust_coverage_config: str | None = "ferrocene-coverage" + bazel_config: list[str] = field(default_factory=lambda: []) @classmethod def from_dict(cls, data: Dict[str, Any]) -> Metadata: @@ -51,8 +61,11 @@ def from_dict(cls, data: Dict[str, Any]) -> Metadata: code_root_path=data.get("code_root_path", "//score/..."), extra_test_config=data.get("extra_test_config", []), exclude_test_targets=data.get("exclude_test_targets", []), + exclude_test_target_reasons=data.get("exclude_test_target_reasons", {}), + legacy_exclude_test_targets=data.get("legacy_exclude_test_targets", []), langs=data.get("langs", ["cpp", "rust"]), rust_coverage_config=data.get("rust_coverage_config", "ferrocene-coverage"), + bazel_config=data.get("bazel_config", []), ) def to_dict(self) -> Dict[str, Any]: @@ -65,8 +78,11 @@ def to_dict(self) -> Dict[str, Any]: "code_root_path": self.code_root_path, "extra_test_config": self.extra_test_config, "exclude_test_targets": self.exclude_test_targets, + "exclude_test_target_reasons": self.exclude_test_target_reasons, + "legacy_exclude_test_targets": self.legacy_exclude_test_targets, "langs": self.langs, "rust_coverage_config": self.rust_coverage_config, + "bazel_config": self.bazel_config, } @@ -97,6 +113,7 @@ def from_dict(cls, name: str, module_data: Dict[str, Any]) -> Module: "code_root_path": "path/to/code/root", "extra_test_config": [""], "exclude_test_targets": [""], + "exclude_test_target_reasons": {"": ""}, "langs": ["cpp", "rust"] } If not present, uses default Metadata values. @@ -124,12 +141,27 @@ def from_dict(cls, name: str, module_data: Dict[str, Any]) -> Module: metadata_data = module_data.get("metadata") if metadata_data is not None: metadata = Metadata.from_dict(metadata_data) - # Enable once we are able to remove '*' in known_good.json - # if any("*" in target for target in metadata.exclude_test_targets): - # raise Exception( - # f"Module {name} has wildcard '*' in exclude_test_targets, which is not allowed. " - # "Please specify explicit test targets to exclude or remove the key if no exclusions are needed." - # ) + # A wildcard hides how much it excludes: '//score/json/examples:*' silently grows with + # every target added to that package, so the report cannot say what was skipped. The + # last two were dropped by the Stage-2 exclusion audit, so this can enforce now. + wildcards = [target for target in metadata.exclude_test_targets if "*" in target] + if wildcards: + raise ValueError( + f"Module '{name}' has wildcard exclude_test_targets: {wildcards}. " + "List explicit test targets instead, so the excluded set cannot grow unnoticed." + ) + # Stage 2 runs each module as the Bazel root, which retired the blanket + # "invisible dev_dependency" justification. Every exclusion states its own reason. + unexplained = [ + target + for target in metadata.exclude_test_targets + if not metadata.exclude_test_target_reasons.get(target, "").strip() + ] + if unexplained: + raise ValueError( + f"Module '{name}' excludes test targets with no recorded reason: {unexplained}. " + "Add metadata.exclude_test_target_reasons[