refactor(python): render packages with Jinja and stage runtimes in Actions

.github/RELEASING.md

@@ -32,15 +32,28 @@ validates archive contents, platform constraints and versions before publishing.
 Node inference tests run against the checkout and its platform workspace in CI.
 No Python wheel extraction is involved. All native packages publish before the wrapper.
 
-Python publishes canonical wrapper-only source archives including `stage_runtime.py`;
-they contain neither native sources nor compiled libraries. CPU wheel jobs extract
-the canonical base archive, download the matching runtime and stage it with
-`python stage_runtime.py PROJECT_DIR RUNTIME_DIR`. cibuildwheel packages without
-CMake (`wheel.cmake=false`) and runs installed-wheel tests for Python 3.9, 3.10,
+Python publishes canonical wrapper-only source archives with rendered direct
+scikit-build-core metadata; neither sdists nor wheels contain tests or `.j2`
+templates. Source archives contain neither native sources nor compiled libraries.
+Release jobs render the repository's shared `pyproject.toml.j2` template with
+Jinja2 for CPU, CUDA, ROCm and Vulkan from the exact `${{ github.sha }}` checkout.
+GPU projects reuse wrapper bytes and the version from the canonical CPU sdist;
+the template is read from the checkout, not from that archive.
+CPU wheel jobs extract the base archive and directly copy the downloaded
+`clap_cpp/` runtime contents and license into its package. scikit-build-core never compiles
+native code (`wheel.cmake=false`, `sdist.cmake=false`); no custom hook/backend
+is used. Actions checks runtime binary headers and nonempty runtime notices before
+assembly and checks the archived payload again before publication. The direct
+backend alone does not reject unstaged projects; never publish their unusable
+runtime-free wheels. The wheel job checks out `${{ github.sha }}` and copies tests
+and the template separately into its working source directory for packaging tests
+(not into published sdists/wheels).
+cibuildwheel runs installed-wheel tests for Python 3.9, 3.10,
 3.11, 3.12, 3.13 and 3.14. ABI-independent `py3-none` wheels are reused across
 these interpreters; Linux wheels retain auditwheel-repaired manylinux tags.
 GPU jobs recursively check out native sources and compile before packaging the
-canonical backend wrapper archive, staging with `--backend cuda|rocm|vulkan`.
+canonical backend wrapper archive, copying the external `clap_cpp/` runtime
+contents into `clap_cpp_cuda/`, `clap_cpp_rocm/` or `clap_cpp_vulkan/`.
 CUDA toolkit/architecture flags, ROCm targets and system-runtime exclusions, and
 Vulkan loader exclusions remain backend-specific. Their existing auditwheel
 policies remain: CUDA/ROCm manylinux_2_35, Vulkan manylinux_2_39; GPU execution is
@@ -91,7 +104,8 @@ cmake --install build/libraries --config Release --component runtime --prefix "$
 python -m build python --sdist --outdir sdists
 mkdir python-dist
 tar -xzf sdists/clap_cpp-*.tar.gz --strip-components=1 -C python-dist
-python python-dist/stage_runtime.py python-dist libraries/linux-x64
+cp -R libraries/linux-x64/clap_cpp/. python-dist/clap_cpp/
+cp python-dist/LICENSE python-dist/clap_cpp/licenses/clap-cpp.txt
 python -m build python-dist --wheel --outdir wheels
 # Host builds above are not a substitute for the release's manylinux Docker build/repair.
 mkdir -p node/packages/linux-x64/runtime

.github/scripts/test_python_release_assets.py

@@ -38,7 +38,15 @@ def unit_wheel(root, distribution="clap_cpp", platform=PLATFORMS[0], interpreter
             archive.writestr(f"{distribution}/clap.cpython-311-x86_64-linux-gnu.so", b"UNIT FIXTURE, NOT NATIVE CODE")
         else:
             directory = "wrong_package" if payload == "wrong-package" else distribution
-            archive.writestr(f"{directory}/{native}", b"UNIT FIXTURE, NOT NATIVE CODE")
+            archive.writestr(f"{directory}/{native}", (b"" if payload == "empty" else b"invalid" if payload == "invalid" else
+                             (b"MZ" if platform == "win_amd64" else b"\xcf\xfa\xed\xfe" if platform.startswith("macosx_") else b"\x7fELF")
+                             + b"UNIT FIXTURE, NOT NATIVE CODE"))
+        if payload != "missing-notice":
+            archive.writestr(f"{distribution}/licenses/clap-cpp.txt", "" if payload == "empty-notice" else "UNIT FIXTURE notice")
+        if payload == "tests":
+            archive.writestr(f"{distribution}/tests/test_leaked.py", "# TEST FIXTURE")
+        if payload == "template":
+            archive.writestr(f"{distribution}/unused.j2", "UNIT FIXTURE template")
     return path
 
 
@@ -59,26 +67,56 @@ def setUp(self):
     def unit_sdist(self, source, distribution, omitted=None, extra=()):
         """Write wrapper-only UNIT FIXTURE source archives for the real heredoc."""
         with tarfile.open(source, "w:gz") as archive:
-            for name in ("pyproject.toml", "stage_runtime.py", f"{distribution}/clap.py", *extra):
+            for name in ("pyproject.toml", "README.md", "LICENSE",
+                         *(f"{distribution}/{name}" for name in ("__init__.py", "clap.py", "clap.pyi", "py.typed")), *extra):
                 if name == omitted:
                     continue
                 data = b"# UNIT FIXTURE wrapper packaging input\n"
                 entry = tarfile.TarInfo(f"{distribution}-0.1.0/{name}")
                 entry.size = len(data)
                 archive.addfile(entry, io.BytesIO(data))
 
-    def test_unit_fixture_sdist_requires_wrapper_metadata_and_helper(self):
-        for required in ("pyproject.toml", "stage_runtime.py", "clap_cpp/clap.py"):
-            with self.subTest(required=required):
-                self.unit_sdist(self.artifacts / "clap_cpp-0.1.0.tar.gz", "clap_cpp", omitted=required)
-                self.assert_rejected("Sdist missing wrapper or staging helper")
+    def test_unit_fixture_sdist_requires_wrapper_and_rendered_metadata(self):
+        for distribution in DISTRIBUTIONS:
+            for required in ("pyproject.toml", "README.md", "LICENSE",
+                             *(f"{distribution}/{name}" for name in ("__init__.py", "clap.py", "clap.pyi", "py.typed"))):
+                with self.subTest(distribution=distribution, required=required):
+                    source = self.artifacts / f"{distribution}-0.1.0.tar.gz"
+                    self.unit_sdist(source, distribution, omitted=required)
+                    self.assert_rejected("Sdist missing wrapper or packaging input")
+                    self.unit_sdist(source, distribution)
 
     def test_unit_fixture_sdist_rejects_native_graph_and_binaries(self):
-        for native in ("native/ggml/CMakeLists.txt", "clap_cpp/libclap.so", "clap_cpp/clap.dll", "clap_cpp/libclap.dylib"):
+        for native in ("native/ggml/CMakeLists.txt", "CMakeLists.txt", "cmake/helpers.cmake", "src/clap.cpp",
+                       "clap_cpp/libclap.so", "clap_cpp/clap.dll", "clap_cpp/libclap.dylib"):
             with self.subTest(native=native):
                 self.unit_sdist(self.artifacts / "clap_cpp-0.1.0.tar.gz", "clap_cpp", extra=(native,))
                 self.assert_rejected("Sdist contains native sources or compiled runtime")
 
+    def test_published_archives_reject_tests(self):
+        for distribution in DISTRIBUTIONS:
+            with self.subTest(distribution=distribution):
+                source = self.artifacts / f"{distribution}-0.1.0.tar.gz"
+                self.unit_sdist(source, distribution, extra=("tests/test_leaked.py",))
+                self.assert_rejected("Sdist contains tests")
+                self.unit_sdist(source, distribution)
+                unit_wheel(self.artifacts, distribution=distribution, payload="tests")
+                self.assert_rejected("Wheel contains tests")
+                unit_wheel(self.artifacts, distribution=distribution)
+
+    def test_published_archives_reject_jinja_templates(self):
+        for distribution in DISTRIBUTIONS:
+            for template in ("pyproject.toml.j2", f"{distribution}/unused.j2"):
+                with self.subTest(distribution=distribution, template=template):
+                    source = self.artifacts / f"{distribution}-0.1.0.tar.gz"
+                    self.unit_sdist(source, distribution, extra=(template,))
+                    self.assert_rejected("Sdist contains Jinja template")
+                    self.unit_sdist(source, distribution)
+            with self.subTest(distribution=distribution, archive="wheel"):
+                unit_wheel(self.artifacts, distribution=distribution, payload="template")
+                self.assert_rejected("Wheel contains Jinja template")
+                unit_wheel(self.artifacts, distribution=distribution)
+
     def run_validator(self):
         shutil.rmtree(self.root / "dist", ignore_errors=True)
         return subprocess.run([sys.executable, "-c", validator_source()], cwd=self.root, capture_output=True, text=True)
@@ -113,6 +151,18 @@ def test_unit_fixture_old_extension_rejected(self):
         self.replace_cpu(payload="extension")
         self.assert_rejected("package-local native library")
 
+    def test_unit_fixture_empty_or_invalid_native_rejected(self):
+        for payload in ("empty", "invalid"):
+            with self.subTest(payload=payload):
+                self.replace_cpu(payload=payload)
+                self.assert_rejected("Invalid package-local native library")
+
+    def test_unit_fixture_runtime_notice_required(self):
+        for payload in ("missing-notice", "empty-notice"):
+            with self.subTest(payload=payload):
+                self.replace_cpu(payload=payload)
+                self.assert_rejected("Missing runtime notices")
+
     def test_unit_fixture_missing_wrapper_rejected(self):
         self.replace_cpu(payload="missing-wrapper")
         self.assert_rejected("ctypes wrapper")
@@ -146,6 +196,46 @@ def test_unit_fixture_six_rocm_manifests_rejected(self):
             (directory / MANIFEST).write_text("UNIT-FIXTURE-SONAME.so\n")
         self.assert_rejected("ROCm exclusion manifest")
 
+    def test_actions_runtime_gates_reject_invalid_staging(self):
+        # Execute all five real Actions heredocs, not a test copy of the checks.
+        ci = WORKFLOW.with_name("ci.yml")
+        cases = [(ci, "Package Python with the shared runtime", "build/python-package/clap_cpp"),
+                 (WORKFLOW, "Copy external runtime into the CPU wrapper project", "python-dist/clap_cpp")]
+        cases += [(WORKFLOW, f"Stage runtime and package canonical {backend.upper()} wrapper sdist",
+                   f"runner/clap-{backend}/clap_cpp_{backend}") for backend in ("cuda", "vulkan", "rocm")]
+        native = "clap.dll" if sys.platform == "win32" else "libclap.dylib" if sys.platform == "darwin" else "libclap.so"
+        magic = b"MZ" if sys.platform == "win32" else b"\xcf\xfa\xed\xfe" if sys.platform == "darwin" else b"\x7fELF"
+        for workflow, step_name, path in cases:
+            step = workflow.read_text().split(f"- name: {step_name}\n", 1)[1]
+            block = step.split("python - <<'PY'\n", 1)[1]
+            source = textwrap.dedent(block.split("\n" + " " * (10 if workflow == ci else 8) + "PY\n", 1)[0])
+            package = self.root / path
+            package.mkdir(parents=True, exist_ok=True)
+            runtime = package / native
+            notice = package / "licenses/clap-cpp.txt"
+            notice.parent.mkdir(exist_ok=True)
+            for payload, license_text, error in ((None, "notice", "Missing staged external runtime"),
+                                                (b"", "notice", "Missing staged external runtime"),
+                                                (b"invalid", "notice", "Invalid staged external runtime"),
+                                                (magic, None, "Missing runtime notices"),
+                                                (magic, "  ", "Missing runtime notices"),
+                                                (magic + b"UNIT FIXTURE, NOT NATIVE CODE", "notice", None)):
+                with self.subTest(step=step_name, payload=payload, license_text=license_text):
+                    runtime.unlink(missing_ok=True)
+                    notice.unlink(missing_ok=True)
+                    if payload is not None:
+                        runtime.write_bytes(payload)
+                    if license_text is not None:
+                        notice.write_text(license_text)
+                    result = subprocess.run([sys.executable, "-c", source], cwd=self.root,
+                                            env=dict(os.environ, RUNNER_TEMP=str(self.root / "runner")),
+                                            capture_output=True, text=True)
+                    if error:
+                        self.assertNotEqual(result.returncode, 0)
+                        self.assertIn(error, result.stderr)
+                    else:
+                        self.assertEqual(result.returncode, 0, result.stderr)
+
     @unittest.skipUnless(os.environ.get("CLAP_TEST_CPU_WHEEL"), "set CLAP_TEST_CPU_WHEEL for real built wheel inspection")
     def test_actual_built_cpu_payload_and_unrepaired_release_rejection(self):
         from packaging.utils import parse_wheel_filename

.github/workflows/ci.yml

@@ -78,6 +78,11 @@ jobs:
           }
           7z x $archive "-o$destination" -y
           (Get-ChildItem $destination -Filter ffmpeg.exe -Recurse).DirectoryName | Out-File -FilePath $env:GITHUB_PATH -Encoding utf8 -Append
+      - name: Set macOS native deployment minimum
+        if: matrix.os == 'macos-latest'
+        shell: bash
+        run: |
+          printf '%s\n' 'MACOSX_DEPLOYMENT_TARGET=11.0' >> "$GITHUB_ENV"
       - name: build-libraries
         uses: ./.github/actions/build-libraries
         with:
@@ -92,15 +97,34 @@ jobs:
         run: cmake --install build/libraries --config Release --prefix "${{ runner.temp }}/clap-install"
       - name: Install Python packaging and test dependencies
         if: matrix.os != 'windows-11-arm'
-        run: python -m pip install build "scikit-build-core>=1.1.2" packaging pytest "mypy>=1.8" "typing_extensions>=4.2"
+        run: python -m pip install build "scikit-build-core>=1.1.2" jinja2 packaging pytest "mypy>=1.8" "typing_extensions>=4.2"
       - name: Package Python with the shared runtime
         if: matrix.os != 'windows-11-arm'
         shell: bash
         run: |
+          set -euo pipefail
           python -m build python --sdist --outdir python-dist
           mkdir -p build/python-package
           tar -xzf python-dist/*.tar.gz --strip-components=1 -C build/python-package
-          python python/stage_runtime.py build/python-package "libraries/${{ matrix.runtime }}"
+          cp -R "libraries/${{ matrix.runtime }}/clap_cpp/." build/python-package/clap_cpp/
+          cp build/python-package/LICENSE build/python-package/clap_cpp/licenses/clap-cpp.txt
+          python - <<'PY'
+          from pathlib import Path
+          import sys
+          package = Path("build/python-package/clap_cpp")
+          native = "clap.dll" if sys.platform == "win32" else ("libclap.dylib" if sys.platform == "darwin" else "libclap.so")
+          runtime = package / native
+          if not runtime.is_file() or not runtime.stat().st_size:
+              raise SystemExit(f"Missing staged external runtime: {runtime}")
+          magic = runtime.read_bytes()[:4]
+          if not (magic == b"\x7fELF" if sys.platform.startswith("linux") else
+                  magic[:2] == b"MZ" if sys.platform == "win32" else
+                  magic in (b"\xcf\xfa\xed\xfe", b"\xfe\xed\xfa\xcf")):
+              raise SystemExit(f"Invalid staged external runtime: {runtime}")
+          notice = package / "licenses" / "clap-cpp.txt"
+          if not notice.is_file() or not notice.read_text(encoding="utf-8").strip():
+              raise SystemExit(f"Missing runtime notices: {notice}")
+          PY
           python -m build build/python-package --wheel --outdir python-dist
           python -m pip install --no-deps --no-index --find-links python-dist clap-cpp
       - name: Test Python