diff --git a/.github/RELEASING.md b/.github/RELEASING.md index 3452ff4..e8516ad 100644 --- a/.github/RELEASING.md +++ b/.github/RELEASING.md @@ -32,15 +32,28 @@ validates archive contents, platform constraints and versions before publishing. Node inference tests run against the checkout and its platform workspace in CI. No Python wheel extraction is involved. All native packages publish before the wrapper. -Python publishes canonical wrapper-only source archives including `stage_runtime.py`; -they contain neither native sources nor compiled libraries. CPU wheel jobs extract -the canonical base archive, download the matching runtime and stage it with -`python stage_runtime.py PROJECT_DIR RUNTIME_DIR`. cibuildwheel packages without -CMake (`wheel.cmake=false`) and runs installed-wheel tests for Python 3.9, 3.10, +Python publishes canonical wrapper-only source archives with rendered direct +scikit-build-core metadata; neither sdists nor wheels contain tests or `.j2` +templates. Source archives contain neither native sources nor compiled libraries. +Release jobs render the repository's shared `pyproject.toml.j2` template with +Jinja2 for CPU, CUDA, ROCm and Vulkan from the exact `${{ github.sha }}` checkout. +GPU projects reuse wrapper bytes and the version from the canonical CPU sdist; +the template is read from the checkout, not from that archive. +CPU wheel jobs extract the base archive and directly copy the downloaded +`clap_cpp/` runtime contents and license into its package. scikit-build-core never compiles +native code (`wheel.cmake=false`, `sdist.cmake=false`); no custom hook/backend +is used. Actions checks runtime binary headers and nonempty runtime notices before +assembly and checks the archived payload again before publication. The direct +backend alone does not reject unstaged projects; never publish their unusable +runtime-free wheels. The wheel job checks out `${{ github.sha }}` and copies tests +and the template separately into its working source directory for packaging tests +(not into published sdists/wheels). +cibuildwheel runs installed-wheel tests for Python 3.9, 3.10, 3.11, 3.12, 3.13 and 3.14. ABI-independent `py3-none` wheels are reused across these interpreters; Linux wheels retain auditwheel-repaired manylinux tags. GPU jobs recursively check out native sources and compile before packaging the -canonical backend wrapper archive, staging with `--backend cuda|rocm|vulkan`. +canonical backend wrapper archive, copying the external `clap_cpp/` runtime +contents into `clap_cpp_cuda/`, `clap_cpp_rocm/` or `clap_cpp_vulkan/`. CUDA toolkit/architecture flags, ROCm targets and system-runtime exclusions, and Vulkan loader exclusions remain backend-specific. Their existing auditwheel policies remain: CUDA/ROCm manylinux_2_35, Vulkan manylinux_2_39; GPU execution is @@ -91,7 +104,8 @@ cmake --install build/libraries --config Release --component runtime --prefix "$ python -m build python --sdist --outdir sdists mkdir python-dist tar -xzf sdists/clap_cpp-*.tar.gz --strip-components=1 -C python-dist -python python-dist/stage_runtime.py python-dist libraries/linux-x64 +cp -R libraries/linux-x64/clap_cpp/. python-dist/clap_cpp/ +cp python-dist/LICENSE python-dist/clap_cpp/licenses/clap-cpp.txt python -m build python-dist --wheel --outdir wheels # Host builds above are not a substitute for the release's manylinux Docker build/repair. mkdir -p node/packages/linux-x64/runtime diff --git a/.github/scripts/test_python_release_assets.py b/.github/scripts/test_python_release_assets.py index 1f8512f..d69b9cf 100644 --- a/.github/scripts/test_python_release_assets.py +++ b/.github/scripts/test_python_release_assets.py @@ -38,7 +38,15 @@ def unit_wheel(root, distribution="clap_cpp", platform=PLATFORMS[0], interpreter archive.writestr(f"{distribution}/clap.cpython-311-x86_64-linux-gnu.so", b"UNIT FIXTURE, NOT NATIVE CODE") else: directory = "wrong_package" if payload == "wrong-package" else distribution - archive.writestr(f"{directory}/{native}", b"UNIT FIXTURE, NOT NATIVE CODE") + archive.writestr(f"{directory}/{native}", (b"" if payload == "empty" else b"invalid" if payload == "invalid" else + (b"MZ" if platform == "win_amd64" else b"\xcf\xfa\xed\xfe" if platform.startswith("macosx_") else b"\x7fELF") + + b"UNIT FIXTURE, NOT NATIVE CODE")) + if payload != "missing-notice": + archive.writestr(f"{distribution}/licenses/clap-cpp.txt", "" if payload == "empty-notice" else "UNIT FIXTURE notice") + if payload == "tests": + archive.writestr(f"{distribution}/tests/test_leaked.py", "# TEST FIXTURE") + if payload == "template": + archive.writestr(f"{distribution}/unused.j2", "UNIT FIXTURE template") return path @@ -59,7 +67,8 @@ def setUp(self): def unit_sdist(self, source, distribution, omitted=None, extra=()): """Write wrapper-only UNIT FIXTURE source archives for the real heredoc.""" with tarfile.open(source, "w:gz") as archive: - for name in ("pyproject.toml", "stage_runtime.py", f"{distribution}/clap.py", *extra): + for name in ("pyproject.toml", "README.md", "LICENSE", + *(f"{distribution}/{name}" for name in ("__init__.py", "clap.py", "clap.pyi", "py.typed")), *extra): if name == omitted: continue data = b"# UNIT FIXTURE wrapper packaging input\n" @@ -67,18 +76,47 @@ def unit_sdist(self, source, distribution, omitted=None, extra=()): entry.size = len(data) archive.addfile(entry, io.BytesIO(data)) - def test_unit_fixture_sdist_requires_wrapper_metadata_and_helper(self): - for required in ("pyproject.toml", "stage_runtime.py", "clap_cpp/clap.py"): - with self.subTest(required=required): - self.unit_sdist(self.artifacts / "clap_cpp-0.1.0.tar.gz", "clap_cpp", omitted=required) - self.assert_rejected("Sdist missing wrapper or staging helper") + def test_unit_fixture_sdist_requires_wrapper_and_rendered_metadata(self): + for distribution in DISTRIBUTIONS: + for required in ("pyproject.toml", "README.md", "LICENSE", + *(f"{distribution}/{name}" for name in ("__init__.py", "clap.py", "clap.pyi", "py.typed"))): + with self.subTest(distribution=distribution, required=required): + source = self.artifacts / f"{distribution}-0.1.0.tar.gz" + self.unit_sdist(source, distribution, omitted=required) + self.assert_rejected("Sdist missing wrapper or packaging input") + self.unit_sdist(source, distribution) def test_unit_fixture_sdist_rejects_native_graph_and_binaries(self): - for native in ("native/ggml/CMakeLists.txt", "clap_cpp/libclap.so", "clap_cpp/clap.dll", "clap_cpp/libclap.dylib"): + for native in ("native/ggml/CMakeLists.txt", "CMakeLists.txt", "cmake/helpers.cmake", "src/clap.cpp", + "clap_cpp/libclap.so", "clap_cpp/clap.dll", "clap_cpp/libclap.dylib"): with self.subTest(native=native): self.unit_sdist(self.artifacts / "clap_cpp-0.1.0.tar.gz", "clap_cpp", extra=(native,)) self.assert_rejected("Sdist contains native sources or compiled runtime") + def test_published_archives_reject_tests(self): + for distribution in DISTRIBUTIONS: + with self.subTest(distribution=distribution): + source = self.artifacts / f"{distribution}-0.1.0.tar.gz" + self.unit_sdist(source, distribution, extra=("tests/test_leaked.py",)) + self.assert_rejected("Sdist contains tests") + self.unit_sdist(source, distribution) + unit_wheel(self.artifacts, distribution=distribution, payload="tests") + self.assert_rejected("Wheel contains tests") + unit_wheel(self.artifacts, distribution=distribution) + + def test_published_archives_reject_jinja_templates(self): + for distribution in DISTRIBUTIONS: + for template in ("pyproject.toml.j2", f"{distribution}/unused.j2"): + with self.subTest(distribution=distribution, template=template): + source = self.artifacts / f"{distribution}-0.1.0.tar.gz" + self.unit_sdist(source, distribution, extra=(template,)) + self.assert_rejected("Sdist contains Jinja template") + self.unit_sdist(source, distribution) + with self.subTest(distribution=distribution, archive="wheel"): + unit_wheel(self.artifacts, distribution=distribution, payload="template") + self.assert_rejected("Wheel contains Jinja template") + unit_wheel(self.artifacts, distribution=distribution) + def run_validator(self): shutil.rmtree(self.root / "dist", ignore_errors=True) return subprocess.run([sys.executable, "-c", validator_source()], cwd=self.root, capture_output=True, text=True) @@ -113,6 +151,18 @@ def test_unit_fixture_old_extension_rejected(self): self.replace_cpu(payload="extension") self.assert_rejected("package-local native library") + def test_unit_fixture_empty_or_invalid_native_rejected(self): + for payload in ("empty", "invalid"): + with self.subTest(payload=payload): + self.replace_cpu(payload=payload) + self.assert_rejected("Invalid package-local native library") + + def test_unit_fixture_runtime_notice_required(self): + for payload in ("missing-notice", "empty-notice"): + with self.subTest(payload=payload): + self.replace_cpu(payload=payload) + self.assert_rejected("Missing runtime notices") + def test_unit_fixture_missing_wrapper_rejected(self): self.replace_cpu(payload="missing-wrapper") self.assert_rejected("ctypes wrapper") @@ -146,6 +196,46 @@ def test_unit_fixture_six_rocm_manifests_rejected(self): (directory / MANIFEST).write_text("UNIT-FIXTURE-SONAME.so\n") self.assert_rejected("ROCm exclusion manifest") + def test_actions_runtime_gates_reject_invalid_staging(self): + # Execute all five real Actions heredocs, not a test copy of the checks. + ci = WORKFLOW.with_name("ci.yml") + cases = [(ci, "Package Python with the shared runtime", "build/python-package/clap_cpp"), + (WORKFLOW, "Copy external runtime into the CPU wrapper project", "python-dist/clap_cpp")] + cases += [(WORKFLOW, f"Stage runtime and package canonical {backend.upper()} wrapper sdist", + f"runner/clap-{backend}/clap_cpp_{backend}") for backend in ("cuda", "vulkan", "rocm")] + native = "clap.dll" if sys.platform == "win32" else "libclap.dylib" if sys.platform == "darwin" else "libclap.so" + magic = b"MZ" if sys.platform == "win32" else b"\xcf\xfa\xed\xfe" if sys.platform == "darwin" else b"\x7fELF" + for workflow, step_name, path in cases: + step = workflow.read_text().split(f"- name: {step_name}\n", 1)[1] + block = step.split("python - <<'PY'\n", 1)[1] + source = textwrap.dedent(block.split("\n" + " " * (10 if workflow == ci else 8) + "PY\n", 1)[0]) + package = self.root / path + package.mkdir(parents=True, exist_ok=True) + runtime = package / native + notice = package / "licenses/clap-cpp.txt" + notice.parent.mkdir(exist_ok=True) + for payload, license_text, error in ((None, "notice", "Missing staged external runtime"), + (b"", "notice", "Missing staged external runtime"), + (b"invalid", "notice", "Invalid staged external runtime"), + (magic, None, "Missing runtime notices"), + (magic, " ", "Missing runtime notices"), + (magic + b"UNIT FIXTURE, NOT NATIVE CODE", "notice", None)): + with self.subTest(step=step_name, payload=payload, license_text=license_text): + runtime.unlink(missing_ok=True) + notice.unlink(missing_ok=True) + if payload is not None: + runtime.write_bytes(payload) + if license_text is not None: + notice.write_text(license_text) + result = subprocess.run([sys.executable, "-c", source], cwd=self.root, + env=dict(os.environ, RUNNER_TEMP=str(self.root / "runner")), + capture_output=True, text=True) + if error: + self.assertNotEqual(result.returncode, 0) + self.assertIn(error, result.stderr) + else: + self.assertEqual(result.returncode, 0, result.stderr) + @unittest.skipUnless(os.environ.get("CLAP_TEST_CPU_WHEEL"), "set CLAP_TEST_CPU_WHEEL for real built wheel inspection") def test_actual_built_cpu_payload_and_unrepaired_release_rejection(self): from packaging.utils import parse_wheel_filename diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index f7bf9af..1e872fd 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -78,6 +78,11 @@ jobs: } 7z x $archive "-o$destination" -y (Get-ChildItem $destination -Filter ffmpeg.exe -Recurse).DirectoryName | Out-File -FilePath $env:GITHUB_PATH -Encoding utf8 -Append + - name: Set macOS native deployment minimum + if: matrix.os == 'macos-latest' + shell: bash + run: | + printf '%s\n' 'MACOSX_DEPLOYMENT_TARGET=11.0' >> "$GITHUB_ENV" - name: build-libraries uses: ./.github/actions/build-libraries with: @@ -92,15 +97,34 @@ jobs: run: cmake --install build/libraries --config Release --prefix "${{ runner.temp }}/clap-install" - name: Install Python packaging and test dependencies if: matrix.os != 'windows-11-arm' - run: python -m pip install build "scikit-build-core>=1.1.2" packaging pytest "mypy>=1.8" "typing_extensions>=4.2" + run: python -m pip install build "scikit-build-core>=1.1.2" jinja2 packaging pytest "mypy>=1.8" "typing_extensions>=4.2" - name: Package Python with the shared runtime if: matrix.os != 'windows-11-arm' shell: bash run: | + set -euo pipefail python -m build python --sdist --outdir python-dist mkdir -p build/python-package tar -xzf python-dist/*.tar.gz --strip-components=1 -C build/python-package - python python/stage_runtime.py build/python-package "libraries/${{ matrix.runtime }}" + cp -R "libraries/${{ matrix.runtime }}/clap_cpp/." build/python-package/clap_cpp/ + cp build/python-package/LICENSE build/python-package/clap_cpp/licenses/clap-cpp.txt + python - <<'PY' + from pathlib import Path + import sys + package = Path("build/python-package/clap_cpp") + native = "clap.dll" if sys.platform == "win32" else ("libclap.dylib" if sys.platform == "darwin" else "libclap.so") + runtime = package / native + if not runtime.is_file() or not runtime.stat().st_size: + raise SystemExit(f"Missing staged external runtime: {runtime}") + magic = runtime.read_bytes()[:4] + if not (magic == b"\x7fELF" if sys.platform.startswith("linux") else + magic[:2] == b"MZ" if sys.platform == "win32" else + magic in (b"\xcf\xfa\xed\xfe", b"\xfe\xed\xfa\xcf")): + raise SystemExit(f"Invalid staged external runtime: {runtime}") + notice = package / "licenses" / "clap-cpp.txt" + if not notice.is_file() or not notice.read_text(encoding="utf-8").strip(): + raise SystemExit(f"Missing runtime notices: {notice}") + PY python -m build build/python-package --wheel --outdir python-dist python -m pip install --no-deps --no-index --find-links python-dist clap-cpp - name: Test Python diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 1167b1d..6605cde 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -15,12 +15,13 @@ jobs: steps: - uses: actions/checkout@v4 with: + ref: ${{ github.sha }} submodules: recursive - uses: actions/setup-python@v5 with: python-version: '3.11' - name: Install packaging tools - run: python -m pip install build packaging twine + run: python -m pip install build jinja2 packaging twine - name: Set Python package version from release tag env: RELEASE_TAG: ${{ github.event.release.tag_name }} @@ -28,21 +29,15 @@ jobs: python - <<'PY' import os from pathlib import Path - import tomllib + from jinja2 import Environment, StrictUndefined from packaging.version import Version version = str(Version(os.environ['RELEASE_TAG'])) - path = Path('python/pyproject.toml') - text = path.read_text() - current = tomllib.loads(text)['project']['version'] - old = f'version = "{current}"' - assert text.count(old) == 1, 'Expected exactly one package version' - text = text.replace(old, f'version = "{version}"') - for backend in ('cuda', 'rocm', 'vulkan'): - old_dependency = f'clap-cpp-{backend}=={current}' - assert text.count(old_dependency) == 1, f'Expected one exact {backend} dependency' - text = text.replace(old_dependency, f'clap-cpp-{backend}=={version}') - path.write_text(text) + source = Path('python') + template = Environment(undefined=StrictUndefined, keep_trailing_newline=True).from_string( + (source / 'pyproject.toml.j2').read_text(encoding='utf-8')) + (source / 'pyproject.toml').write_text( + template.render(backend='cpu', version=version), encoding='utf-8') PY - name: Build source distribution run: python -m build python --sdist --outdir dist @@ -54,7 +49,23 @@ jobs: tar -xzf dist/clap_cpp-[0-9]*.tar.gz --strip-components=1 -C python-dist for backend in cuda rocm vulkan; do destination="$RUNNER_TEMP/clap-$backend" - python python-dist/prepare_backend.py "$backend" "$destination" + package="clap_cpp_$backend" + mkdir -p "$destination/$package" + cp python-dist/{README.md,LICENSE} "$destination/" + cp python-dist/clap_cpp/{clap.py,clap.pyi,py.typed} "$destination/$package/" + printf '"""Native %s runtime; import clap_cpp for the public API."""\n' "$backend" > "$destination/$package/__init__.py" + BACKEND="$backend" DESTINATION="$destination" python - <<'PY' + import os + from pathlib import Path + import tomllib + from jinja2 import Environment, StrictUndefined + source = Path('python-dist') + version = tomllib.loads((source / 'pyproject.toml').read_text(encoding='utf-8'))['project']['version'] + template = Environment(undefined=StrictUndefined, keep_trailing_newline=True).from_string( + Path('python/pyproject.toml.j2').read_text(encoding='utf-8')) + (Path(os.environ['DESTINATION']) / 'pyproject.toml').write_text( + template.render(backend=os.environ['BACKEND'], version=version), encoding='utf-8') + PY python -m build --sdist "$destination" --outdir dist done - name: Check source distribution @@ -134,6 +145,9 @@ jobs: target: win32-x64 runs-on: ${{ matrix.os }} steps: + - uses: actions/checkout@v4 + with: + ref: ${{ github.sha }} - uses: actions/download-artifact@v4 with: name: python-sdists @@ -147,6 +161,34 @@ jobs: with: name: libraries-${{ matrix.target }} path: python-dist/external-runtime + - name: Copy external runtime into the CPU wrapper project + shell: bash + run: | + set -euo pipefail + cp -R python-dist/external-runtime/clap_cpp/. python-dist/clap_cpp/ + cp python-dist/LICENSE python-dist/clap_cpp/licenses/clap-cpp.txt + python - <<'PY' + from pathlib import Path + import sys + package = Path("python-dist/clap_cpp") + native = "clap.dll" if sys.platform == "win32" else ("libclap.dylib" if sys.platform == "darwin" else "libclap.so") + runtime = package / native + if not runtime.is_file() or not runtime.stat().st_size: + raise SystemExit(f"Missing staged external runtime: {runtime}") + magic = runtime.read_bytes()[:4] + if not (magic == b"\x7fELF" if sys.platform.startswith("linux") else + magic[:2] == b"MZ" if sys.platform == "win32" else + magic in (b"\xcf\xfa\xed\xfe", b"\xfe\xed\xfa\xcf")): + raise SystemExit(f"Invalid staged external runtime: {runtime}") + notice = package / "licenses" / "clap-cpp.txt" + if not notice.is_file() or not notice.read_text(encoding="utf-8").strip(): + raise SystemExit(f"Missing runtime notices: {notice}") + PY + - name: Supply installed-wheel tests and template from the exact release checkout + shell: bash + run: | + cp -R python/tests python-dist/tests + cp python/pyproject.toml.j2 python-dist/pyproject.toml.j2 - name: Package and test CPU wheels from canonical wrapper sdist and shared runtime uses: pypa/cibuildwheel@v4.3.4 with: @@ -159,9 +201,7 @@ jobs: CIBW_MANYLINUX_X86_64_IMAGE: manylinux_2_28 CIBW_MANYLINUX_AARCH64_IMAGE: manylinux_2_28 CIBW_ENVIRONMENT_MACOS: MACOSX_DEPLOYMENT_TARGET=11.0 - # before-build only copies runtime files; wheel.cmake=false in the sdist. - CIBW_BEFORE_BUILD: python {package}/stage_runtime.py {package} {package}/external-runtime - CIBW_TEST_REQUIRES: pytest scikit-build-core>=1.1.2 mypy>=1.8 typing_extensions>=4.2 + CIBW_TEST_REQUIRES: pytest build scikit-build-core>=1.1.2 jinja2 packaging tomli mypy>=1.8 typing_extensions>=4.2 CIBW_TEST_COMMAND: python -m pytest {package}/tests -v - uses: actions/upload-artifact@v4 with: @@ -236,7 +276,25 @@ jobs: - name: Stage runtime and package canonical CUDA wrapper sdist run: | set -euo pipefail - python "$RUNNER_TEMP/clap-cuda/stage_runtime.py" "$RUNNER_TEMP/clap-cuda" "libraries/cuda" --backend cuda + cp -R "libraries/cuda/clap_cpp/." "$RUNNER_TEMP/clap-cuda/clap_cpp_cuda/" + cp "$RUNNER_TEMP/clap-cuda/LICENSE" "$RUNNER_TEMP/clap-cuda/clap_cpp_cuda/licenses/clap-cpp.txt" + python - <<'PY' + from pathlib import Path + import sys + package = Path(__import__("os").environ["RUNNER_TEMP"]) / "clap-cuda" / "clap_cpp_cuda" + native = "clap.dll" if sys.platform == "win32" else ("libclap.dylib" if sys.platform == "darwin" else "libclap.so") + runtime = package / native + if not runtime.is_file() or not runtime.stat().st_size: + raise SystemExit(f"Missing staged external runtime: {runtime}") + magic = runtime.read_bytes()[:4] + if not (magic == b"\x7fELF" if sys.platform.startswith("linux") else + magic[:2] == b"MZ" if sys.platform == "win32" else + magic in (b"\xcf\xfa\xed\xfe", b"\xfe\xed\xfa\xcf")): + raise SystemExit(f"Invalid staged external runtime: {runtime}") + notice = package / "licenses" / "clap-cpp.txt" + if not notice.is_file() or not notice.read_text(encoding="utf-8").strip(): + raise SystemExit(f"Missing runtime notices: {notice}") + PY python -m build --wheel "$RUNNER_TEMP/clap-cuda" --outdir raw - name: Repair CUDA wheel run: | @@ -316,7 +374,25 @@ jobs: - name: Stage runtime and package canonical VULKAN wrapper sdist run: | set -euo pipefail - python "$RUNNER_TEMP/clap-vulkan/stage_runtime.py" "$RUNNER_TEMP/clap-vulkan" "libraries/vulkan" --backend vulkan + cp -R "libraries/vulkan/clap_cpp/." "$RUNNER_TEMP/clap-vulkan/clap_cpp_vulkan/" + cp "$RUNNER_TEMP/clap-vulkan/LICENSE" "$RUNNER_TEMP/clap-vulkan/clap_cpp_vulkan/licenses/clap-cpp.txt" + python - <<'PY' + from pathlib import Path + import sys + package = Path(__import__("os").environ["RUNNER_TEMP"]) / "clap-vulkan" / "clap_cpp_vulkan" + native = "clap.dll" if sys.platform == "win32" else ("libclap.dylib" if sys.platform == "darwin" else "libclap.so") + runtime = package / native + if not runtime.is_file() or not runtime.stat().st_size: + raise SystemExit(f"Missing staged external runtime: {runtime}") + magic = runtime.read_bytes()[:4] + if not (magic == b"\x7fELF" if sys.platform.startswith("linux") else + magic[:2] == b"MZ" if sys.platform == "win32" else + magic in (b"\xcf\xfa\xed\xfe", b"\xfe\xed\xfa\xcf")): + raise SystemExit(f"Invalid staged external runtime: {runtime}") + notice = package / "licenses" / "clap-cpp.txt" + if not notice.is_file() or not notice.read_text(encoding="utf-8").strip(): + raise SystemExit(f"Missing runtime notices: {notice}") + PY python -m build --wheel "$RUNNER_TEMP/clap-vulkan" --outdir raw - name: Repair Vulkan wheel (Ubuntu 24.04 glibc floor) run: | @@ -414,7 +490,25 @@ jobs: - name: Stage runtime and package canonical ROCM wrapper sdist run: | set -euo pipefail - python "$RUNNER_TEMP/clap-rocm/stage_runtime.py" "$RUNNER_TEMP/clap-rocm" "libraries/rocm" --backend rocm + cp -R "libraries/rocm/clap_cpp/." "$RUNNER_TEMP/clap-rocm/clap_cpp_rocm/" + cp "$RUNNER_TEMP/clap-rocm/LICENSE" "$RUNNER_TEMP/clap-rocm/clap_cpp_rocm/licenses/clap-cpp.txt" + python - <<'PY' + from pathlib import Path + import sys + package = Path(__import__("os").environ["RUNNER_TEMP"]) / "clap-rocm" / "clap_cpp_rocm" + native = "clap.dll" if sys.platform == "win32" else ("libclap.dylib" if sys.platform == "darwin" else "libclap.so") + runtime = package / native + if not runtime.is_file() or not runtime.stat().st_size: + raise SystemExit(f"Missing staged external runtime: {runtime}") + magic = runtime.read_bytes()[:4] + if not (magic == b"\x7fELF" if sys.platform.startswith("linux") else + magic[:2] == b"MZ" if sys.platform == "win32" else + magic in (b"\xcf\xfa\xed\xfe", b"\xfe\xed\xfa\xcf")): + raise SystemExit(f"Invalid staged external runtime: {runtime}") + notice = package / "licenses" / "clap-cpp.txt" + if not notice.is_file() or not notice.read_text(encoding="utf-8").strip(): + raise SystemExit(f"Missing runtime notices: {notice}") + PY python -m build --wheel "$RUNNER_TEMP/clap-rocm" --outdir raw - name: Repair ROCm wheel, retaining system ROCm runtime run: | @@ -520,12 +614,25 @@ jobs: with zipfile.ZipFile(wheel) as archive: package = name.replace('-', '_') entries = set(archive.namelist()) + if any(entry.endswith('.j2') for entry in entries): + raise SystemExit(f'Wheel contains Jinja template: {wheel.name}') + if any('tests' in pathlib.PurePosixPath(entry).parts for entry in entries): + raise SystemExit(f'Wheel contains tests: {wheel.name}') if f'{package}/clap.py' not in entries: raise SystemExit(f'Missing Python ctypes wrapper: {wheel.name}') native = ('clap.dll' if platform == 'win-amd64' else 'libclap.dylib' if platform == 'macosx-arm64' else 'libclap.so') if f'{package}/{native}' not in entries: raise SystemExit(f'Missing package-local native library {package}/{native}: {wheel.name}') + payload = archive.read(f'{package}/{native}') + valid = (payload.startswith(b'\x7fELF') if platform.startswith('linux-') else + payload.startswith(b'MZ') if platform == 'win-amd64' else + payload[:4] in (b'\xcf\xfa\xed\xfe', b'\xfe\xed\xfa\xcf')) + if not valid: + raise SystemExit(f'Invalid package-local native library: {wheel.name}') + notice = f'{package}/licenses/clap-cpp.txt' + if notice not in entries or not archive.read(notice).strip(): + raise SystemExit(f'Missing runtime notices: {wheel.name}') if (dist / wheel.name).exists(): raise SystemExit(f'Duplicate release asset: {wheel.name}') shutil.copy2(wheel, dist) @@ -540,10 +647,18 @@ jobs: with tarfile.open(source) as archive: entries = {entry.name.split('/', 1)[1] for entry in archive if '/' in entry.name and entry.isfile()} package = name.replace('-', '_') - required = {'pyproject.toml', 'stage_runtime.py', f'{package}/clap.py'} + if any('tests' in pathlib.PurePosixPath(entry).parts for entry in entries): + raise SystemExit(f'Sdist contains tests: {source.name}') + if any(entry.endswith('.j2') for entry in entries): + raise SystemExit(f'Sdist contains Jinja template: {source.name}') + required = {'pyproject.toml', 'README.md', 'LICENSE', + f'{package}/__init__.py', f'{package}/clap.py', f'{package}/clap.pyi', f'{package}/py.typed'} if not required <= entries: - raise SystemExit(f'Sdist missing wrapper or staging helper: {source.name}: {required - entries}') - if any(entry.startswith('native/') or entry.endswith(('.so', '.dylib', '.dll', '.pyd')) for entry in entries): + raise SystemExit(f'Sdist missing wrapper or packaging input: {source.name}: {required - entries}') + if any(entry.startswith(('native/', 'cmake/')) or + pathlib.PurePosixPath(entry).name == 'CMakeLists.txt' or + entry.endswith(('.so', '.dylib', '.dll', '.pyd', '.cpp', '.c', '.h')) + for entry in entries): raise SystemExit(f'Sdist contains native sources or compiled runtime: {source.name}') if (dist / source.name).exists(): raise SystemExit(f'Duplicate release asset: {source.name}') @@ -574,7 +689,7 @@ jobs: ROCm GPU targets: gfx908,gfx90a,gfx942,gfx1030,gfx1100,gfx1101,gfx1200,gfx1201. Linux wheels are auditwheel-repaired manylinux wheels; the external GPU runtimes above remain deliberate exceptions, not portability promises. - Source archives contain wrappers and stage_runtime.py, not native sources. + Source archives contain wrappers and rendered direct backend metadata, not Jinja templates or native sources. Compile native runtimes separately from a recursive checkout, then stage them before packaging; Python wheel builds do not run CMake. CPU wheels pass installed-wheel tests on Python 3.9-3.14; GPU execution is not tested here. diff --git a/python/README.md b/python/README.md index 03ff75e..cec0bbe 100644 --- a/python/README.md +++ b/python/README.md @@ -102,51 +102,92 @@ stage it into the wrapper project and build a platform wheel: ```sh python -m pip install build 'scikit-build-core>=1.1.2' -python python/stage_runtime.py python libraries/linux-x64 +cp -R libraries/linux-x64/clap_cpp/. python/clap_cpp/ +cp python/LICENSE python/clap_cpp/licenses/clap-cpp.txt python -m build --wheel python python -m pip install python/dist/clap_cpp-*.whl ``` -`stage_runtime.py PROJECT_DIR RUNTIME_DIR [--backend cuda|rocm|vulkan]` -copies `RUNTIME_DIR/clap_cpp/` contents into the project's Python namespace, -preserving relative paths **within that namespace**. For example, -`clap_cpp/companions/libfoo.so` becomes `clap_cpp_cuda/companions/libfoo.so` for -`--backend cuda`. The common runtime artifact must contain only the `clap_cpp/` -namespace. Sibling payloads such as `clap_cpp.libs/` or a sibling `licenses/` -directory are rejected before any files are copied: moving them beneath the -package would break loader paths such as `$ORIGIN/../clap_cpp.libs`. Such artifacts -need separate loader-aware wheel packaging; the staging helper does not silently -relocate them. Keep native dependency notices inside `clap_cpp/licenses/`. -It adds the project's root MIT license as `licenses/clap-cpp.txt`, preserves -package-local native dependency notices, rejects missing runtimes and wrapper-source collisions, -and replaces an earlier staged payload without retaining stale dependencies. -Only stage trusted runtime artifacts matching the wheel's target platform; the -helper does not cross-compile, inspect binary architecture or repair Linux tags. -The native runtime's own loader paths must resolve any companion libraries. - -The backend guards wheel construction, failing with an actionable staging command -if the host-platform runtime or staged license is missing. It delegates packaging -to scikit-build-core with `wheel.cmake=false`, `wheel.platlib=true` and -`wheel.py-api="py3"`. A local Linux build is tagged `py3-none-linux_`; -release wheels still need normal platform portability checks/repair. Metadata -preparation and sdist creation do not require a native runtime. +Copy only trusted, matching-platform runtime artifacts. Preserve all relative +paths and dependency notices beneath `clap_cpp/`; sibling library directories +need separate loader-aware wheel repair, not relocation into the package. +Start from a fresh wrapper project when switching runtimes so stale companion +libraries cannot leak into the next wheel. Do not copy Python sources from a +runtime artifact or overwrite the wrapper files. + +The direct `scikit_build_core.build` backend only assembles the copied files: +`wheel.cmake=false`, `wheel.platlib=true`, `wheel.py-api="py3"` and +`sdist.cmake=false`. Wheel force-inclusion preserves Git-ignored runtime files; +explicit source inclusion keeps sdists wrapper-only. There is no custom backend, +build hook, staging or generation script. + +Actions validates a nonempty host-platform library with the expected binary +header and a nonempty `licenses/clap-cpp.txt` before building wheels. The final +release gate validates the archived payload and notices again before publication. +Local builds must perform equivalent checks: the direct backend does **not** +reject an unstaged project, and a wheel without its matching native runtime is +unusable and must never be published. Metadata and wrapper-only source archives +need no runtime. Editable installs reference staged sources and are development +artifacts, not distributable runtime wheels. A local Linux wheel has a +`linux_` tag and still needs portability checks/repair before release. + +On macOS, scikit-build-core selects the executing architecture via packaging's +platform tags, rather than treating a universal2 interpreter as proof of a +universal2 runtime. Keep `MACOSX_DEPLOYMENT_TARGET=11.0` for both the external +native build and wheel assembly to preserve the supported deployment minimum. +For an ARM64 runtime built at that minimum on an ARM64 host: + +```sh +MACOSX_DEPLOYMENT_TARGET=11.0 python -m build --wheel python +``` + +This does not convert runtime architectures or change an existing binary's +minimum. Cross-packaging requires explicit matching platform configuration; +never label a single-architecture runtime universal2. ### Optional backend projects -Generate a wrapper-only project, then stage the independently compiled backend -runtime (the native artifact still uses its common `clap_cpp/` namespace): +`pyproject.toml.j2` is the shared CPU/CUDA/ROCm/Vulkan metadata template. The +checked-in `pyproject.toml` is its CPU rendering for ordinary local builds. +The template stays in the repository; wheels and sdists contain only rendered +metadata, never `.j2` files. +Render with `backend=cpu|cuda|rocm|vulkan` and one PEP 440 `version` (default +`0.1.0`); CPU extras pin that exact version of each backend distribution. +For example, using the canonical CPU sdist extracted into `python-dist` and +`python/pyproject.toml.j2` from the exact checkout that produced that archive: ```sh -python python/prepare_backend.py cuda build/python-cuda -python python/stage_runtime.py build/python-cuda libraries/linux-x64-cuda --backend cuda -python -m build --wheel build/python-cuda +python -m pip install jinja2 +backend=cuda # or rocm, vulkan +project="build/python-$backend" +package="clap_cpp_$backend" +mkdir -p "$project/$package" +cp python-dist/{README.md,LICENSE} "$project/" +cp python-dist/clap_cpp/{clap.py,clap.pyi,py.typed} "$project/$package/" +printf '"""Native %s runtime; import clap_cpp for the public API."""\n' "$backend" > "$project/$package/__init__.py" +BACKEND="$backend" PROJECT="$project" python - <<'PY' +import os +from pathlib import Path +from jinja2 import Environment, StrictUndefined +try: + import tomllib +except ImportError: + import tomli as tomllib # install tomli on Python 3.9/3.10 +source = Path('python-dist') +version = tomllib.loads((source / 'pyproject.toml').read_text())['project']['version'] +template = Environment(undefined=StrictUndefined, keep_trailing_newline=True).from_string( + Path('python/pyproject.toml.j2').read_text()) +(Path(os.environ['PROJECT']) / 'pyproject.toml').write_text( + template.render(backend=os.environ['BACKEND'], version=version)) +PY +cp -R "libraries/linux-x64-$backend/clap_cpp/." "$project/$package/" +cp "$project/LICENSE" "$project/$package/licenses/clap-cpp.txt" +python -m build --wheel "$project" ``` -Replace `cuda` with `rocm` or `vulkan`. The helper preserves the base version, -wrapper and typing files; no native sources or CMake settings are copied. The -staging helper and build guard are included in each backend source distribution. -The public selector continues to use `clap_cpp_cuda`, `clap_cpp_rocm` and -`clap_cpp_vulkan` without changing the base package or its public APIs. +The release workflow performs these copies and Jinja renders directly; no +project-generation or staging script is needed. The selector still uses +`clap_cpp_cuda`, `clap_cpp_rocm` and `clap_cpp_vulkan` with unchanged public APIs. ## Development and source distributions @@ -154,20 +195,25 @@ The public selector continues to use `clap_cpp_cuda`, `clap_cpp_rocm` and python -m build --sdist python # Unpack the archive, then explicitly stage its external runtime: tar -xf python/dist/clap_cpp-0.1.0.tar.gz -C build -python build/clap_cpp-0.1.0/stage_runtime.py build/clap_cpp-0.1.0 libraries/linux-x64 +cp -R libraries/linux-x64/clap_cpp/. build/clap_cpp-0.1.0/clap_cpp/ +cp build/clap_cpp-0.1.0/LICENSE build/clap_cpp-0.1.0/clap_cpp/licenses/clap-cpp.txt python -m build --wheel build/clap_cpp-0.1.0 python -m pip install 'build/clap_cpp-0.1.0/dist/'*.whl -python -m pip install pytest mypy typing_extensions build 'scikit-build-core>=1.1.2' +python -m pip install pytest mypy typing_extensions build 'scikit-build-core>=1.1.2' jinja2 python -m pytest python/tests ``` -Source distributions contain wrapper/typing files, tests, `LICENSE` and packaging -helpers, not native C++ snapshots, staged binaries or native build definitions. -An unstaged sdist intentionally cannot build a wheel: plain `python -m build -python` creates an sdist then fails to build its unstaged wheel. Use the explicit -`--sdist`, unpack, stage, `--wheel` sequence above. Backend projects can also be -generated from the extracted CPU sdist before staging their runtimes. No NumPy, -Python development headers or pybind11 are required. +Source distributions contain wrapper/typing files, `LICENSE` and rendered +metadata, not Jinja templates, native C++ snapshots, staged binaries, build hooks +or native build definitions. Tests and `.j2` files are excluded from both wheels +and sdists; release CI supplies tests and the metadata template separately from +the exact release checkout and tests the installed wheel on each supported +interpreter. Plain `python -m build python` builds from an +unstaged sdist and therefore produces an unusable runtime-free wheel; do not +publish it. Use the explicit `--sdist`, unpack, stage, `--wheel` sequence above. +Backend projects reuse wrappers and the version from the extracted CPU sdist, +rendering metadata with the template from that same checkout before staging +their runtimes. No NumPy, Python development headers or pybind11 are required. ## Releases @@ -181,4 +227,4 @@ trusted publisher (owner `gorse-io`, repository `clap.cpp`, workflow `release.ym environment `pypi`) for **each** of `clap-cpp`, `clap-cpp-cuda`, `clap-cpp-rocm` and `clap-cpp-vulkan`, and any required GitHub `pypi` environment approvals. Prereleases only attach assets. Normalize the release version and the three exact -extra pins together; `prepare_backend.py` copies that version to each backend. +extra pins together by rendering the shared Jinja template for each backend. diff --git a/python/_build_backend.py b/python/_build_backend.py deleted file mode 100644 index ae4175e..0000000 --- a/python/_build_backend.py +++ /dev/null @@ -1,66 +0,0 @@ -"""Thin scikit-build-core guard: a wheel must have a staged native runtime. - -Metadata and source distributions need no runtime. Wheel/editable construction -checks the package-only contract before delegating; no native build fallback. -""" - -from pathlib import Path -import sys - -from scikit_build_core import build as _build -from scikit_build_core.settings.skbuild_read_settings import SettingsReader - -from stage_runtime import RUNTIME_NAMES - - -# Reading the project here keeps CPU and generated backend projects identical. -try: - import tomllib -except ImportError: - import tomli as tomllib - - -def _guard(config_settings, state): - with Path("pyproject.toml").open("rb") as stream: - metadata = tomllib.load(stream) - reader = SettingsReader(metadata, config_settings or {}, state=state) - reader.validate_may_exit() - settings = reader.settings - if settings.wheel.cmake or not settings.wheel.platlib or settings.wheel.py_api != "py3": - raise RuntimeError("Python wheels require wheel.cmake=false, wheel.platlib=true, wheel.py-api=py3") - packages = settings.wheel.packages - expected = metadata["project"]["name"].replace("-", "_") - if packages != [expected]: - raise RuntimeError(f"Expected wheel.packages=[{expected!r}] for this runtime project") - package = Path(expected) - native = "clap.dll" if sys.platform == "win32" else ( - "libclap.dylib" if sys.platform == "darwin" else "libclap.so" - ) - if native not in RUNTIME_NAMES or not (package / native).is_file() or (package / native).stat().st_size == 0: - backend = expected.removeprefix("clap_cpp_") if expected != "clap_cpp" else None - option = f" --backend {backend}" if backend else "" - raise RuntimeError( - f"Missing staged external runtime: {package / native}. " - f"Run: python stage_runtime.py . /path/to/libraries/TARGET{option}. " - "Python packaging does not compile native libraries." - ) - if not (package / "licenses" / "clap-cpp.txt").is_file(): - raise RuntimeError("Missing runtime notices; run stage_runtime.py before building a wheel") - - -def build_wheel(wheel_directory, config_settings=None, metadata_directory=None): - _guard(config_settings, "wheel") - return _build.build_wheel(wheel_directory, config_settings, metadata_directory) - - -def build_editable(wheel_directory, config_settings=None, metadata_directory=None): - _guard(config_settings, "editable") - return _build.build_editable(wheel_directory, config_settings, metadata_directory) - - -build_sdist = _build.build_sdist -get_requires_for_build_sdist = _build.get_requires_for_build_sdist -get_requires_for_build_wheel = _build.get_requires_for_build_wheel -get_requires_for_build_editable = _build.get_requires_for_build_editable -prepare_metadata_for_build_wheel = _build.prepare_metadata_for_build_wheel -prepare_metadata_for_build_editable = _build.prepare_metadata_for_build_editable diff --git a/python/prepare_backend.py b/python/prepare_backend.py deleted file mode 100644 index 77b58b3..0000000 --- a/python/prepare_backend.py +++ /dev/null @@ -1,72 +0,0 @@ -"""Create a wrapper-only GPU wheel project from a checkout or base sdist. - -Stage its externally compiled runtime later using stage_runtime.py --backend. -""" - -import argparse -from pathlib import Path -import re -import shutil - - -BACKENDS = ("cuda", "rocm", "vulkan") - - -def prepare(backend, destination): - if backend not in BACKENDS: - raise ValueError(f"Unknown backend: {backend}") - destination = Path(destination) - source = Path(__file__).resolve().parent - metadata = (source / "pyproject.toml").read_text(encoding="utf-8") - match = re.search(r'^version = "([^"]+)"$', metadata, re.MULTILINE) - if match is None: - raise ValueError("Missing package version") - version = match.group(1) - package = f"clap_cpp_{backend}" - destination.mkdir(parents=True, exist_ok=False) - for name in ("README.md", "LICENSE", "stage_runtime.py", "_build_backend.py"): - shutil.copy2(source / name, destination / name) - (destination / package).mkdir() - for name in ("clap.py", "clap.pyi", "py.typed"): - shutil.copy2(source / "clap_cpp" / name, destination / package / name) - (destination / package / "__init__.py").write_text( - f'"""Native {backend} runtime; import clap_cpp for the public API."""\n', encoding="utf-8" - ) - (destination / "pyproject.toml").write_text( - f'''[build-system] -requires = ["scikit-build-core>=1.1.2"] -build-backend = "_build_backend" -backend-path = ["."] - -[project] -name = "clap-cpp-{backend}" -version = "{version}" -description = "{backend.upper()} backend for clap-cpp" -readme = "README.md" -requires-python = ">=3.9" -license = "MIT" -license-files = ["LICENSE"] -dependencies = [] - -[tool.scikit-build] -minimum-version = "build-system.requires" -wheel.packages = ["{package}"] -wheel.force-include = {{ {package} = "{package}" }} -wheel.cmake = false -wheel.platlib = true -wheel.py-api = "py3" -sdist.cmake = false -sdist.inclusion-mode = "explicit" -sdist.include = ["pyproject.toml", "README.md", "LICENSE", "_build_backend.py", "stage_runtime.py", "{package}/*.py", "{package}/*.pyi", "{package}/py.typed"] -sdist.exclude = ["**/__pycache__/**"] -''', encoding="utf-8" - ) - print(f"Prepared clap-cpp-{backend} {version} in {destination}") - - -if __name__ == "__main__": - parser = argparse.ArgumentParser(description=__doc__) - parser.add_argument("backend", choices=BACKENDS) - parser.add_argument("destination", type=Path) - args = parser.parse_args() - prepare(args.backend, args.destination.resolve()) diff --git a/python/pyproject.toml b/python/pyproject.toml index e3297e8..97e52a9 100644 --- a/python/pyproject.toml +++ b/python/pyproject.toml @@ -1,7 +1,7 @@ +# Generated from pyproject.toml.j2; render for cpu, cuda, rocm or vulkan. [build-system] requires = ["scikit-build-core>=1.1.2"] -build-backend = "_build_backend" -backend-path = ["."] +build-backend = "scikit_build_core.build" [project] name = "clap-cpp" @@ -20,19 +20,21 @@ rocm = ["clap-cpp-rocm==0.1.0"] vulkan = ["clap-cpp-vulkan==0.1.0"] [tool.scikit-build] -minimum-version = "build-system.requires" -wheel.packages = ["clap_cpp"] -# Wheel payload includes the staged runtime despite the wrapper-only sdist list. -wheel.force-include = { clap_cpp = "clap_cpp" } -# Package an externally built C ABI runtime, never configure or compile it here. +minimum-version = "1.1.2" wheel.cmake = false wheel.platlib = true wheel.py-api = "py3" +wheel.packages = [] +wheel.exclude = ["**/__pycache__/**", "**/*.pyc", "**/.pytest_cache/**", "**/tests/**", "**/*.j2"] sdist.cmake = false -# Explicit wrapper-only inventory also excludes staged binaries and their notices. +sdist.exclude = ["**/__pycache__/**", "**/*.pyc", "**/.pytest_cache/**", "**/tests/**", "**/*.j2"] sdist.inclusion-mode = "explicit" -sdist.include = ["pyproject.toml", "README.md", "LICENSE", "_build_backend.py", "stage_runtime.py", "prepare_backend.py", "clap_cpp/*.py", "clap_cpp/*.pyi", "clap_cpp/py.typed", "tests/**"] -sdist.exclude = ["**/__pycache__/**", "**/.pytest_cache/**"] +# Never ship staged binaries, companion data or runtime notices in source archives. +sdist.include = ["/pyproject.toml", "/README.md", "/LICENSE", "/clap_cpp/*.py", "/clap_cpp/*.pyi", "/clap_cpp/py.typed"] + +[tool.scikit-build.wheel.force-include] +# Include Git-ignored external payloads without broadening the source archive. +"clap_cpp" = "clap_cpp" [tool.pytest.ini_options] testpaths = ["tests"] diff --git a/python/pyproject.toml.j2 b/python/pyproject.toml.j2 new file mode 100644 index 0000000..b60f194 --- /dev/null +++ b/python/pyproject.toml.j2 @@ -0,0 +1,44 @@ +{% set backend = backend | default('cpu') -%} +{% set version = version | default('0.1.0') -%} +{% set package = 'clap_cpp' ~ ('_' ~ backend if backend != 'cpu' else '') -%} +# Generated from pyproject.toml.j2; render for cpu, cuda, rocm or vulkan. +[build-system] +requires = ["scikit-build-core>=1.1.2"] +build-backend = "scikit_build_core.build" + +[project] +name = "{{ package | replace('_', '-') }}" +version = "{{ version }}" +description = "{{ 'Python bindings for clap.cpp audio and text embeddings' if backend == 'cpu' else backend | upper ~ ' backend for clap-cpp' }}" +readme = "README.md" +requires-python = ">=3.9" +license = "MIT" +license-files = ["LICENSE"] +dependencies = [] +{% if backend == 'cpu' %} +[project.optional-dependencies] +test = ["pytest>=7", "mypy>=1.8", "typing_extensions>=4.2"] +cuda = ["clap-cpp-cuda=={{ version }}"] +rocm = ["clap-cpp-rocm=={{ version }}"] +vulkan = ["clap-cpp-vulkan=={{ version }}"] +{% endif %} +[tool.scikit-build] +minimum-version = "1.1.2" +wheel.cmake = false +wheel.platlib = true +wheel.py-api = "py3" +wheel.packages = [] +wheel.exclude = ["**/__pycache__/**", "**/*.pyc", "**/.pytest_cache/**", "**/tests/**", "**/*.j2"] +sdist.cmake = false +sdist.exclude = ["**/__pycache__/**", "**/*.pyc", "**/.pytest_cache/**", "**/tests/**", "**/*.j2"] +sdist.inclusion-mode = "explicit" +# Never ship staged binaries, companion data or runtime notices in source archives. +sdist.include = ["/pyproject.toml", "/README.md", "/LICENSE", "/{{ package }}/*.py", "/{{ package }}/*.pyi", "/{{ package }}/py.typed"] + +[tool.scikit-build.wheel.force-include] +# Include Git-ignored external payloads without broadening the source archive. +"{{ package }}" = "{{ package }}" +{% if backend == 'cpu' %} +[tool.pytest.ini_options] +testpaths = ["tests"] +{% endif %} \ No newline at end of file diff --git a/python/stage_runtime.py b/python/stage_runtime.py deleted file mode 100644 index 3da4759..0000000 --- a/python/stage_runtime.py +++ /dev/null @@ -1,100 +0,0 @@ -"""Stage an external runtime: stage_runtime.py PROJECT_DIR RUNTIME_DIR [--backend NAME]. - -RUNTIME_DIR contains only clap_cpp/, the common native install namespace. -Its contents go into the target Python namespace without changing package-local -relative paths. Sibling payloads such as clap_cpp.libs are explicitly rejected: -relocating them would break loader paths like $ORIGIN/../clap_cpp.libs. No -compilation or native source snapshot is performed by Python packaging. -""" - -import argparse -from pathlib import Path -import shutil - - -BACKENDS = ("cuda", "rocm", "vulkan") -RUNTIME_NAMES = ("libclap.so", "libclap.dylib", "clap.dll") -WRAPPERS = ("__init__.py", "clap.py", "clap.pyi", "py.typed") - - -def stage(project_dir, runtime_dir, backend=None): - """Copy binaries/notices into an existing wrapper project; reject missing input.""" - if backend is not None and backend not in BACKENDS: - raise ValueError(f"Unknown backend: {backend}") - project = Path(project_dir).resolve() - runtime = Path(runtime_dir).resolve() - package = project / (f"clap_cpp_{backend}" if backend else "clap_cpp") - if not (project / "pyproject.toml").is_file() or not all( - (package / name).is_file() for name in WRAPPERS - ): - raise ValueError(f"Not a matching wrapper project: {package}") - source = runtime / "clap_cpp" - if not source.is_dir() or not any((source / name).is_file() for name in RUNTIME_NAMES): - raise FileNotFoundError( - f"Missing external runtime in {source}: expected " + ", ".join(RUNTIME_NAMES) - ) - siblings = sorted(path.name for path in runtime.iterdir() if path.name != "clap_cpp") - if siblings: - raise ValueError( - f"Unsupported runtime sibling payloads: {', '.join(siblings)}. " - "Only package-local native payloads beneath clap_cpp/ are accepted; " - "sibling library directories require separate loader-aware wheel packaging." - ) - if runtime == project or runtime in package.parents or package in runtime.parents: - raise ValueError("External runtime and target package must not overlap") - license_file = project / "LICENSE" - if not license_file.is_file(): - raise FileNotFoundError(f"Missing project LICENSE: {license_file}") - # Validate the complete copy before touching the destination. Never let a - # runtime artifact replace the wrapper, even accidentally. - files = {} - for origin in sorted(source.rglob("*")): - if not origin.is_file(): - continue - try: - origin.resolve().relative_to(runtime) - except ValueError: - raise ValueError(f"Runtime symlink escapes runtime directory: {origin}") from None - relative = origin.relative_to(source) - if relative.parts[0] in WRAPPERS or origin.suffix in (".py", ".pyi", ".pyc"): - raise ValueError(f"External runtime must not contain Python wrapper sources: {origin}") - if relative in files: - raise ValueError(f"Runtime paths collide after namespace mapping: {relative}") - target = package / relative - if not target.resolve().is_relative_to(package): - raise ValueError(f"Runtime target escapes package: {target}") - files[relative] = origin - # Restaging replaces the old native payload, so switching runtimes cannot - # leave stale dependencies/notices in the next wheel. Keep wrapper files. - for existing in package.iterdir(): - if existing.name in WRAPPERS: - continue - if existing.is_dir() and not existing.is_symlink(): - shutil.rmtree(existing) - else: - existing.unlink() - for relative, origin in files.items(): - target = package / relative - target.parent.mkdir(parents=True, exist_ok=True) - shutil.copy2(origin, target) - notices = package / "licenses" - notices.mkdir(exist_ok=True) - shutil.copy2(license_file, notices / "clap-cpp.txt") - return package - - -def main(): - parser = argparse.ArgumentParser(description=__doc__) - parser.add_argument("project_dir", type=Path) - parser.add_argument("runtime_dir", type=Path) - parser.add_argument("--backend", choices=BACKENDS) - args = parser.parse_args() - try: - package = stage(args.project_dir, args.runtime_dir, args.backend) - except (ValueError, OSError) as exc: - parser.exit(1, f"{exc}\n") - print(f"Staged external runtime in {package}") - - -if __name__ == "__main__": - main() diff --git a/python/tests/test_packaging.py b/python/tests/test_packaging.py index 2e0565f..e61db96 100644 --- a/python/tests/test_packaging.py +++ b/python/tests/test_packaging.py @@ -1,6 +1,5 @@ -"""Package-only wheels consume an explicitly staged external C API runtime.""" +"""Package-only scikit-build-core wheels consume an explicitly copied external runtime.""" -import importlib.util import os from pathlib import Path import shutil @@ -9,6 +8,7 @@ import tarfile import zipfile +from jinja2 import Environment, StrictUndefined import pytest try: @@ -27,19 +27,25 @@ def metadata(path=PROJECT): return tomllib.loads((path / "pyproject.toml").read_text(encoding="utf-8")) -def load(name, path): - spec = importlib.util.spec_from_file_location(name, path) - assert spec is not None and spec.loader is not None - module = importlib.util.module_from_spec(spec) - spec.loader.exec_module(module) - return module - - -def copy_project(destination): - shutil.copytree(PROJECT, destination, ignore=shutil.ignore_patterns( - "__pycache__", ".pytest_cache", "dist", "build", "*.so", "*.dylib", "*.dll", "licenses" - )) - return destination +def render(source, destination, backend="cpu", version=None, template_source=PROJECT): + """Mirror the workflow's Jinja rendering and plain wrapper copies.""" + destination.mkdir(parents=True) + package = "clap_cpp" + (f"_{backend}" if backend != "cpu" else "") + for name in ("README.md", "LICENSE"): + shutil.copy2(source / name, destination / name) + shutil.copy2(template_source / "pyproject.toml.j2", destination / "pyproject.toml.j2") + shutil.copytree(source / "clap_cpp", destination / package, + ignore=shutil.ignore_patterns("__pycache__", "*.so", "*.dylib", "*.dll", "licenses")) + if backend != "cpu": + (destination / package / "__init__.py").write_text( + f'"""Native {backend} runtime; import clap_cpp for the public API."""\n', encoding="utf-8") + else: + shutil.copytree(source / "tests", destination / "tests") + template = Environment(undefined=StrictUndefined, keep_trailing_newline=True).from_string( + (template_source / "pyproject.toml.j2").read_text(encoding="utf-8")) + (destination / "pyproject.toml").write_text(template.render( + backend=backend, version=version or metadata(source)["project"]["version"]), encoding="utf-8") + return destination, package def build(project, out, kind, env=None): @@ -49,181 +55,134 @@ def build(project, out, kind, env=None): ) -def test_package_only_settings(tmp_path, monkeypatch): - from scikit_build_core.settings.skbuild_read_settings import SettingsReader - - monkeypatch.chdir(tmp_path) - config = metadata() - assert config["build-system"]["build-backend"] == "_build_backend" - assert config["build-system"]["backend-path"] == ["."] - assert "cmake" not in config["tool"]["scikit-build"] - assert "force-include" not in config["tool"]["scikit-build"]["sdist"] - for state in ("wheel", "editable", "sdist"): - reader = SettingsReader(config, {}, state=state) - reader.validate_may_exit() - settings = reader.settings - assert settings.wheel.cmake is False - assert settings.wheel.platlib is True - assert settings.wheel.py_api == "py3" - assert settings.sdist.cmake is False - - -@pytest.mark.parametrize("backend", ["cuda", "rocm", "vulkan"]) -def test_backend_projects_wrapper_only(tmp_path, backend): - destination = tmp_path / backend - load("generator", PROJECT / "prepare_backend.py").prepare(backend, destination) - config = metadata(destination) - assert config["project"]["name"] == f"clap-cpp-{backend}" - assert config["project"]["version"] == metadata()["project"]["version"] - assert config["project"]["dependencies"] == [] +def test_template_matches_checked_in_cpu_metadata(tmp_path): + project, _ = render(PROJECT, tmp_path / "cpu") + assert (project / "pyproject.toml").read_bytes() == (PROJECT / "pyproject.toml").read_bytes() + config = metadata(project) + assert config["build-system"] == {"requires": ["scikit-build-core>=1.1.2"], "build-backend": "scikit_build_core.build"} settings = config["tool"]["scikit-build"] - assert settings["wheel"] == dict(packages=[f"clap_cpp_{backend}"], **{ - "py-api": "py3", "cmake": False, "platlib": True, - "force-include": {f"clap_cpp_{backend}": f"clap_cpp_{backend}"}, - }) - assert "cmake" not in settings - assert not (destination / "native").exists() - assert not list(destination.rglob("CMakeLists.txt")) - for name in ("stage_runtime.py", "_build_backend.py", "LICENSE"): - assert (destination / name).read_bytes() == (PROJECT / name).read_bytes() + assert settings["wheel"]["cmake"] is False + assert settings["wheel"]["platlib"] is True + assert settings["wheel"]["py-api"] == "py3" + assert settings["sdist"]["cmake"] is False + assert settings["sdist"]["inclusion-mode"] == "explicit" + assert settings["wheel"]["force-include"] == {"clap_cpp": "clap_cpp"} + + +@pytest.mark.parametrize("backend", ["cpu", "cuda", "rocm", "vulkan"]) +def test_wheel_and_wrapper_only_sdist(tmp_path, backend): + project, package = render(PROJECT, tmp_path / "project", backend, "2.3.4rc1") + config = metadata(project) + (project / ".gitignore").write_text("*.so\n*.dll\n*.dylib\ncompanions/\nlicenses/\n") + (project / "unlisted.txt").write_text("Must not enter explicit sdists") + (project / "tests").mkdir(exist_ok=True) + (project / "tests/test_not_for_distribution.py").write_text("# Source-only test fixture") + (project / package / "tests").mkdir() + (project / package / "tests/test_not_for_distribution.py").write_text("# Must not leak via force inclusion") + (project / package / "unused.j2").write_text("Must not leak via force inclusion") + (project / package / "__pycache__").mkdir() + (project / package / "__pycache__/clap.pyc").write_bytes(b"cache fixture") + assert config["project"]["name"] == package.replace("_", "-") + assert config["project"]["version"] == "2.3.4rc1" + assert config["project"]["dependencies"] == [] + if backend == "cpu": + for name in ("cuda", "rocm", "vulkan"): + assert config["project"]["optional-dependencies"][name] == [f"clap-cpp-{name}==2.3.4rc1"] for name in ("clap.py", "clap.pyi", "py.typed"): - assert (destination / f"clap_cpp_{backend}" / name).read_bytes() == (PROJECT / "clap_cpp" / name).read_bytes() - result = build(destination, tmp_path / "missing-wheel", "wheel") - assert result.returncode != 0 and "stage_runtime.py" in result.stdout - result = build(destination, tmp_path / "dist", "sdist") - assert result.returncode == 0, result.stdout - with tarfile.open(next((tmp_path / "dist").glob("*.tar.gz"))) as archive: - names = archive.getnames() - assert any(n.endswith("/stage_runtime.py") for n in names) - assert any(n.endswith("/_build_backend.py") for n in names) - assert not any("/native/" in n or n.endswith((".so", ".dll", ".dylib")) for n in names) - archive.extractall(tmp_path / "extracted", filter="data") - source = next((tmp_path / "extracted").iterdir()) - native = "clap.dll" if sys.platform == "win32" else ("libclap.dylib" if sys.platform == "darwin" else "libclap.so") - runtime = tmp_path / "runtime" / "clap_cpp" - runtime.mkdir(parents=True) - (runtime / native).write_bytes(b"backend packaging fixture, not executable") - load("backend_stage", source / "stage_runtime.py").stage(source, runtime.parent, backend) - result = build(source, tmp_path / "staged-wheel", "wheel") - assert result.returncode == 0, result.stdout - wheel = next((tmp_path / "staged-wheel").glob("*.whl")) - assert "-py3-none-" in wheel.name and not wheel.name.endswith("-any.whl") - with zipfile.ZipFile(wheel) as archive: - assert archive.read(f"clap_cpp_{backend}/{native}") == (runtime / native).read_bytes() - assert f"clap_cpp_{backend}/licenses/clap-cpp.txt" in archive.namelist() - - -@pytest.mark.parametrize("backend", [None, "cuda", "rocm", "vulkan"]) -def test_stage_runtime(tmp_path, backend): - stage = load("stage", PROJECT / "stage_runtime.py") - project = tmp_path / "project" - if backend: - load("generator", PROJECT / "prepare_backend.py").prepare(backend, project) - else: - copy_project(project) - runtime = tmp_path / "runtime" - (runtime / "clap_cpp" / "licenses").mkdir(parents=True) - (runtime / "clap_cpp" / "libclap.so").write_bytes(b"unit-test runtime, not executable") - (runtime / "clap_cpp" / "licenses" / "icu.txt").write_text("ICU notice") - (runtime / "clap_cpp" / "licenses" / "other.txt").write_text("companion notice") - (runtime / "clap_cpp" / "companions").mkdir() - (runtime / "clap_cpp" / "companions" / "libdependency.so.1").write_bytes(b"test companion") - stage.stage(project, runtime, backend=backend) - package = project / (f"clap_cpp_{backend}" if backend else "clap_cpp") - assert (package / "libclap.so").read_bytes() == (runtime / "clap_cpp" / "libclap.so").read_bytes() - assert (package / "licenses" / "icu.txt").read_text() == "ICU notice" - assert (package / "licenses" / "other.txt").read_text() == "companion notice" - assert (package / "companions" / "libdependency.so.1").exists() - assert (package / "licenses" / "clap-cpp.txt").read_bytes() == (PROJECT / "LICENSE").read_bytes() - assert (package / "clap.py").read_bytes() == (PROJECT / "clap_cpp" / "clap.py").read_bytes() - - -@pytest.mark.parametrize("sibling", ["clap_cpp.libs", "licenses"]) -def test_stage_rejects_sibling_directories_without_relocation(tmp_path, sibling): - project = copy_project(tmp_path / "project") - runtime = tmp_path / "runtime" - (runtime / "clap_cpp").mkdir(parents=True) - (runtime / "clap_cpp" / RUNTIME_NAME).write_bytes(b"packaging fixture") - (runtime / sibling).mkdir() - (runtime / sibling / "companion.so").write_bytes(b"sibling payload") - with pytest.raises(ValueError, match="sibling"): - load("stage", PROJECT / "stage_runtime.py").stage(project, runtime) - assert not (project / "clap_cpp" / RUNTIME_NAME).exists() - assert not (project / "clap_cpp" / sibling).exists() - assert not (project / sibling).exists() - - -def test_missing_runtime_and_wrapper_collision(tmp_path): - stage = load("stage", PROJECT / "stage_runtime.py") - project = copy_project(tmp_path / "project") - with pytest.raises((ValueError, FileNotFoundError), match="runtime"): - stage.stage(project, tmp_path / "absent") - runtime = tmp_path / "runtime" - (runtime / "clap_cpp").mkdir(parents=True) - with pytest.raises((ValueError, FileNotFoundError), match="runtime"): - stage.stage(project, runtime) - (runtime / "clap_cpp" / "libclap.so").write_bytes(b"test") - (runtime / "clap_cpp" / "clap.py").write_text("overwrite") - with pytest.raises(ValueError, match="wrapper"): - stage.stage(project, runtime) - assert not (project / "clap_cpp" / "libclap.so").exists() - - -def test_missing_and_staged_wheel_sdist_contract(tmp_path): - project = copy_project(tmp_path / "project") - out = tmp_path / "dist" - # Any attempt to execute CMake/Ninja fails and leaves visible evidence. + assert (project / package / name).read_bytes() == (PROJECT / "clap_cpp" / name).read_bytes() + # Assembly must not invoke a native build, including when building from sdist. tools = tmp_path / "tools" tools.mkdir() marker = tmp_path / "compiler-called" - for tool in ("cmake", "ninja", "cc", "c++"): + for tool in ("cmake", "ninja", "cc", "c++", "gcc", "g++", "clang", "clang++", "cl"): executable = tools / tool executable.write_text(f'#!/bin/sh\nprintf called > "{marker}"\nexit 91\n') executable.chmod(0o755) env = dict(os.environ, PATH=str(tools) + os.pathsep + os.environ.get("PATH", "")) + out = tmp_path / "dist" + metadata_dir = project / "metadata" + metadata_dir.mkdir() result = subprocess.run( - [sys.executable, "-c", "import _build_backend as b; " - "assert b.get_requires_for_build_wheel() == []; " - "assert b.get_requires_for_build_sdist() == []; " - "b.prepare_metadata_for_build_wheel('metadata')"], - cwd=project, env=env, text=True, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, + [sys.executable, "-c", "from scikit_build_core.build import prepare_metadata_for_build_wheel; " + "prepare_metadata_for_build_wheel('metadata')"], + cwd=project, env=env, capture_output=True, text=True, ) - assert result.returncode == 0, result.stdout - assert list((project / "metadata").glob("*.dist-info/METADATA")) - result = build(project, out, "wheel", env) - assert result.returncode != 0 - assert "stage_runtime.py" in result.stdout - assert not list(out.glob("*.whl")) - runtime = tmp_path / "runtime" / "clap_cpp" - runtime.mkdir(parents=True) - (runtime / RUNTIME_NAME).write_bytes(b"unit packaging payload only, not inference") - load("stage", PROJECT / "stage_runtime.py").stage(project, runtime.parent) + assert result.returncode == 0, result.stdout + result.stderr + assert list(metadata_dir.glob("*.dist-info/METADATA")) + # Validation belongs to Actions, not a replacement Python backend/hook. + # Prefer an actual externally compiled runtime; fixture-only jobs still test copying. + native = project / package / RUNTIME_NAME + external = os.environ.get("CLAP_TEST_EXTERNAL_RUNTIME") + if external: + shutil.copy2(Path(external) / RUNTIME_NAME, native) + else: + native.write_bytes(b"packaging fixture only, NOT an executable runtime") + notices = project / package / "licenses" + notices.mkdir() + shutil.copy2(project / "LICENSE", notices / "clap-cpp.txt") + (notices / "dependency.txt").write_text("Dependency notice") + companion = project / package / "companions" / "libdependency.so.1" + companion.parent.mkdir() + companion.write_bytes(b"companion fixture") result = build(project, out, "wheel", env) assert result.returncode == 0, result.stdout wheel = next(out.glob("*.whl")) assert "-py3-none-" in wheel.name and not wheel.name.endswith("-any.whl") with zipfile.ZipFile(wheel) as archive: - wheel_metadata = archive.read(next(n for n in archive.namelist() if n.endswith(".dist-info/WHEEL"))).decode() - assert "Root-Is-Purelib: false" in wheel_metadata - assert archive.read(f"clap_cpp/{RUNTIME_NAME}") == (runtime / RUNTIME_NAME).read_bytes() - assert "clap_cpp/clap.pyi" in archive.namelist() - assert "clap_cpp/py.typed" in archive.namelist() - assert "clap_cpp/licenses/clap-cpp.txt" in archive.namelist() + entries = archive.namelist() + info = archive.read(next(n for n in entries if n.endswith(".dist-info/WHEEL"))) + assert b"Root-Is-Purelib: false" in info + assert archive.read(f"{package}/{RUNTIME_NAME}") == native.read_bytes() + assert archive.read(f"{package}/companions/libdependency.so.1") == companion.read_bytes() + assert archive.read(f"{package}/licenses/clap-cpp.txt") == (project / "LICENSE").read_bytes() + assert f"{package}/licenses/dependency.txt" in entries + for name in ("__init__.py", "clap.py", "clap.pyi", "py.typed"): + assert f"{package}/{name}" in entries + assert not any("__pycache__" in n or "tests" in Path(n).parts for n in entries) + assert not any(n.endswith(".j2") for n in entries) result = build(project, out, "sdist", env) assert result.returncode == 0, result.stdout - extracted = tmp_path / "extracted" with tarfile.open(next(out.glob("*.tar.gz"))) as archive: - names = archive.getnames() - assert len(names) == len(set(names)) - assert not any("/native/" in n or n.endswith((".so", ".dylib", ".dll", ".cpp")) for n in names) - for name in ("stage_runtime.py", "prepare_backend.py", "_build_backend.py", "LICENSE", "clap_cpp/clap.pyi", "clap_cpp/py.typed"): - assert any(n.endswith("/" + name) for n in names) - archive.extractall(extracted, filter="data") - source = next(extracted.iterdir()) - result = build(source, tmp_path / "from-sdist", "wheel", env) - assert result.returncode != 0 and "stage_runtime.py" in result.stdout - load("sdist_stage", source / "stage_runtime.py").stage(source, runtime.parent) + entries = {n.split("/", 1)[1] for n in archive.getnames() if "/" in n} + required = {"pyproject.toml", "README.md", "LICENSE", + *(f"{package}/{name}" for name in ("__init__.py", "clap.py", "clap.pyi", "py.typed"))} + assert required <= entries + assert not any(n.startswith("native/") or "licenses/" in n or "companions/" in n or + n.endswith((".so", ".dll", ".dylib", ".cpp")) for n in entries) + assert not any("__pycache__" in n or n.endswith(".pyc") for n in entries) + assert "unlisted.txt" not in entries + assert not any("tests" in Path(n).parts for n in entries) + assert not any(n.endswith(".j2") for n in entries) + assert not {"stage_runtime.py", "prepare_backend.py", "_build_backend.py", "hatch_build.py"} & entries + archive.extractall(tmp_path / "extracted", filter="data") + source = next((tmp_path / "extracted").iterdir()) + shutil.copytree(project / package, source / package, dirs_exist_ok=True) result = build(source, tmp_path / "from-sdist", "wheel", env) assert result.returncode == 0, result.stdout - load("sdist_backend", source / "prepare_backend.py").prepare("cuda", tmp_path / "cuda") + if backend == "cpu": + assert not (source / "pyproject.toml.j2").exists() + gpu, gpu_package = render(source, tmp_path / "cuda-from-cpu-sdist", "cuda", template_source=PROJECT) + for name in ("clap.py", "clap.pyi", "py.typed"): + assert (gpu / gpu_package / name).read_bytes() == (source / "clap_cpp" / name).read_bytes() assert not marker.exists() + + +def test_editable_is_platform_tagged_without_compilation(tmp_path): + project, _ = render(PROJECT, tmp_path / "project") + result = subprocess.run( + [sys.executable, "-c", "from scikit_build_core.build import build_editable; build_editable('editable')"], + cwd=project, capture_output=True, text=True) + assert result.returncode == 0, result.stdout + result.stderr + wheel = next((project / "editable").glob("*.whl")) + assert "-py3-none-" in wheel.name and not wheel.name.endswith("-any.whl") + + +def test_macos_native_tag_and_minimum(monkeypatch): + from scikit_build_core.builder import wheel_tag + import sysconfig + sysconfig.get_config_vars() # Initialize real host configuration before mocking Darwin. + monkeypatch.setattr(wheel_tag.sys, "platform", "darwin") + # packaging uses the executing architecture, not a universal2 interpreter tag. + monkeypatch.setattr(wheel_tag.packaging.tags.platform, "mac_ver", lambda: ("15.0", ("", "", ""), "arm64")) + tag = wheel_tag.WheelTag.compute_best([], "py3", env={"MACOSX_DEPLOYMENT_TARGET": "11.0"}) + assert str(tag) == "py3-none-macosx_11_0_arm64"