From f211e0efc63dfafd10c44d205986c3d10332bdba Mon Sep 17 00:00:00 2001 From: zhenghaoz Date: Sat, 7 Nov 2026 02:08:56 +0800 Subject: [PATCH 1/4] refactor(python): package external runtimes with Hatchling and Jinja --- .github/RELEASING.md | 18 +- .github/scripts/test_python_release_assets.py | 18 +- .github/workflows/ci.yml | 5 +- .github/workflows/release.yml | 68 ++-- python/README.md | 110 ++++--- python/_build_backend.py | 66 ---- python/hatch_build.py | 36 +++ python/prepare_backend.py | 72 ----- python/pyproject.toml | 34 +- python/pyproject.toml.j2 | 42 +++ python/stage_runtime.py | 100 ------ python/tests/test_packaging.py | 303 ++++++++---------- 12 files changed, 373 insertions(+), 499 deletions(-) delete mode 100644 python/_build_backend.py create mode 100644 python/hatch_build.py delete mode 100644 python/prepare_backend.py create mode 100644 python/pyproject.toml.j2 delete mode 100644 python/stage_runtime.py diff --git a/.github/RELEASING.md b/.github/RELEASING.md index 3452ff4..82490b7 100644 --- a/.github/RELEASING.md +++ b/.github/RELEASING.md @@ -32,15 +32,18 @@ validates archive contents, platform constraints and versions before publishing. Node inference tests run against the checkout and its platform workspace in CI. No Python wheel extraction is involved. All native packages publish before the wrapper. -Python publishes canonical wrapper-only source archives including `stage_runtime.py`; -they contain neither native sources nor compiled libraries. CPU wheel jobs extract -the canonical base archive, download the matching runtime and stage it with -`python stage_runtime.py PROJECT_DIR RUNTIME_DIR`. cibuildwheel packages without -CMake (`wheel.cmake=false`) and runs installed-wheel tests for Python 3.9, 3.10, +Python publishes canonical wrapper-only source archives including the shared +`pyproject.toml.j2` template and thin `hatch_build.py` validation/platform-tag hook; +they contain neither native sources nor compiled libraries. Release jobs render +the template with Jinja2 for CPU, CUDA, ROCm and Vulkan, preserving exact versions. +CPU wheel jobs extract the base archive and directly copy the downloaded +`clap_cpp/` runtime contents and license into its package. Hatchling never compiles +native code; cibuildwheel runs installed-wheel tests for Python 3.9, 3.10, 3.11, 3.12, 3.13 and 3.14. ABI-independent `py3-none` wheels are reused across these interpreters; Linux wheels retain auditwheel-repaired manylinux tags. GPU jobs recursively check out native sources and compile before packaging the -canonical backend wrapper archive, staging with `--backend cuda|rocm|vulkan`. +canonical backend wrapper archive, copying the external `clap_cpp/` runtime +contents into `clap_cpp_cuda/`, `clap_cpp_rocm/` or `clap_cpp_vulkan/`. CUDA toolkit/architecture flags, ROCm targets and system-runtime exclusions, and Vulkan loader exclusions remain backend-specific. Their existing auditwheel policies remain: CUDA/ROCm manylinux_2_35, Vulkan manylinux_2_39; GPU execution is @@ -91,7 +94,8 @@ cmake --install build/libraries --config Release --component runtime --prefix "$ python -m build python --sdist --outdir sdists mkdir python-dist tar -xzf sdists/clap_cpp-*.tar.gz --strip-components=1 -C python-dist -python python-dist/stage_runtime.py python-dist libraries/linux-x64 +cp -R libraries/linux-x64/clap_cpp/. python-dist/clap_cpp/ +cp python-dist/LICENSE python-dist/clap_cpp/licenses/clap-cpp.txt python -m build python-dist --wheel --outdir wheels # Host builds above are not a substitute for the release's manylinux Docker build/repair. mkdir -p node/packages/linux-x64/runtime diff --git a/.github/scripts/test_python_release_assets.py b/.github/scripts/test_python_release_assets.py index 1f8512f..2c3940e 100644 --- a/.github/scripts/test_python_release_assets.py +++ b/.github/scripts/test_python_release_assets.py @@ -59,7 +59,8 @@ def setUp(self): def unit_sdist(self, source, distribution, omitted=None, extra=()): """Write wrapper-only UNIT FIXTURE source archives for the real heredoc.""" with tarfile.open(source, "w:gz") as archive: - for name in ("pyproject.toml", "stage_runtime.py", f"{distribution}/clap.py", *extra): + for name in ("pyproject.toml", "pyproject.toml.j2", "hatch_build.py", "README.md", "LICENSE", + *(f"{distribution}/{name}" for name in ("__init__.py", "clap.py", "clap.pyi", "py.typed")), *extra): if name == omitted: continue data = b"# UNIT FIXTURE wrapper packaging input\n" @@ -68,13 +69,18 @@ def unit_sdist(self, source, distribution, omitted=None, extra=()): archive.addfile(entry, io.BytesIO(data)) def test_unit_fixture_sdist_requires_wrapper_metadata_and_helper(self): - for required in ("pyproject.toml", "stage_runtime.py", "clap_cpp/clap.py"): - with self.subTest(required=required): - self.unit_sdist(self.artifacts / "clap_cpp-0.1.0.tar.gz", "clap_cpp", omitted=required) - self.assert_rejected("Sdist missing wrapper or staging helper") + for distribution in DISTRIBUTIONS: + for required in ("pyproject.toml", "pyproject.toml.j2", "hatch_build.py", "README.md", "LICENSE", + *(f"{distribution}/{name}" for name in ("__init__.py", "clap.py", "clap.pyi", "py.typed"))): + with self.subTest(distribution=distribution, required=required): + source = self.artifacts / f"{distribution}-0.1.0.tar.gz" + self.unit_sdist(source, distribution, omitted=required) + self.assert_rejected("Sdist missing wrapper or packaging input") + self.unit_sdist(source, distribution) def test_unit_fixture_sdist_rejects_native_graph_and_binaries(self): - for native in ("native/ggml/CMakeLists.txt", "clap_cpp/libclap.so", "clap_cpp/clap.dll", "clap_cpp/libclap.dylib"): + for native in ("native/ggml/CMakeLists.txt", "CMakeLists.txt", "cmake/helpers.cmake", "src/clap.cpp", + "clap_cpp/libclap.so", "clap_cpp/clap.dll", "clap_cpp/libclap.dylib"): with self.subTest(native=native): self.unit_sdist(self.artifacts / "clap_cpp-0.1.0.tar.gz", "clap_cpp", extra=(native,)) self.assert_rejected("Sdist contains native sources or compiled runtime") diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index f7bf9af..38e3251 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -92,7 +92,7 @@ jobs: run: cmake --install build/libraries --config Release --prefix "${{ runner.temp }}/clap-install" - name: Install Python packaging and test dependencies if: matrix.os != 'windows-11-arm' - run: python -m pip install build "scikit-build-core>=1.1.2" packaging pytest "mypy>=1.8" "typing_extensions>=4.2" + run: python -m pip install build "hatchling>=1.27" jinja2 packaging pytest "mypy>=1.8" "typing_extensions>=4.2" - name: Package Python with the shared runtime if: matrix.os != 'windows-11-arm' shell: bash @@ -100,7 +100,8 @@ jobs: python -m build python --sdist --outdir python-dist mkdir -p build/python-package tar -xzf python-dist/*.tar.gz --strip-components=1 -C build/python-package - python python/stage_runtime.py build/python-package "libraries/${{ matrix.runtime }}" + cp -R "libraries/${{ matrix.runtime }}/clap_cpp/." build/python-package/clap_cpp/ + cp build/python-package/LICENSE build/python-package/clap_cpp/licenses/clap-cpp.txt python -m build build/python-package --wheel --outdir python-dist python -m pip install --no-deps --no-index --find-links python-dist clap-cpp - name: Test Python diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 1167b1d..3e02ac5 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -20,7 +20,7 @@ jobs: with: python-version: '3.11' - name: Install packaging tools - run: python -m pip install build packaging twine + run: python -m pip install build jinja2 packaging twine - name: Set Python package version from release tag env: RELEASE_TAG: ${{ github.event.release.tag_name }} @@ -28,21 +28,15 @@ jobs: python - <<'PY' import os from pathlib import Path - import tomllib + from jinja2 import Environment, StrictUndefined from packaging.version import Version version = str(Version(os.environ['RELEASE_TAG'])) - path = Path('python/pyproject.toml') - text = path.read_text() - current = tomllib.loads(text)['project']['version'] - old = f'version = "{current}"' - assert text.count(old) == 1, 'Expected exactly one package version' - text = text.replace(old, f'version = "{version}"') - for backend in ('cuda', 'rocm', 'vulkan'): - old_dependency = f'clap-cpp-{backend}=={current}' - assert text.count(old_dependency) == 1, f'Expected one exact {backend} dependency' - text = text.replace(old_dependency, f'clap-cpp-{backend}=={version}') - path.write_text(text) + source = Path('python') + template = Environment(undefined=StrictUndefined, keep_trailing_newline=True).from_string( + (source / 'pyproject.toml.j2').read_text(encoding='utf-8')) + (source / 'pyproject.toml').write_text( + template.render(backend='cpu', version=version), encoding='utf-8') PY - name: Build source distribution run: python -m build python --sdist --outdir dist @@ -54,7 +48,23 @@ jobs: tar -xzf dist/clap_cpp-[0-9]*.tar.gz --strip-components=1 -C python-dist for backend in cuda rocm vulkan; do destination="$RUNNER_TEMP/clap-$backend" - python python-dist/prepare_backend.py "$backend" "$destination" + package="clap_cpp_$backend" + mkdir -p "$destination/$package" + cp python-dist/{README.md,LICENSE,hatch_build.py,pyproject.toml.j2} "$destination/" + cp python-dist/clap_cpp/{clap.py,clap.pyi,py.typed} "$destination/$package/" + printf '"""Native %s runtime; import clap_cpp for the public API."""\n' "$backend" > "$destination/$package/__init__.py" + BACKEND="$backend" DESTINATION="$destination" python - <<'PY' + import os + from pathlib import Path + import tomllib + from jinja2 import Environment, StrictUndefined + source = Path('python-dist') + version = tomllib.loads((source / 'pyproject.toml').read_text(encoding='utf-8'))['project']['version'] + template = Environment(undefined=StrictUndefined, keep_trailing_newline=True).from_string( + (source / 'pyproject.toml.j2').read_text(encoding='utf-8')) + (Path(os.environ['DESTINATION']) / 'pyproject.toml').write_text( + template.render(backend=os.environ['BACKEND'], version=version), encoding='utf-8') + PY python -m build --sdist "$destination" --outdir dist done - name: Check source distribution @@ -147,6 +157,11 @@ jobs: with: name: libraries-${{ matrix.target }} path: python-dist/external-runtime + - name: Copy external runtime into the CPU wrapper project + shell: bash + run: | + cp -R python-dist/external-runtime/clap_cpp/. python-dist/clap_cpp/ + cp python-dist/LICENSE python-dist/clap_cpp/licenses/clap-cpp.txt - name: Package and test CPU wheels from canonical wrapper sdist and shared runtime uses: pypa/cibuildwheel@v4.3.4 with: @@ -159,9 +174,7 @@ jobs: CIBW_MANYLINUX_X86_64_IMAGE: manylinux_2_28 CIBW_MANYLINUX_AARCH64_IMAGE: manylinux_2_28 CIBW_ENVIRONMENT_MACOS: MACOSX_DEPLOYMENT_TARGET=11.0 - # before-build only copies runtime files; wheel.cmake=false in the sdist. - CIBW_BEFORE_BUILD: python {package}/stage_runtime.py {package} {package}/external-runtime - CIBW_TEST_REQUIRES: pytest scikit-build-core>=1.1.2 mypy>=1.8 typing_extensions>=4.2 + CIBW_TEST_REQUIRES: pytest build hatchling>=1.27 jinja2 packaging tomli mypy>=1.8 typing_extensions>=4.2 CIBW_TEST_COMMAND: python -m pytest {package}/tests -v - uses: actions/upload-artifact@v4 with: @@ -236,7 +249,8 @@ jobs: - name: Stage runtime and package canonical CUDA wrapper sdist run: | set -euo pipefail - python "$RUNNER_TEMP/clap-cuda/stage_runtime.py" "$RUNNER_TEMP/clap-cuda" "libraries/cuda" --backend cuda + cp -R "libraries/cuda/clap_cpp/." "$RUNNER_TEMP/clap-cuda/clap_cpp_cuda/" + cp "$RUNNER_TEMP/clap-cuda/LICENSE" "$RUNNER_TEMP/clap-cuda/clap_cpp_cuda/licenses/clap-cpp.txt" python -m build --wheel "$RUNNER_TEMP/clap-cuda" --outdir raw - name: Repair CUDA wheel run: | @@ -316,7 +330,8 @@ jobs: - name: Stage runtime and package canonical VULKAN wrapper sdist run: | set -euo pipefail - python "$RUNNER_TEMP/clap-vulkan/stage_runtime.py" "$RUNNER_TEMP/clap-vulkan" "libraries/vulkan" --backend vulkan + cp -R "libraries/vulkan/clap_cpp/." "$RUNNER_TEMP/clap-vulkan/clap_cpp_vulkan/" + cp "$RUNNER_TEMP/clap-vulkan/LICENSE" "$RUNNER_TEMP/clap-vulkan/clap_cpp_vulkan/licenses/clap-cpp.txt" python -m build --wheel "$RUNNER_TEMP/clap-vulkan" --outdir raw - name: Repair Vulkan wheel (Ubuntu 24.04 glibc floor) run: | @@ -414,7 +429,8 @@ jobs: - name: Stage runtime and package canonical ROCM wrapper sdist run: | set -euo pipefail - python "$RUNNER_TEMP/clap-rocm/stage_runtime.py" "$RUNNER_TEMP/clap-rocm" "libraries/rocm" --backend rocm + cp -R "libraries/rocm/clap_cpp/." "$RUNNER_TEMP/clap-rocm/clap_cpp_rocm/" + cp "$RUNNER_TEMP/clap-rocm/LICENSE" "$RUNNER_TEMP/clap-rocm/clap_cpp_rocm/licenses/clap-cpp.txt" python -m build --wheel "$RUNNER_TEMP/clap-rocm" --outdir raw - name: Repair ROCm wheel, retaining system ROCm runtime run: | @@ -540,10 +556,14 @@ jobs: with tarfile.open(source) as archive: entries = {entry.name.split('/', 1)[1] for entry in archive if '/' in entry.name and entry.isfile()} package = name.replace('-', '_') - required = {'pyproject.toml', 'stage_runtime.py', f'{package}/clap.py'} + required = {'pyproject.toml', 'pyproject.toml.j2', 'hatch_build.py', 'README.md', 'LICENSE', + f'{package}/__init__.py', f'{package}/clap.py', f'{package}/clap.pyi', f'{package}/py.typed'} if not required <= entries: - raise SystemExit(f'Sdist missing wrapper or staging helper: {source.name}: {required - entries}') - if any(entry.startswith('native/') or entry.endswith(('.so', '.dylib', '.dll', '.pyd')) for entry in entries): + raise SystemExit(f'Sdist missing wrapper or packaging input: {source.name}: {required - entries}') + if any(entry.startswith(('native/', 'cmake/')) or + pathlib.PurePosixPath(entry).name == 'CMakeLists.txt' or + entry.endswith(('.so', '.dylib', '.dll', '.pyd', '.cpp', '.c', '.h')) + for entry in entries): raise SystemExit(f'Sdist contains native sources or compiled runtime: {source.name}') if (dist / source.name).exists(): raise SystemExit(f'Duplicate release asset: {source.name}') @@ -574,7 +594,7 @@ jobs: ROCm GPU targets: gfx908,gfx90a,gfx942,gfx1030,gfx1100,gfx1101,gfx1200,gfx1201. Linux wheels are auditwheel-repaired manylinux wheels; the external GPU runtimes above remain deliberate exceptions, not portability promises. - Source archives contain wrappers and stage_runtime.py, not native sources. + Source archives contain wrappers, a Jinja metadata template and a validation/tag hook, not native sources. Compile native runtimes separately from a recursive checkout, then stage them before packaging; Python wheel builds do not run CMake. CPU wheels pass installed-wheel tests on Python 3.9-3.14; GPU execution is not tested here. diff --git a/python/README.md b/python/README.md index 03ff75e..fc90e16 100644 --- a/python/README.md +++ b/python/README.md @@ -101,52 +101,85 @@ libraries/linux-x64/ stage it into the wrapper project and build a platform wheel: ```sh -python -m pip install build 'scikit-build-core>=1.1.2' -python python/stage_runtime.py python libraries/linux-x64 +python -m pip install build 'hatchling>=1.27' +cp -R libraries/linux-x64/clap_cpp/. python/clap_cpp/ +cp python/LICENSE python/clap_cpp/licenses/clap-cpp.txt python -m build --wheel python python -m pip install python/dist/clap_cpp-*.whl ``` -`stage_runtime.py PROJECT_DIR RUNTIME_DIR [--backend cuda|rocm|vulkan]` -copies `RUNTIME_DIR/clap_cpp/` contents into the project's Python namespace, -preserving relative paths **within that namespace**. For example, -`clap_cpp/companions/libfoo.so` becomes `clap_cpp_cuda/companions/libfoo.so` for -`--backend cuda`. The common runtime artifact must contain only the `clap_cpp/` -namespace. Sibling payloads such as `clap_cpp.libs/` or a sibling `licenses/` -directory are rejected before any files are copied: moving them beneath the -package would break loader paths such as `$ORIGIN/../clap_cpp.libs`. Such artifacts -need separate loader-aware wheel packaging; the staging helper does not silently -relocate them. Keep native dependency notices inside `clap_cpp/licenses/`. -It adds the project's root MIT license as `licenses/clap-cpp.txt`, preserves -package-local native dependency notices, rejects missing runtimes and wrapper-source collisions, -and replaces an earlier staged payload without retaining stale dependencies. -Only stage trusted runtime artifacts matching the wheel's target platform; the -helper does not cross-compile, inspect binary architecture or repair Linux tags. -The native runtime's own loader paths must resolve any companion libraries. - -The backend guards wheel construction, failing with an actionable staging command -if the host-platform runtime or staged license is missing. It delegates packaging -to scikit-build-core with `wheel.cmake=false`, `wheel.platlib=true` and -`wheel.py-api="py3"`. A local Linux build is tagged `py3-none-linux_`; -release wheels still need normal platform portability checks/repair. Metadata -preparation and sdist creation do not require a native runtime. +Copy only trusted, matching-platform runtime artifacts. Preserve all relative +paths and dependency notices beneath `clap_cpp/`; sibling library directories +need separate loader-aware wheel repair, not relocation into the package. +Start from a fresh wrapper project when switching runtimes so stale companion +libraries cannot leak into the next wheel. Do not copy Python sources from a +runtime artifact or overwrite the wrapper files. + +Hatchling only assembles the copied files. Its thin `hatch_build.py` hook rejects +missing/empty host-platform libraries or `licenses/clap-cpp.txt`, and sets +non-pure `py3-none` platform tags. It never stages files, renders metadata or +compiles code. A local Linux wheel has a `linux_` tag and still needs +portability checks/repair before release. Metadata and wrapper-only source +archives do not require a runtime. Editable installs also require a copied runtime; +Hatchling's temporary editable wheel references the source directory and is not +a distributable runtime wheel. + +On macOS, the wheel tag must match the staged runtime's architecture and minimum +macOS deployment target, not just the Python interpreter's build configuration. +A universal2 interpreter can report a universal2 platform even when the copied +runtime is single-architecture. The hook rejects all universal2 tags, including +explicit overrides; it does not verify native binary architectures. Set an +explicit single-architecture `_PYTHON_HOST_PLATFORM` matching the staged runtime +when building with such an interpreter. Supported macOS releases are ARM64; for +an ARM64 runtime built with deployment target 11.0, use: + +```sh +_PYTHON_HOST_PLATFORM=macosx-11.0-arm64 python -m build --wheel python +``` + +This only sets packaging metadata; it does not convert the runtime or change its +deployment target. The release cibuildwheel ARM64 override follows the same rule. ### Optional backend projects -Generate a wrapper-only project, then stage the independently compiled backend -runtime (the native artifact still uses its common `clap_cpp/` namespace): +`pyproject.toml.j2` is the shared CPU/CUDA/ROCm/Vulkan metadata template. The +checked-in `pyproject.toml` is its CPU rendering for ordinary local builds. +Render with `backend=cpu|cuda|rocm|vulkan` and one PEP 440 `version` (default +`0.1.0`); CPU extras pin that exact version of each backend distribution. +For example, using the canonical CPU sdist extracted into `python-dist`: ```sh -python python/prepare_backend.py cuda build/python-cuda -python python/stage_runtime.py build/python-cuda libraries/linux-x64-cuda --backend cuda -python -m build --wheel build/python-cuda +python -m pip install jinja2 +backend=cuda # or rocm, vulkan +project="build/python-$backend" +package="clap_cpp_$backend" +mkdir -p "$project/$package" +cp python-dist/{README.md,LICENSE,hatch_build.py,pyproject.toml.j2} "$project/" +cp python-dist/clap_cpp/{clap.py,clap.pyi,py.typed} "$project/$package/" +printf '"""Native %s runtime; import clap_cpp for the public API."""\n' "$backend" > "$project/$package/__init__.py" +BACKEND="$backend" PROJECT="$project" python - <<'PY' +import os +from pathlib import Path +from jinja2 import Environment, StrictUndefined +try: + import tomllib +except ImportError: + import tomli as tomllib # install tomli on Python 3.9/3.10 +source = Path('python-dist') +version = tomllib.loads((source / 'pyproject.toml').read_text())['project']['version'] +template = Environment(undefined=StrictUndefined, keep_trailing_newline=True).from_string( + (source / 'pyproject.toml.j2').read_text()) +(Path(os.environ['PROJECT']) / 'pyproject.toml').write_text( + template.render(backend=os.environ['BACKEND'], version=version)) +PY +cp -R "libraries/linux-x64-$backend/clap_cpp/." "$project/$package/" +cp "$project/LICENSE" "$project/$package/licenses/clap-cpp.txt" +python -m build --wheel "$project" ``` -Replace `cuda` with `rocm` or `vulkan`. The helper preserves the base version, -wrapper and typing files; no native sources or CMake settings are copied. The -staging helper and build guard are included in each backend source distribution. -The public selector continues to use `clap_cpp_cuda`, `clap_cpp_rocm` and -`clap_cpp_vulkan` without changing the base package or its public APIs. +The release workflow performs these copies and Jinja renders directly; no +project-generation or staging script is needed. The selector still uses +`clap_cpp_cuda`, `clap_cpp_rocm` and `clap_cpp_vulkan` with unchanged public APIs. ## Development and source distributions @@ -154,10 +187,11 @@ The public selector continues to use `clap_cpp_cuda`, `clap_cpp_rocm` and python -m build --sdist python # Unpack the archive, then explicitly stage its external runtime: tar -xf python/dist/clap_cpp-0.1.0.tar.gz -C build -python build/clap_cpp-0.1.0/stage_runtime.py build/clap_cpp-0.1.0 libraries/linux-x64 +cp -R libraries/linux-x64/clap_cpp/. build/clap_cpp-0.1.0/clap_cpp/ +cp build/clap_cpp-0.1.0/LICENSE build/clap_cpp-0.1.0/clap_cpp/licenses/clap-cpp.txt python -m build --wheel build/clap_cpp-0.1.0 python -m pip install 'build/clap_cpp-0.1.0/dist/'*.whl -python -m pip install pytest mypy typing_extensions build 'scikit-build-core>=1.1.2' +python -m pip install pytest mypy typing_extensions build 'hatchling>=1.27' jinja2 python -m pytest python/tests ``` @@ -181,4 +215,4 @@ trusted publisher (owner `gorse-io`, repository `clap.cpp`, workflow `release.ym environment `pypi`) for **each** of `clap-cpp`, `clap-cpp-cuda`, `clap-cpp-rocm` and `clap-cpp-vulkan`, and any required GitHub `pypi` environment approvals. Prereleases only attach assets. Normalize the release version and the three exact -extra pins together; `prepare_backend.py` copies that version to each backend. +extra pins together by rendering the shared Jinja template for each backend. diff --git a/python/_build_backend.py b/python/_build_backend.py deleted file mode 100644 index ae4175e..0000000 --- a/python/_build_backend.py +++ /dev/null @@ -1,66 +0,0 @@ -"""Thin scikit-build-core guard: a wheel must have a staged native runtime. - -Metadata and source distributions need no runtime. Wheel/editable construction -checks the package-only contract before delegating; no native build fallback. -""" - -from pathlib import Path -import sys - -from scikit_build_core import build as _build -from scikit_build_core.settings.skbuild_read_settings import SettingsReader - -from stage_runtime import RUNTIME_NAMES - - -# Reading the project here keeps CPU and generated backend projects identical. -try: - import tomllib -except ImportError: - import tomli as tomllib - - -def _guard(config_settings, state): - with Path("pyproject.toml").open("rb") as stream: - metadata = tomllib.load(stream) - reader = SettingsReader(metadata, config_settings or {}, state=state) - reader.validate_may_exit() - settings = reader.settings - if settings.wheel.cmake or not settings.wheel.platlib or settings.wheel.py_api != "py3": - raise RuntimeError("Python wheels require wheel.cmake=false, wheel.platlib=true, wheel.py-api=py3") - packages = settings.wheel.packages - expected = metadata["project"]["name"].replace("-", "_") - if packages != [expected]: - raise RuntimeError(f"Expected wheel.packages=[{expected!r}] for this runtime project") - package = Path(expected) - native = "clap.dll" if sys.platform == "win32" else ( - "libclap.dylib" if sys.platform == "darwin" else "libclap.so" - ) - if native not in RUNTIME_NAMES or not (package / native).is_file() or (package / native).stat().st_size == 0: - backend = expected.removeprefix("clap_cpp_") if expected != "clap_cpp" else None - option = f" --backend {backend}" if backend else "" - raise RuntimeError( - f"Missing staged external runtime: {package / native}. " - f"Run: python stage_runtime.py . /path/to/libraries/TARGET{option}. " - "Python packaging does not compile native libraries." - ) - if not (package / "licenses" / "clap-cpp.txt").is_file(): - raise RuntimeError("Missing runtime notices; run stage_runtime.py before building a wheel") - - -def build_wheel(wheel_directory, config_settings=None, metadata_directory=None): - _guard(config_settings, "wheel") - return _build.build_wheel(wheel_directory, config_settings, metadata_directory) - - -def build_editable(wheel_directory, config_settings=None, metadata_directory=None): - _guard(config_settings, "editable") - return _build.build_editable(wheel_directory, config_settings, metadata_directory) - - -build_sdist = _build.build_sdist -get_requires_for_build_sdist = _build.get_requires_for_build_sdist -get_requires_for_build_wheel = _build.get_requires_for_build_wheel -get_requires_for_build_editable = _build.get_requires_for_build_editable -prepare_metadata_for_build_wheel = _build.prepare_metadata_for_build_wheel -prepare_metadata_for_build_editable = _build.prepare_metadata_for_build_editable diff --git a/python/hatch_build.py b/python/hatch_build.py new file mode 100644 index 0000000..ef3032e --- /dev/null +++ b/python/hatch_build.py @@ -0,0 +1,36 @@ +"""Validate an externally copied runtime and mark its wheel as platform-specific.""" + +from pathlib import Path +import sys +import sysconfig + +from hatchling.builders.hooks.plugin.interface import BuildHookInterface + + +class CustomBuildHook(BuildHookInterface): + def initialize(self, version, build_data): + package = Path(self.root) / self.metadata.name.replace("-", "_") + native = "clap.dll" if sys.platform == "win32" else ( + "libclap.dylib" if sys.platform == "darwin" else "libclap.so" + ) + runtime = package / native + if not runtime.is_file() or runtime.stat().st_size == 0: + raise RuntimeError( + f"Missing staged external runtime: {runtime}. " + "Copy the externally compiled runtime's clap_cpp/ contents into " + f"{package.name}/ before building a wheel. " + "Python packaging does not compile native libraries." + ) + notice = package / "licenses" / "clap-cpp.txt" + if not notice.is_file() or notice.stat().st_size == 0: + raise RuntimeError("Missing runtime notices; copy LICENSE to " + str(notice)) + platform = sysconfig.get_platform().replace("-", "_").replace(".", "_") + if sys.platform == "darwin" and "universal2" in platform: + raise RuntimeError( + "Cannot tag an unverified external runtime as universal2. " + "Set _PYTHON_HOST_PLATFORM to a single-architecture macOS platform " + "matching the staged runtime's architecture and deployment target " + "(for example, macosx-11.0-arm64)." + ) + build_data["pure_python"] = False + build_data["tag"] = f"py3-none-{platform}" diff --git a/python/prepare_backend.py b/python/prepare_backend.py deleted file mode 100644 index 77b58b3..0000000 --- a/python/prepare_backend.py +++ /dev/null @@ -1,72 +0,0 @@ -"""Create a wrapper-only GPU wheel project from a checkout or base sdist. - -Stage its externally compiled runtime later using stage_runtime.py --backend. -""" - -import argparse -from pathlib import Path -import re -import shutil - - -BACKENDS = ("cuda", "rocm", "vulkan") - - -def prepare(backend, destination): - if backend not in BACKENDS: - raise ValueError(f"Unknown backend: {backend}") - destination = Path(destination) - source = Path(__file__).resolve().parent - metadata = (source / "pyproject.toml").read_text(encoding="utf-8") - match = re.search(r'^version = "([^"]+)"$', metadata, re.MULTILINE) - if match is None: - raise ValueError("Missing package version") - version = match.group(1) - package = f"clap_cpp_{backend}" - destination.mkdir(parents=True, exist_ok=False) - for name in ("README.md", "LICENSE", "stage_runtime.py", "_build_backend.py"): - shutil.copy2(source / name, destination / name) - (destination / package).mkdir() - for name in ("clap.py", "clap.pyi", "py.typed"): - shutil.copy2(source / "clap_cpp" / name, destination / package / name) - (destination / package / "__init__.py").write_text( - f'"""Native {backend} runtime; import clap_cpp for the public API."""\n', encoding="utf-8" - ) - (destination / "pyproject.toml").write_text( - f'''[build-system] -requires = ["scikit-build-core>=1.1.2"] -build-backend = "_build_backend" -backend-path = ["."] - -[project] -name = "clap-cpp-{backend}" -version = "{version}" -description = "{backend.upper()} backend for clap-cpp" -readme = "README.md" -requires-python = ">=3.9" -license = "MIT" -license-files = ["LICENSE"] -dependencies = [] - -[tool.scikit-build] -minimum-version = "build-system.requires" -wheel.packages = ["{package}"] -wheel.force-include = {{ {package} = "{package}" }} -wheel.cmake = false -wheel.platlib = true -wheel.py-api = "py3" -sdist.cmake = false -sdist.inclusion-mode = "explicit" -sdist.include = ["pyproject.toml", "README.md", "LICENSE", "_build_backend.py", "stage_runtime.py", "{package}/*.py", "{package}/*.pyi", "{package}/py.typed"] -sdist.exclude = ["**/__pycache__/**"] -''', encoding="utf-8" - ) - print(f"Prepared clap-cpp-{backend} {version} in {destination}") - - -if __name__ == "__main__": - parser = argparse.ArgumentParser(description=__doc__) - parser.add_argument("backend", choices=BACKENDS) - parser.add_argument("destination", type=Path) - args = parser.parse_args() - prepare(args.backend, args.destination.resolve()) diff --git a/python/pyproject.toml b/python/pyproject.toml index e3297e8..4ea3122 100644 --- a/python/pyproject.toml +++ b/python/pyproject.toml @@ -1,7 +1,7 @@ +# Generated from pyproject.toml.j2; render for cpu, cuda, rocm or vulkan. [build-system] -requires = ["scikit-build-core>=1.1.2"] -build-backend = "_build_backend" -backend-path = ["."] +requires = ["hatchling>=1.27"] +build-backend = "hatchling.build" [project] name = "clap-cpp" @@ -19,20 +19,20 @@ cuda = ["clap-cpp-cuda==0.1.0"] rocm = ["clap-cpp-rocm==0.1.0"] vulkan = ["clap-cpp-vulkan==0.1.0"] -[tool.scikit-build] -minimum-version = "build-system.requires" -wheel.packages = ["clap_cpp"] -# Wheel payload includes the staged runtime despite the wrapper-only sdist list. -wheel.force-include = { clap_cpp = "clap_cpp" } -# Package an externally built C ABI runtime, never configure or compile it here. -wheel.cmake = false -wheel.platlib = true -wheel.py-api = "py3" -sdist.cmake = false -# Explicit wrapper-only inventory also excludes staged binaries and their notices. -sdist.inclusion-mode = "explicit" -sdist.include = ["pyproject.toml", "README.md", "LICENSE", "_build_backend.py", "stage_runtime.py", "prepare_backend.py", "clap_cpp/*.py", "clap_cpp/*.pyi", "clap_cpp/py.typed", "tests/**"] -sdist.exclude = ["**/__pycache__/**", "**/.pytest_cache/**"] +[tool.hatch.build] +exclude = ["**/__pycache__", "**/*.pyc", "**/.pytest_cache"] + +[tool.hatch.build.targets.wheel] +packages = ["clap_cpp"] +# The explicitly copied native payload is ignored by Git, not by the wheel. +ignore-vcs = true + +[tool.hatch.build.targets.wheel.hooks.custom] +path = "hatch_build.py" + +[tool.hatch.build.targets.sdist] +# Never ship staged binaries, companion data or runtime notices in source archives. +include = ["/pyproject.toml", "/pyproject.toml.j2", "/README.md", "/LICENSE", "/hatch_build.py", "/clap_cpp/*.py", "/clap_cpp/*.pyi", "/clap_cpp/py.typed", "/tests/**"] [tool.pytest.ini_options] testpaths = ["tests"] diff --git a/python/pyproject.toml.j2 b/python/pyproject.toml.j2 new file mode 100644 index 0000000..3e868f7 --- /dev/null +++ b/python/pyproject.toml.j2 @@ -0,0 +1,42 @@ +{% set backend = backend | default('cpu') -%} +{% set version = version | default('0.1.0') -%} +{% set package = 'clap_cpp' ~ ('_' ~ backend if backend != 'cpu' else '') -%} +# Generated from pyproject.toml.j2; render for cpu, cuda, rocm or vulkan. +[build-system] +requires = ["hatchling>=1.27"] +build-backend = "hatchling.build" + +[project] +name = "{{ package | replace('_', '-') }}" +version = "{{ version }}" +description = "{{ 'Python bindings for clap.cpp audio and text embeddings' if backend == 'cpu' else backend | upper ~ ' backend for clap-cpp' }}" +readme = "README.md" +requires-python = ">=3.9" +license = "MIT" +license-files = ["LICENSE"] +dependencies = [] +{% if backend == 'cpu' %} +[project.optional-dependencies] +test = ["pytest>=7", "mypy>=1.8", "typing_extensions>=4.2"] +cuda = ["clap-cpp-cuda=={{ version }}"] +rocm = ["clap-cpp-rocm=={{ version }}"] +vulkan = ["clap-cpp-vulkan=={{ version }}"] +{% endif %} +[tool.hatch.build] +exclude = ["**/__pycache__", "**/*.pyc", "**/.pytest_cache"] + +[tool.hatch.build.targets.wheel] +packages = ["{{ package }}"] +# The explicitly copied native payload is ignored by Git, not by the wheel. +ignore-vcs = true + +[tool.hatch.build.targets.wheel.hooks.custom] +path = "hatch_build.py" + +[tool.hatch.build.targets.sdist] +# Never ship staged binaries, companion data or runtime notices in source archives. +include = ["/pyproject.toml", "/pyproject.toml.j2", "/README.md", "/LICENSE", "/hatch_build.py", "/{{ package }}/*.py", "/{{ package }}/*.pyi", "/{{ package }}/py.typed"{% if backend == 'cpu' %}, "/tests/**"{% endif %}] +{% if backend == 'cpu' %} +[tool.pytest.ini_options] +testpaths = ["tests"] +{% endif %} \ No newline at end of file diff --git a/python/stage_runtime.py b/python/stage_runtime.py deleted file mode 100644 index 3da4759..0000000 --- a/python/stage_runtime.py +++ /dev/null @@ -1,100 +0,0 @@ -"""Stage an external runtime: stage_runtime.py PROJECT_DIR RUNTIME_DIR [--backend NAME]. - -RUNTIME_DIR contains only clap_cpp/, the common native install namespace. -Its contents go into the target Python namespace without changing package-local -relative paths. Sibling payloads such as clap_cpp.libs are explicitly rejected: -relocating them would break loader paths like $ORIGIN/../clap_cpp.libs. No -compilation or native source snapshot is performed by Python packaging. -""" - -import argparse -from pathlib import Path -import shutil - - -BACKENDS = ("cuda", "rocm", "vulkan") -RUNTIME_NAMES = ("libclap.so", "libclap.dylib", "clap.dll") -WRAPPERS = ("__init__.py", "clap.py", "clap.pyi", "py.typed") - - -def stage(project_dir, runtime_dir, backend=None): - """Copy binaries/notices into an existing wrapper project; reject missing input.""" - if backend is not None and backend not in BACKENDS: - raise ValueError(f"Unknown backend: {backend}") - project = Path(project_dir).resolve() - runtime = Path(runtime_dir).resolve() - package = project / (f"clap_cpp_{backend}" if backend else "clap_cpp") - if not (project / "pyproject.toml").is_file() or not all( - (package / name).is_file() for name in WRAPPERS - ): - raise ValueError(f"Not a matching wrapper project: {package}") - source = runtime / "clap_cpp" - if not source.is_dir() or not any((source / name).is_file() for name in RUNTIME_NAMES): - raise FileNotFoundError( - f"Missing external runtime in {source}: expected " + ", ".join(RUNTIME_NAMES) - ) - siblings = sorted(path.name for path in runtime.iterdir() if path.name != "clap_cpp") - if siblings: - raise ValueError( - f"Unsupported runtime sibling payloads: {', '.join(siblings)}. " - "Only package-local native payloads beneath clap_cpp/ are accepted; " - "sibling library directories require separate loader-aware wheel packaging." - ) - if runtime == project or runtime in package.parents or package in runtime.parents: - raise ValueError("External runtime and target package must not overlap") - license_file = project / "LICENSE" - if not license_file.is_file(): - raise FileNotFoundError(f"Missing project LICENSE: {license_file}") - # Validate the complete copy before touching the destination. Never let a - # runtime artifact replace the wrapper, even accidentally. - files = {} - for origin in sorted(source.rglob("*")): - if not origin.is_file(): - continue - try: - origin.resolve().relative_to(runtime) - except ValueError: - raise ValueError(f"Runtime symlink escapes runtime directory: {origin}") from None - relative = origin.relative_to(source) - if relative.parts[0] in WRAPPERS or origin.suffix in (".py", ".pyi", ".pyc"): - raise ValueError(f"External runtime must not contain Python wrapper sources: {origin}") - if relative in files: - raise ValueError(f"Runtime paths collide after namespace mapping: {relative}") - target = package / relative - if not target.resolve().is_relative_to(package): - raise ValueError(f"Runtime target escapes package: {target}") - files[relative] = origin - # Restaging replaces the old native payload, so switching runtimes cannot - # leave stale dependencies/notices in the next wheel. Keep wrapper files. - for existing in package.iterdir(): - if existing.name in WRAPPERS: - continue - if existing.is_dir() and not existing.is_symlink(): - shutil.rmtree(existing) - else: - existing.unlink() - for relative, origin in files.items(): - target = package / relative - target.parent.mkdir(parents=True, exist_ok=True) - shutil.copy2(origin, target) - notices = package / "licenses" - notices.mkdir(exist_ok=True) - shutil.copy2(license_file, notices / "clap-cpp.txt") - return package - - -def main(): - parser = argparse.ArgumentParser(description=__doc__) - parser.add_argument("project_dir", type=Path) - parser.add_argument("runtime_dir", type=Path) - parser.add_argument("--backend", choices=BACKENDS) - args = parser.parse_args() - try: - package = stage(args.project_dir, args.runtime_dir, args.backend) - except (ValueError, OSError) as exc: - parser.exit(1, f"{exc}\n") - print(f"Staged external runtime in {package}") - - -if __name__ == "__main__": - main() diff --git a/python/tests/test_packaging.py b/python/tests/test_packaging.py index 2e0565f..acba722 100644 --- a/python/tests/test_packaging.py +++ b/python/tests/test_packaging.py @@ -1,14 +1,16 @@ -"""Package-only wheels consume an explicitly staged external C API runtime.""" +"""Package-only Hatchling wheels consume an explicitly copied external runtime.""" -import importlib.util import os from pathlib import Path +import runpy import shutil import subprocess import sys import tarfile +from types import SimpleNamespace import zipfile +from jinja2 import Environment, StrictUndefined import pytest try: @@ -27,19 +29,24 @@ def metadata(path=PROJECT): return tomllib.loads((path / "pyproject.toml").read_text(encoding="utf-8")) -def load(name, path): - spec = importlib.util.spec_from_file_location(name, path) - assert spec is not None and spec.loader is not None - module = importlib.util.module_from_spec(spec) - spec.loader.exec_module(module) - return module - - -def copy_project(destination): - shutil.copytree(PROJECT, destination, ignore=shutil.ignore_patterns( - "__pycache__", ".pytest_cache", "dist", "build", "*.so", "*.dylib", "*.dll", "licenses" - )) - return destination +def render(source, destination, backend="cpu", version=None): + """Mirror the workflow's Jinja rendering and plain wrapper copies.""" + destination.mkdir(parents=True) + package = "clap_cpp" + (f"_{backend}" if backend != "cpu" else "") + for name in ("README.md", "LICENSE", "hatch_build.py", "pyproject.toml.j2"): + shutil.copy2(source / name, destination / name) + shutil.copytree(source / "clap_cpp", destination / package, + ignore=shutil.ignore_patterns("__pycache__", "*.so", "*.dylib", "*.dll", "licenses")) + if backend != "cpu": + (destination / package / "__init__.py").write_text( + f'"""Native {backend} runtime; import clap_cpp for the public API."""\n', encoding="utf-8") + else: + shutil.copytree(source / "tests", destination / "tests") + template = Environment(undefined=StrictUndefined, keep_trailing_newline=True).from_string( + (source / "pyproject.toml.j2").read_text(encoding="utf-8")) + (destination / "pyproject.toml").write_text(template.render( + backend=backend, version=version or metadata(source)["project"]["version"]), encoding="utf-8") + return destination, package def build(project, out, kind, env=None): @@ -49,130 +56,66 @@ def build(project, out, kind, env=None): ) -def test_package_only_settings(tmp_path, monkeypatch): - from scikit_build_core.settings.skbuild_read_settings import SettingsReader - - monkeypatch.chdir(tmp_path) - config = metadata() - assert config["build-system"]["build-backend"] == "_build_backend" - assert config["build-system"]["backend-path"] == ["."] - assert "cmake" not in config["tool"]["scikit-build"] - assert "force-include" not in config["tool"]["scikit-build"]["sdist"] - for state in ("wheel", "editable", "sdist"): - reader = SettingsReader(config, {}, state=state) - reader.validate_may_exit() - settings = reader.settings - assert settings.wheel.cmake is False - assert settings.wheel.platlib is True - assert settings.wheel.py_api == "py3" - assert settings.sdist.cmake is False - - -@pytest.mark.parametrize("backend", ["cuda", "rocm", "vulkan"]) -def test_backend_projects_wrapper_only(tmp_path, backend): - destination = tmp_path / backend - load("generator", PROJECT / "prepare_backend.py").prepare(backend, destination) - config = metadata(destination) - assert config["project"]["name"] == f"clap-cpp-{backend}" - assert config["project"]["version"] == metadata()["project"]["version"] +@pytest.fixture +def macos_hook(tmp_path, monkeypatch): + module = runpy.run_path(str(PROJECT / "hatch_build.py")) + package = tmp_path / "clap_cpp" + (package / "licenses").mkdir(parents=True) + (package / "libclap.dylib").write_bytes(b"single-architecture packaging fixture") + (package / "licenses" / "clap-cpp.txt").write_text("Runtime notice") + hook = module["CustomBuildHook"]( + str(tmp_path), {}, None, SimpleNamespace(name="clap-cpp"), str(tmp_path), "wheel") + monkeypatch.setattr(module["sys"], "platform", "darwin") + # Model a universal2 interpreter, including sysconfig's explicit host override. + monkeypatch.setattr(module["sysconfig"], "get_platform", lambda: os.environ.get( + "_PYTHON_HOST_PLATFORM", "macosx-11.0-universal2")) + monkeypatch.delenv("_PYTHON_HOST_PLATFORM", raising=False) + return hook + + +@pytest.mark.parametrize("explicit", [False, True]) +def test_macos_universal2_tag_rejected(macos_hook, monkeypatch, explicit): + if explicit: + monkeypatch.setenv("_PYTHON_HOST_PLATFORM", "macosx-11.0-universal2") + build_data = {} + with pytest.raises(RuntimeError, match="universal2") as error: + macos_hook.initialize("standard", build_data) + assert "_PYTHON_HOST_PLATFORM" in str(error.value) + assert "single-architecture" in str(error.value) + assert "staged runtime" in str(error.value) + assert "deployment target" in str(error.value) + assert "tag" not in build_data + + +def test_macos_explicit_arm64_runtime_tag(macos_hook, monkeypatch): + monkeypatch.setenv("_PYTHON_HOST_PLATFORM", "macosx-11.0-arm64") + build_data = {} + macos_hook.initialize("standard", build_data) + assert build_data == {"pure_python": False, "tag": "py3-none-macosx_11_0_arm64"} + + +def test_template_matches_checked_in_cpu_metadata(tmp_path): + project, _ = render(PROJECT, tmp_path / "cpu") + assert (project / "pyproject.toml").read_bytes() == (PROJECT / "pyproject.toml").read_bytes() + config = metadata(project) + assert config["build-system"] == {"requires": ["hatchling>=1.27"], "build-backend": "hatchling.build"} + assert "scikit-build" not in config["tool"] + assert config["tool"]["hatch"]["build"]["targets"]["wheel"]["hooks"]["custom"]["path"] == "hatch_build.py" + + +@pytest.mark.parametrize("backend", ["cpu", "cuda", "rocm", "vulkan"]) +def test_wheel_and_wrapper_only_sdist(tmp_path, backend): + project, package = render(PROJECT, tmp_path / "project", backend, "2.3.4rc1") + config = metadata(project) + assert config["project"]["name"] == package.replace("_", "-") + assert config["project"]["version"] == "2.3.4rc1" assert config["project"]["dependencies"] == [] - settings = config["tool"]["scikit-build"] - assert settings["wheel"] == dict(packages=[f"clap_cpp_{backend}"], **{ - "py-api": "py3", "cmake": False, "platlib": True, - "force-include": {f"clap_cpp_{backend}": f"clap_cpp_{backend}"}, - }) - assert "cmake" not in settings - assert not (destination / "native").exists() - assert not list(destination.rglob("CMakeLists.txt")) - for name in ("stage_runtime.py", "_build_backend.py", "LICENSE"): - assert (destination / name).read_bytes() == (PROJECT / name).read_bytes() + if backend == "cpu": + for name in ("cuda", "rocm", "vulkan"): + assert config["project"]["optional-dependencies"][name] == [f"clap-cpp-{name}==2.3.4rc1"] for name in ("clap.py", "clap.pyi", "py.typed"): - assert (destination / f"clap_cpp_{backend}" / name).read_bytes() == (PROJECT / "clap_cpp" / name).read_bytes() - result = build(destination, tmp_path / "missing-wheel", "wheel") - assert result.returncode != 0 and "stage_runtime.py" in result.stdout - result = build(destination, tmp_path / "dist", "sdist") - assert result.returncode == 0, result.stdout - with tarfile.open(next((tmp_path / "dist").glob("*.tar.gz"))) as archive: - names = archive.getnames() - assert any(n.endswith("/stage_runtime.py") for n in names) - assert any(n.endswith("/_build_backend.py") for n in names) - assert not any("/native/" in n or n.endswith((".so", ".dll", ".dylib")) for n in names) - archive.extractall(tmp_path / "extracted", filter="data") - source = next((tmp_path / "extracted").iterdir()) - native = "clap.dll" if sys.platform == "win32" else ("libclap.dylib" if sys.platform == "darwin" else "libclap.so") - runtime = tmp_path / "runtime" / "clap_cpp" - runtime.mkdir(parents=True) - (runtime / native).write_bytes(b"backend packaging fixture, not executable") - load("backend_stage", source / "stage_runtime.py").stage(source, runtime.parent, backend) - result = build(source, tmp_path / "staged-wheel", "wheel") - assert result.returncode == 0, result.stdout - wheel = next((tmp_path / "staged-wheel").glob("*.whl")) - assert "-py3-none-" in wheel.name and not wheel.name.endswith("-any.whl") - with zipfile.ZipFile(wheel) as archive: - assert archive.read(f"clap_cpp_{backend}/{native}") == (runtime / native).read_bytes() - assert f"clap_cpp_{backend}/licenses/clap-cpp.txt" in archive.namelist() - - -@pytest.mark.parametrize("backend", [None, "cuda", "rocm", "vulkan"]) -def test_stage_runtime(tmp_path, backend): - stage = load("stage", PROJECT / "stage_runtime.py") - project = tmp_path / "project" - if backend: - load("generator", PROJECT / "prepare_backend.py").prepare(backend, project) - else: - copy_project(project) - runtime = tmp_path / "runtime" - (runtime / "clap_cpp" / "licenses").mkdir(parents=True) - (runtime / "clap_cpp" / "libclap.so").write_bytes(b"unit-test runtime, not executable") - (runtime / "clap_cpp" / "licenses" / "icu.txt").write_text("ICU notice") - (runtime / "clap_cpp" / "licenses" / "other.txt").write_text("companion notice") - (runtime / "clap_cpp" / "companions").mkdir() - (runtime / "clap_cpp" / "companions" / "libdependency.so.1").write_bytes(b"test companion") - stage.stage(project, runtime, backend=backend) - package = project / (f"clap_cpp_{backend}" if backend else "clap_cpp") - assert (package / "libclap.so").read_bytes() == (runtime / "clap_cpp" / "libclap.so").read_bytes() - assert (package / "licenses" / "icu.txt").read_text() == "ICU notice" - assert (package / "licenses" / "other.txt").read_text() == "companion notice" - assert (package / "companions" / "libdependency.so.1").exists() - assert (package / "licenses" / "clap-cpp.txt").read_bytes() == (PROJECT / "LICENSE").read_bytes() - assert (package / "clap.py").read_bytes() == (PROJECT / "clap_cpp" / "clap.py").read_bytes() - - -@pytest.mark.parametrize("sibling", ["clap_cpp.libs", "licenses"]) -def test_stage_rejects_sibling_directories_without_relocation(tmp_path, sibling): - project = copy_project(tmp_path / "project") - runtime = tmp_path / "runtime" - (runtime / "clap_cpp").mkdir(parents=True) - (runtime / "clap_cpp" / RUNTIME_NAME).write_bytes(b"packaging fixture") - (runtime / sibling).mkdir() - (runtime / sibling / "companion.so").write_bytes(b"sibling payload") - with pytest.raises(ValueError, match="sibling"): - load("stage", PROJECT / "stage_runtime.py").stage(project, runtime) - assert not (project / "clap_cpp" / RUNTIME_NAME).exists() - assert not (project / "clap_cpp" / sibling).exists() - assert not (project / sibling).exists() - - -def test_missing_runtime_and_wrapper_collision(tmp_path): - stage = load("stage", PROJECT / "stage_runtime.py") - project = copy_project(tmp_path / "project") - with pytest.raises((ValueError, FileNotFoundError), match="runtime"): - stage.stage(project, tmp_path / "absent") - runtime = tmp_path / "runtime" - (runtime / "clap_cpp").mkdir(parents=True) - with pytest.raises((ValueError, FileNotFoundError), match="runtime"): - stage.stage(project, runtime) - (runtime / "clap_cpp" / "libclap.so").write_bytes(b"test") - (runtime / "clap_cpp" / "clap.py").write_text("overwrite") - with pytest.raises(ValueError, match="wrapper"): - stage.stage(project, runtime) - assert not (project / "clap_cpp" / "libclap.so").exists() - - -def test_missing_and_staged_wheel_sdist_contract(tmp_path): - project = copy_project(tmp_path / "project") - out = tmp_path / "dist" - # Any attempt to execute CMake/Ninja fails and leaves visible evidence. + assert (project / package / name).read_bytes() == (PROJECT / "clap_cpp" / name).read_bytes() + # Assembly must not invoke a native build, including when building from sdist. tools = tmp_path / "tools" tools.mkdir() marker = tmp_path / "compiler-called" @@ -181,49 +124,75 @@ def test_missing_and_staged_wheel_sdist_contract(tmp_path): executable.write_text(f'#!/bin/sh\nprintf called > "{marker}"\nexit 91\n') executable.chmod(0o755) env = dict(os.environ, PATH=str(tools) + os.pathsep + os.environ.get("PATH", "")) + out = tmp_path / "dist" + metadata_dir = project / "metadata" + metadata_dir.mkdir() result = subprocess.run( - [sys.executable, "-c", "import _build_backend as b; " - "assert b.get_requires_for_build_wheel() == []; " - "assert b.get_requires_for_build_sdist() == []; " - "b.prepare_metadata_for_build_wheel('metadata')"], - cwd=project, env=env, text=True, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, + [sys.executable, "-c", "from hatchling.build import prepare_metadata_for_build_wheel; " + "prepare_metadata_for_build_wheel('metadata')"], + cwd=project, env=env, capture_output=True, text=True, ) - assert result.returncode == 0, result.stdout - assert list((project / "metadata").glob("*.dist-info/METADATA")) + assert result.returncode == 0, result.stdout + result.stderr + assert list(metadata_dir.glob("*.dist-info/METADATA")) result = build(project, out, "wheel", env) - assert result.returncode != 0 - assert "stage_runtime.py" in result.stdout + assert result.returncode != 0 and "Missing staged external runtime" in result.stdout, result.stdout assert not list(out.glob("*.whl")) - runtime = tmp_path / "runtime" / "clap_cpp" - runtime.mkdir(parents=True) - (runtime / RUNTIME_NAME).write_bytes(b"unit packaging payload only, not inference") - load("stage", PROJECT / "stage_runtime.py").stage(project, runtime.parent) + native = project / package / RUNTIME_NAME + native.touch() + result = build(project, out, "wheel", env) + assert result.returncode != 0 and "Missing staged external runtime" in result.stdout, result.stdout + native.write_bytes(b"packaging fixture only, NOT an executable runtime") + result = build(project, out, "wheel", env) + assert result.returncode != 0 and "Missing runtime notices" in result.stdout, result.stdout + notices = project / package / "licenses" + notices.mkdir() + shutil.copy2(project / "LICENSE", notices / "clap-cpp.txt") + (notices / "dependency.txt").write_text("Dependency notice") + companion = project / package / "companions" / "libdependency.so.1" + companion.parent.mkdir() + companion.write_bytes(b"companion fixture") result = build(project, out, "wheel", env) assert result.returncode == 0, result.stdout wheel = next(out.glob("*.whl")) assert "-py3-none-" in wheel.name and not wheel.name.endswith("-any.whl") with zipfile.ZipFile(wheel) as archive: - wheel_metadata = archive.read(next(n for n in archive.namelist() if n.endswith(".dist-info/WHEEL"))).decode() - assert "Root-Is-Purelib: false" in wheel_metadata - assert archive.read(f"clap_cpp/{RUNTIME_NAME}") == (runtime / RUNTIME_NAME).read_bytes() - assert "clap_cpp/clap.pyi" in archive.namelist() - assert "clap_cpp/py.typed" in archive.namelist() - assert "clap_cpp/licenses/clap-cpp.txt" in archive.namelist() + entries = archive.namelist() + info = archive.read(next(n for n in entries if n.endswith(".dist-info/WHEEL"))) + assert b"Root-Is-Purelib: false" in info + assert archive.read(f"{package}/{RUNTIME_NAME}") == native.read_bytes() + assert archive.read(f"{package}/companions/libdependency.so.1") == companion.read_bytes() + assert archive.read(f"{package}/licenses/clap-cpp.txt") == (project / "LICENSE").read_bytes() + assert f"{package}/licenses/dependency.txt" in entries + for name in ("__init__.py", "clap.py", "clap.pyi", "py.typed"): + assert f"{package}/{name}" in entries + assert not any("__pycache__" in n for n in entries) result = build(project, out, "sdist", env) assert result.returncode == 0, result.stdout - extracted = tmp_path / "extracted" with tarfile.open(next(out.glob("*.tar.gz"))) as archive: - names = archive.getnames() - assert len(names) == len(set(names)) - assert not any("/native/" in n or n.endswith((".so", ".dylib", ".dll", ".cpp")) for n in names) - for name in ("stage_runtime.py", "prepare_backend.py", "_build_backend.py", "LICENSE", "clap_cpp/clap.pyi", "clap_cpp/py.typed"): - assert any(n.endswith("/" + name) for n in names) - archive.extractall(extracted, filter="data") - source = next(extracted.iterdir()) + entries = {n.split("/", 1)[1] for n in archive.getnames() if "/" in n} + required = {"pyproject.toml", "pyproject.toml.j2", "README.md", "LICENSE", "hatch_build.py", + *(f"{package}/{name}" for name in ("__init__.py", "clap.py", "clap.pyi", "py.typed"))} + assert required <= entries + assert not any(n.startswith("native/") or "licenses/" in n or "companions/" in n or + n.endswith((".so", ".dll", ".dylib", ".cpp")) for n in entries) + assert not {"stage_runtime.py", "prepare_backend.py", "_build_backend.py"} & entries + archive.extractall(tmp_path / "extracted", filter="data") + source = next((tmp_path / "extracted").iterdir()) result = build(source, tmp_path / "from-sdist", "wheel", env) - assert result.returncode != 0 and "stage_runtime.py" in result.stdout - load("sdist_stage", source / "stage_runtime.py").stage(source, runtime.parent) + assert result.returncode != 0 and "Missing staged external runtime" in result.stdout, result.stdout + shutil.copytree(project / package, source / package, dirs_exist_ok=True) result = build(source, tmp_path / "from-sdist", "wheel", env) assert result.returncode == 0, result.stdout - load("sdist_backend", source / "prepare_backend.py").prepare("cuda", tmp_path / "cuda") + if backend == "cpu": + render(source, tmp_path / "cuda-from-cpu-sdist", "cuda") assert not marker.exists() + + +def test_editable_requires_runtime(tmp_path): + project, package = render(PROJECT, tmp_path / "project") + command = [sys.executable, "-c", "from hatchling.build import build_editable; build_editable('editable')"] + result = subprocess.run(command, cwd=project, capture_output=True, text=True) + assert result.returncode != 0 and "Missing staged external runtime" in result.stderr + (project / package / RUNTIME_NAME).write_bytes(b"editable fixture") + result = subprocess.run(command, cwd=project, capture_output=True, text=True) + assert result.returncode != 0 and "Missing runtime notices" in result.stderr From 8cd0cd8d5726bb087c1da0c2f0214b3eda1f262e Mon Sep 17 00:00:00 2001 From: zhenghaoz Date: Sat, 7 Nov 2026 02:17:48 +0800 Subject: [PATCH 2/4] fix(ci): set explicit macOS native and wheel target --- .github/workflows/ci.yml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 38e3251..6180824 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -78,6 +78,11 @@ jobs: } 7z x $archive "-o$destination" -y (Get-ChildItem $destination -Filter ffmpeg.exe -Recurse).DirectoryName | Out-File -FilePath $env:GITHUB_PATH -Encoding utf8 -Append + - name: Set macOS native and wheel target + if: matrix.os == 'macos-latest' + shell: bash + run: | + printf '%s\n' 'MACOSX_DEPLOYMENT_TARGET=11.0' '_PYTHON_HOST_PLATFORM=macosx-11.0-arm64' >> "$GITHUB_ENV" - name: build-libraries uses: ./.github/actions/build-libraries with: From 482f105e0716fbb5afc81e9105946a3d2d6d11b8 Mon Sep 17 00:00:00 2001 From: zhenghaoz Date: Sat, 7 Nov 2026 03:07:29 +0800 Subject: [PATCH 3/4] refactor(python): use direct scikit-build backend and exclude packaged tests --- .github/RELEASING.md | 15 ++- .github/scripts/test_python_release_assets.py | 77 ++++++++++- .github/workflows/ci.yml | 24 +++- .github/workflows/release.yml | 96 +++++++++++++- python/README.md | 67 +++++----- python/hatch_build.py | 36 ------ python/pyproject.toml | 32 ++--- python/pyproject.toml.j2 | 32 ++--- python/tests/test_packaging.py | 121 ++++++++---------- 9 files changed, 320 insertions(+), 180 deletions(-) delete mode 100644 python/hatch_build.py diff --git a/.github/RELEASING.md b/.github/RELEASING.md index 82490b7..04baa6d 100644 --- a/.github/RELEASING.md +++ b/.github/RELEASING.md @@ -33,12 +33,19 @@ Node inference tests run against the checkout and its platform workspace in CI. No Python wheel extraction is involved. All native packages publish before the wrapper. Python publishes canonical wrapper-only source archives including the shared -`pyproject.toml.j2` template and thin `hatch_build.py` validation/platform-tag hook; -they contain neither native sources nor compiled libraries. Release jobs render +`pyproject.toml.j2` template and direct scikit-build-core metadata; +neither sdists nor wheels contain tests. Source archives contain neither native +sources nor compiled libraries. Release jobs render the template with Jinja2 for CPU, CUDA, ROCm and Vulkan, preserving exact versions. CPU wheel jobs extract the base archive and directly copy the downloaded -`clap_cpp/` runtime contents and license into its package. Hatchling never compiles -native code; cibuildwheel runs installed-wheel tests for Python 3.9, 3.10, +`clap_cpp/` runtime contents and license into its package. scikit-build-core never compiles +native code (`wheel.cmake=false`, `sdist.cmake=false`); no custom hook/backend +is used. Actions checks runtime binary headers and nonempty runtime notices before +assembly and checks the archived payload again before publication. The direct +backend alone does not reject unstaged projects; never publish their unusable +runtime-free wheels. The wheel job checks out `${{ github.sha }}` and copies tests +separately into its working source directory (not into published sdists/wheels). +cibuildwheel runs installed-wheel tests for Python 3.9, 3.10, 3.11, 3.12, 3.13 and 3.14. ABI-independent `py3-none` wheels are reused across these interpreters; Linux wheels retain auditwheel-repaired manylinux tags. GPU jobs recursively check out native sources and compile before packaging the diff --git a/.github/scripts/test_python_release_assets.py b/.github/scripts/test_python_release_assets.py index 2c3940e..56c5b9b 100644 --- a/.github/scripts/test_python_release_assets.py +++ b/.github/scripts/test_python_release_assets.py @@ -38,7 +38,13 @@ def unit_wheel(root, distribution="clap_cpp", platform=PLATFORMS[0], interpreter archive.writestr(f"{distribution}/clap.cpython-311-x86_64-linux-gnu.so", b"UNIT FIXTURE, NOT NATIVE CODE") else: directory = "wrong_package" if payload == "wrong-package" else distribution - archive.writestr(f"{directory}/{native}", b"UNIT FIXTURE, NOT NATIVE CODE") + archive.writestr(f"{directory}/{native}", (b"" if payload == "empty" else b"invalid" if payload == "invalid" else + (b"MZ" if platform == "win_amd64" else b"\xcf\xfa\xed\xfe" if platform.startswith("macosx_") else b"\x7fELF") + + b"UNIT FIXTURE, NOT NATIVE CODE")) + if payload != "missing-notice": + archive.writestr(f"{distribution}/licenses/clap-cpp.txt", "" if payload == "empty-notice" else "UNIT FIXTURE notice") + if payload == "tests": + archive.writestr(f"{distribution}/tests/test_leaked.py", "# TEST FIXTURE") return path @@ -59,7 +65,7 @@ def setUp(self): def unit_sdist(self, source, distribution, omitted=None, extra=()): """Write wrapper-only UNIT FIXTURE source archives for the real heredoc.""" with tarfile.open(source, "w:gz") as archive: - for name in ("pyproject.toml", "pyproject.toml.j2", "hatch_build.py", "README.md", "LICENSE", + for name in ("pyproject.toml", "pyproject.toml.j2", "README.md", "LICENSE", *(f"{distribution}/{name}" for name in ("__init__.py", "clap.py", "clap.pyi", "py.typed")), *extra): if name == omitted: continue @@ -68,9 +74,9 @@ def unit_sdist(self, source, distribution, omitted=None, extra=()): entry.size = len(data) archive.addfile(entry, io.BytesIO(data)) - def test_unit_fixture_sdist_requires_wrapper_metadata_and_helper(self): + def test_unit_fixture_sdist_requires_wrapper_metadata_and_template(self): for distribution in DISTRIBUTIONS: - for required in ("pyproject.toml", "pyproject.toml.j2", "hatch_build.py", "README.md", "LICENSE", + for required in ("pyproject.toml", "pyproject.toml.j2", "README.md", "LICENSE", *(f"{distribution}/{name}" for name in ("__init__.py", "clap.py", "clap.pyi", "py.typed"))): with self.subTest(distribution=distribution, required=required): source = self.artifacts / f"{distribution}-0.1.0.tar.gz" @@ -85,6 +91,17 @@ def test_unit_fixture_sdist_rejects_native_graph_and_binaries(self): self.unit_sdist(self.artifacts / "clap_cpp-0.1.0.tar.gz", "clap_cpp", extra=(native,)) self.assert_rejected("Sdist contains native sources or compiled runtime") + def test_published_archives_reject_tests(self): + for distribution in DISTRIBUTIONS: + with self.subTest(distribution=distribution): + source = self.artifacts / f"{distribution}-0.1.0.tar.gz" + self.unit_sdist(source, distribution, extra=("tests/test_leaked.py",)) + self.assert_rejected("Sdist contains tests") + self.unit_sdist(source, distribution) + unit_wheel(self.artifacts, distribution=distribution, payload="tests") + self.assert_rejected("Wheel contains tests") + unit_wheel(self.artifacts, distribution=distribution) + def run_validator(self): shutil.rmtree(self.root / "dist", ignore_errors=True) return subprocess.run([sys.executable, "-c", validator_source()], cwd=self.root, capture_output=True, text=True) @@ -119,6 +136,18 @@ def test_unit_fixture_old_extension_rejected(self): self.replace_cpu(payload="extension") self.assert_rejected("package-local native library") + def test_unit_fixture_empty_or_invalid_native_rejected(self): + for payload in ("empty", "invalid"): + with self.subTest(payload=payload): + self.replace_cpu(payload=payload) + self.assert_rejected("Invalid package-local native library") + + def test_unit_fixture_runtime_notice_required(self): + for payload in ("missing-notice", "empty-notice"): + with self.subTest(payload=payload): + self.replace_cpu(payload=payload) + self.assert_rejected("Missing runtime notices") + def test_unit_fixture_missing_wrapper_rejected(self): self.replace_cpu(payload="missing-wrapper") self.assert_rejected("ctypes wrapper") @@ -152,6 +181,46 @@ def test_unit_fixture_six_rocm_manifests_rejected(self): (directory / MANIFEST).write_text("UNIT-FIXTURE-SONAME.so\n") self.assert_rejected("ROCm exclusion manifest") + def test_actions_runtime_gates_reject_invalid_staging(self): + # Execute all five real Actions heredocs, not a test copy of the checks. + ci = WORKFLOW.with_name("ci.yml") + cases = [(ci, "Package Python with the shared runtime", "build/python-package/clap_cpp"), + (WORKFLOW, "Copy external runtime into the CPU wrapper project", "python-dist/clap_cpp")] + cases += [(WORKFLOW, f"Stage runtime and package canonical {backend.upper()} wrapper sdist", + f"runner/clap-{backend}/clap_cpp_{backend}") for backend in ("cuda", "vulkan", "rocm")] + native = "clap.dll" if sys.platform == "win32" else "libclap.dylib" if sys.platform == "darwin" else "libclap.so" + magic = b"MZ" if sys.platform == "win32" else b"\xcf\xfa\xed\xfe" if sys.platform == "darwin" else b"\x7fELF" + for workflow, step_name, path in cases: + step = workflow.read_text().split(f"- name: {step_name}\n", 1)[1] + block = step.split("python - <<'PY'\n", 1)[1] + source = textwrap.dedent(block.split("\n" + " " * (10 if workflow == ci else 8) + "PY\n", 1)[0]) + package = self.root / path + package.mkdir(parents=True, exist_ok=True) + runtime = package / native + notice = package / "licenses/clap-cpp.txt" + notice.parent.mkdir(exist_ok=True) + for payload, license_text, error in ((None, "notice", "Missing staged external runtime"), + (b"", "notice", "Missing staged external runtime"), + (b"invalid", "notice", "Invalid staged external runtime"), + (magic, None, "Missing runtime notices"), + (magic, " ", "Missing runtime notices"), + (magic + b"UNIT FIXTURE, NOT NATIVE CODE", "notice", None)): + with self.subTest(step=step_name, payload=payload, license_text=license_text): + runtime.unlink(missing_ok=True) + notice.unlink(missing_ok=True) + if payload is not None: + runtime.write_bytes(payload) + if license_text is not None: + notice.write_text(license_text) + result = subprocess.run([sys.executable, "-c", source], cwd=self.root, + env=dict(os.environ, RUNNER_TEMP=str(self.root / "runner")), + capture_output=True, text=True) + if error: + self.assertNotEqual(result.returncode, 0) + self.assertIn(error, result.stderr) + else: + self.assertEqual(result.returncode, 0, result.stderr) + @unittest.skipUnless(os.environ.get("CLAP_TEST_CPU_WHEEL"), "set CLAP_TEST_CPU_WHEEL for real built wheel inspection") def test_actual_built_cpu_payload_and_unrepaired_release_rejection(self): from packaging.utils import parse_wheel_filename diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 6180824..1e872fd 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -78,11 +78,11 @@ jobs: } 7z x $archive "-o$destination" -y (Get-ChildItem $destination -Filter ffmpeg.exe -Recurse).DirectoryName | Out-File -FilePath $env:GITHUB_PATH -Encoding utf8 -Append - - name: Set macOS native and wheel target + - name: Set macOS native deployment minimum if: matrix.os == 'macos-latest' shell: bash run: | - printf '%s\n' 'MACOSX_DEPLOYMENT_TARGET=11.0' '_PYTHON_HOST_PLATFORM=macosx-11.0-arm64' >> "$GITHUB_ENV" + printf '%s\n' 'MACOSX_DEPLOYMENT_TARGET=11.0' >> "$GITHUB_ENV" - name: build-libraries uses: ./.github/actions/build-libraries with: @@ -97,16 +97,34 @@ jobs: run: cmake --install build/libraries --config Release --prefix "${{ runner.temp }}/clap-install" - name: Install Python packaging and test dependencies if: matrix.os != 'windows-11-arm' - run: python -m pip install build "hatchling>=1.27" jinja2 packaging pytest "mypy>=1.8" "typing_extensions>=4.2" + run: python -m pip install build "scikit-build-core>=1.1.2" jinja2 packaging pytest "mypy>=1.8" "typing_extensions>=4.2" - name: Package Python with the shared runtime if: matrix.os != 'windows-11-arm' shell: bash run: | + set -euo pipefail python -m build python --sdist --outdir python-dist mkdir -p build/python-package tar -xzf python-dist/*.tar.gz --strip-components=1 -C build/python-package cp -R "libraries/${{ matrix.runtime }}/clap_cpp/." build/python-package/clap_cpp/ cp build/python-package/LICENSE build/python-package/clap_cpp/licenses/clap-cpp.txt + python - <<'PY' + from pathlib import Path + import sys + package = Path("build/python-package/clap_cpp") + native = "clap.dll" if sys.platform == "win32" else ("libclap.dylib" if sys.platform == "darwin" else "libclap.so") + runtime = package / native + if not runtime.is_file() or not runtime.stat().st_size: + raise SystemExit(f"Missing staged external runtime: {runtime}") + magic = runtime.read_bytes()[:4] + if not (magic == b"\x7fELF" if sys.platform.startswith("linux") else + magic[:2] == b"MZ" if sys.platform == "win32" else + magic in (b"\xcf\xfa\xed\xfe", b"\xfe\xed\xfa\xcf")): + raise SystemExit(f"Invalid staged external runtime: {runtime}") + notice = package / "licenses" / "clap-cpp.txt" + if not notice.is_file() or not notice.read_text(encoding="utf-8").strip(): + raise SystemExit(f"Missing runtime notices: {notice}") + PY python -m build build/python-package --wheel --outdir python-dist python -m pip install --no-deps --no-index --find-links python-dist clap-cpp - name: Test Python diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 3e02ac5..0451940 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -50,7 +50,7 @@ jobs: destination="$RUNNER_TEMP/clap-$backend" package="clap_cpp_$backend" mkdir -p "$destination/$package" - cp python-dist/{README.md,LICENSE,hatch_build.py,pyproject.toml.j2} "$destination/" + cp python-dist/{README.md,LICENSE,pyproject.toml.j2} "$destination/" cp python-dist/clap_cpp/{clap.py,clap.pyi,py.typed} "$destination/$package/" printf '"""Native %s runtime; import clap_cpp for the public API."""\n' "$backend" > "$destination/$package/__init__.py" BACKEND="$backend" DESTINATION="$destination" python - <<'PY' @@ -144,6 +144,9 @@ jobs: target: win32-x64 runs-on: ${{ matrix.os }} steps: + - uses: actions/checkout@v4 + with: + ref: ${{ github.sha }} - uses: actions/download-artifact@v4 with: name: python-sdists @@ -160,8 +163,29 @@ jobs: - name: Copy external runtime into the CPU wrapper project shell: bash run: | + set -euo pipefail cp -R python-dist/external-runtime/clap_cpp/. python-dist/clap_cpp/ cp python-dist/LICENSE python-dist/clap_cpp/licenses/clap-cpp.txt + python - <<'PY' + from pathlib import Path + import sys + package = Path("python-dist/clap_cpp") + native = "clap.dll" if sys.platform == "win32" else ("libclap.dylib" if sys.platform == "darwin" else "libclap.so") + runtime = package / native + if not runtime.is_file() or not runtime.stat().st_size: + raise SystemExit(f"Missing staged external runtime: {runtime}") + magic = runtime.read_bytes()[:4] + if not (magic == b"\x7fELF" if sys.platform.startswith("linux") else + magic[:2] == b"MZ" if sys.platform == "win32" else + magic in (b"\xcf\xfa\xed\xfe", b"\xfe\xed\xfa\xcf")): + raise SystemExit(f"Invalid staged external runtime: {runtime}") + notice = package / "licenses" / "clap-cpp.txt" + if not notice.is_file() or not notice.read_text(encoding="utf-8").strip(): + raise SystemExit(f"Missing runtime notices: {notice}") + PY + - name: Supply installed-wheel tests from the exact release checkout + shell: bash + run: cp -R python/tests python-dist/tests - name: Package and test CPU wheels from canonical wrapper sdist and shared runtime uses: pypa/cibuildwheel@v4.3.4 with: @@ -174,7 +198,7 @@ jobs: CIBW_MANYLINUX_X86_64_IMAGE: manylinux_2_28 CIBW_MANYLINUX_AARCH64_IMAGE: manylinux_2_28 CIBW_ENVIRONMENT_MACOS: MACOSX_DEPLOYMENT_TARGET=11.0 - CIBW_TEST_REQUIRES: pytest build hatchling>=1.27 jinja2 packaging tomli mypy>=1.8 typing_extensions>=4.2 + CIBW_TEST_REQUIRES: pytest build scikit-build-core>=1.1.2 jinja2 packaging tomli mypy>=1.8 typing_extensions>=4.2 CIBW_TEST_COMMAND: python -m pytest {package}/tests -v - uses: actions/upload-artifact@v4 with: @@ -251,6 +275,23 @@ jobs: set -euo pipefail cp -R "libraries/cuda/clap_cpp/." "$RUNNER_TEMP/clap-cuda/clap_cpp_cuda/" cp "$RUNNER_TEMP/clap-cuda/LICENSE" "$RUNNER_TEMP/clap-cuda/clap_cpp_cuda/licenses/clap-cpp.txt" + python - <<'PY' + from pathlib import Path + import sys + package = Path(__import__("os").environ["RUNNER_TEMP"]) / "clap-cuda" / "clap_cpp_cuda" + native = "clap.dll" if sys.platform == "win32" else ("libclap.dylib" if sys.platform == "darwin" else "libclap.so") + runtime = package / native + if not runtime.is_file() or not runtime.stat().st_size: + raise SystemExit(f"Missing staged external runtime: {runtime}") + magic = runtime.read_bytes()[:4] + if not (magic == b"\x7fELF" if sys.platform.startswith("linux") else + magic[:2] == b"MZ" if sys.platform == "win32" else + magic in (b"\xcf\xfa\xed\xfe", b"\xfe\xed\xfa\xcf")): + raise SystemExit(f"Invalid staged external runtime: {runtime}") + notice = package / "licenses" / "clap-cpp.txt" + if not notice.is_file() or not notice.read_text(encoding="utf-8").strip(): + raise SystemExit(f"Missing runtime notices: {notice}") + PY python -m build --wheel "$RUNNER_TEMP/clap-cuda" --outdir raw - name: Repair CUDA wheel run: | @@ -332,6 +373,23 @@ jobs: set -euo pipefail cp -R "libraries/vulkan/clap_cpp/." "$RUNNER_TEMP/clap-vulkan/clap_cpp_vulkan/" cp "$RUNNER_TEMP/clap-vulkan/LICENSE" "$RUNNER_TEMP/clap-vulkan/clap_cpp_vulkan/licenses/clap-cpp.txt" + python - <<'PY' + from pathlib import Path + import sys + package = Path(__import__("os").environ["RUNNER_TEMP"]) / "clap-vulkan" / "clap_cpp_vulkan" + native = "clap.dll" if sys.platform == "win32" else ("libclap.dylib" if sys.platform == "darwin" else "libclap.so") + runtime = package / native + if not runtime.is_file() or not runtime.stat().st_size: + raise SystemExit(f"Missing staged external runtime: {runtime}") + magic = runtime.read_bytes()[:4] + if not (magic == b"\x7fELF" if sys.platform.startswith("linux") else + magic[:2] == b"MZ" if sys.platform == "win32" else + magic in (b"\xcf\xfa\xed\xfe", b"\xfe\xed\xfa\xcf")): + raise SystemExit(f"Invalid staged external runtime: {runtime}") + notice = package / "licenses" / "clap-cpp.txt" + if not notice.is_file() or not notice.read_text(encoding="utf-8").strip(): + raise SystemExit(f"Missing runtime notices: {notice}") + PY python -m build --wheel "$RUNNER_TEMP/clap-vulkan" --outdir raw - name: Repair Vulkan wheel (Ubuntu 24.04 glibc floor) run: | @@ -431,6 +489,23 @@ jobs: set -euo pipefail cp -R "libraries/rocm/clap_cpp/." "$RUNNER_TEMP/clap-rocm/clap_cpp_rocm/" cp "$RUNNER_TEMP/clap-rocm/LICENSE" "$RUNNER_TEMP/clap-rocm/clap_cpp_rocm/licenses/clap-cpp.txt" + python - <<'PY' + from pathlib import Path + import sys + package = Path(__import__("os").environ["RUNNER_TEMP"]) / "clap-rocm" / "clap_cpp_rocm" + native = "clap.dll" if sys.platform == "win32" else ("libclap.dylib" if sys.platform == "darwin" else "libclap.so") + runtime = package / native + if not runtime.is_file() or not runtime.stat().st_size: + raise SystemExit(f"Missing staged external runtime: {runtime}") + magic = runtime.read_bytes()[:4] + if not (magic == b"\x7fELF" if sys.platform.startswith("linux") else + magic[:2] == b"MZ" if sys.platform == "win32" else + magic in (b"\xcf\xfa\xed\xfe", b"\xfe\xed\xfa\xcf")): + raise SystemExit(f"Invalid staged external runtime: {runtime}") + notice = package / "licenses" / "clap-cpp.txt" + if not notice.is_file() or not notice.read_text(encoding="utf-8").strip(): + raise SystemExit(f"Missing runtime notices: {notice}") + PY python -m build --wheel "$RUNNER_TEMP/clap-rocm" --outdir raw - name: Repair ROCm wheel, retaining system ROCm runtime run: | @@ -536,12 +611,23 @@ jobs: with zipfile.ZipFile(wheel) as archive: package = name.replace('-', '_') entries = set(archive.namelist()) + if any('tests' in pathlib.PurePosixPath(entry).parts for entry in entries): + raise SystemExit(f'Wheel contains tests: {wheel.name}') if f'{package}/clap.py' not in entries: raise SystemExit(f'Missing Python ctypes wrapper: {wheel.name}') native = ('clap.dll' if platform == 'win-amd64' else 'libclap.dylib' if platform == 'macosx-arm64' else 'libclap.so') if f'{package}/{native}' not in entries: raise SystemExit(f'Missing package-local native library {package}/{native}: {wheel.name}') + payload = archive.read(f'{package}/{native}') + valid = (payload.startswith(b'\x7fELF') if platform.startswith('linux-') else + payload.startswith(b'MZ') if platform == 'win-amd64' else + payload[:4] in (b'\xcf\xfa\xed\xfe', b'\xfe\xed\xfa\xcf')) + if not valid: + raise SystemExit(f'Invalid package-local native library: {wheel.name}') + notice = f'{package}/licenses/clap-cpp.txt' + if notice not in entries or not archive.read(notice).strip(): + raise SystemExit(f'Missing runtime notices: {wheel.name}') if (dist / wheel.name).exists(): raise SystemExit(f'Duplicate release asset: {wheel.name}') shutil.copy2(wheel, dist) @@ -556,7 +642,9 @@ jobs: with tarfile.open(source) as archive: entries = {entry.name.split('/', 1)[1] for entry in archive if '/' in entry.name and entry.isfile()} package = name.replace('-', '_') - required = {'pyproject.toml', 'pyproject.toml.j2', 'hatch_build.py', 'README.md', 'LICENSE', + if any('tests' in pathlib.PurePosixPath(entry).parts for entry in entries): + raise SystemExit(f'Sdist contains tests: {source.name}') + required = {'pyproject.toml', 'pyproject.toml.j2', 'README.md', 'LICENSE', f'{package}/__init__.py', f'{package}/clap.py', f'{package}/clap.pyi', f'{package}/py.typed'} if not required <= entries: raise SystemExit(f'Sdist missing wrapper or packaging input: {source.name}: {required - entries}') @@ -594,7 +682,7 @@ jobs: ROCm GPU targets: gfx908,gfx90a,gfx942,gfx1030,gfx1100,gfx1101,gfx1200,gfx1201. Linux wheels are auditwheel-repaired manylinux wheels; the external GPU runtimes above remain deliberate exceptions, not portability promises. - Source archives contain wrappers, a Jinja metadata template and a validation/tag hook, not native sources. + Source archives contain wrappers, a Jinja metadata template and direct backend metadata, not native sources. Compile native runtimes separately from a recursive checkout, then stage them before packaging; Python wheel builds do not run CMake. CPU wheels pass installed-wheel tests on Python 3.9-3.14; GPU execution is not tested here. diff --git a/python/README.md b/python/README.md index fc90e16..fed28dd 100644 --- a/python/README.md +++ b/python/README.md @@ -101,7 +101,7 @@ libraries/linux-x64/ stage it into the wrapper project and build a platform wheel: ```sh -python -m pip install build 'hatchling>=1.27' +python -m pip install build 'scikit-build-core>=1.1.2' cp -R libraries/linux-x64/clap_cpp/. python/clap_cpp/ cp python/LICENSE python/clap_cpp/licenses/clap-cpp.txt python -m build --wheel python @@ -115,30 +115,35 @@ Start from a fresh wrapper project when switching runtimes so stale companion libraries cannot leak into the next wheel. Do not copy Python sources from a runtime artifact or overwrite the wrapper files. -Hatchling only assembles the copied files. Its thin `hatch_build.py` hook rejects -missing/empty host-platform libraries or `licenses/clap-cpp.txt`, and sets -non-pure `py3-none` platform tags. It never stages files, renders metadata or -compiles code. A local Linux wheel has a `linux_` tag and still needs -portability checks/repair before release. Metadata and wrapper-only source -archives do not require a runtime. Editable installs also require a copied runtime; -Hatchling's temporary editable wheel references the source directory and is not -a distributable runtime wheel. - -On macOS, the wheel tag must match the staged runtime's architecture and minimum -macOS deployment target, not just the Python interpreter's build configuration. -A universal2 interpreter can report a universal2 platform even when the copied -runtime is single-architecture. The hook rejects all universal2 tags, including -explicit overrides; it does not verify native binary architectures. Set an -explicit single-architecture `_PYTHON_HOST_PLATFORM` matching the staged runtime -when building with such an interpreter. Supported macOS releases are ARM64; for -an ARM64 runtime built with deployment target 11.0, use: +The direct `scikit_build_core.build` backend only assembles the copied files: +`wheel.cmake=false`, `wheel.platlib=true`, `wheel.py-api="py3"` and +`sdist.cmake=false`. Wheel force-inclusion preserves Git-ignored runtime files; +explicit source inclusion keeps sdists wrapper-only. There is no custom backend, +build hook, staging or generation script. + +Actions validates a nonempty host-platform library with the expected binary +header and a nonempty `licenses/clap-cpp.txt` before building wheels. The final +release gate validates the archived payload and notices again before publication. +Local builds must perform equivalent checks: the direct backend does **not** +reject an unstaged project, and a wheel without its matching native runtime is +unusable and must never be published. Metadata and wrapper-only source archives +need no runtime. Editable installs reference staged sources and are development +artifacts, not distributable runtime wheels. A local Linux wheel has a +`linux_` tag and still needs portability checks/repair before release. + +On macOS, scikit-build-core selects the executing architecture via packaging's +platform tags, rather than treating a universal2 interpreter as proof of a +universal2 runtime. Keep `MACOSX_DEPLOYMENT_TARGET=11.0` for both the external +native build and wheel assembly to preserve the supported deployment minimum. +For an ARM64 runtime built at that minimum on an ARM64 host: ```sh -_PYTHON_HOST_PLATFORM=macosx-11.0-arm64 python -m build --wheel python +MACOSX_DEPLOYMENT_TARGET=11.0 python -m build --wheel python ``` -This only sets packaging metadata; it does not convert the runtime or change its -deployment target. The release cibuildwheel ARM64 override follows the same rule. +This does not convert runtime architectures or change an existing binary's +minimum. Cross-packaging requires explicit matching platform configuration; +never label a single-architecture runtime universal2. ### Optional backend projects @@ -154,7 +159,7 @@ backend=cuda # or rocm, vulkan project="build/python-$backend" package="clap_cpp_$backend" mkdir -p "$project/$package" -cp python-dist/{README.md,LICENSE,hatch_build.py,pyproject.toml.j2} "$project/" +cp python-dist/{README.md,LICENSE,pyproject.toml.j2} "$project/" cp python-dist/clap_cpp/{clap.py,clap.pyi,py.typed} "$project/$package/" printf '"""Native %s runtime; import clap_cpp for the public API."""\n' "$backend" > "$project/$package/__init__.py" BACKEND="$backend" PROJECT="$project" python - <<'PY' @@ -191,17 +196,19 @@ cp -R libraries/linux-x64/clap_cpp/. build/clap_cpp-0.1.0/clap_cpp/ cp build/clap_cpp-0.1.0/LICENSE build/clap_cpp-0.1.0/clap_cpp/licenses/clap-cpp.txt python -m build --wheel build/clap_cpp-0.1.0 python -m pip install 'build/clap_cpp-0.1.0/dist/'*.whl -python -m pip install pytest mypy typing_extensions build 'hatchling>=1.27' jinja2 +python -m pip install pytest mypy typing_extensions build 'scikit-build-core>=1.1.2' jinja2 python -m pytest python/tests ``` -Source distributions contain wrapper/typing files, tests, `LICENSE` and packaging -helpers, not native C++ snapshots, staged binaries or native build definitions. -An unstaged sdist intentionally cannot build a wheel: plain `python -m build -python` creates an sdist then fails to build its unstaged wheel. Use the explicit -`--sdist`, unpack, stage, `--wheel` sequence above. Backend projects can also be -generated from the extracted CPU sdist before staging their runtimes. No NumPy, -Python development headers or pybind11 are required. +Source distributions contain wrapper/typing files, `LICENSE` and +metadata/template files, not native C++ snapshots, staged binaries, build hooks +or native build definitions. Tests are excluded from both wheels and sdists; release +CI supplies them separately from the exact release checkout and tests the installed +wheel on each supported interpreter. Plain `python -m build python` builds from an +unstaged sdist and therefore produces an unusable runtime-free wheel; do not +publish it. Use the explicit `--sdist`, unpack, stage, `--wheel` sequence above. +Backend projects can also be rendered from the extracted CPU sdist before staging +their runtimes. No NumPy, Python development headers or pybind11 are required. ## Releases diff --git a/python/hatch_build.py b/python/hatch_build.py deleted file mode 100644 index ef3032e..0000000 --- a/python/hatch_build.py +++ /dev/null @@ -1,36 +0,0 @@ -"""Validate an externally copied runtime and mark its wheel as platform-specific.""" - -from pathlib import Path -import sys -import sysconfig - -from hatchling.builders.hooks.plugin.interface import BuildHookInterface - - -class CustomBuildHook(BuildHookInterface): - def initialize(self, version, build_data): - package = Path(self.root) / self.metadata.name.replace("-", "_") - native = "clap.dll" if sys.platform == "win32" else ( - "libclap.dylib" if sys.platform == "darwin" else "libclap.so" - ) - runtime = package / native - if not runtime.is_file() or runtime.stat().st_size == 0: - raise RuntimeError( - f"Missing staged external runtime: {runtime}. " - "Copy the externally compiled runtime's clap_cpp/ contents into " - f"{package.name}/ before building a wheel. " - "Python packaging does not compile native libraries." - ) - notice = package / "licenses" / "clap-cpp.txt" - if not notice.is_file() or notice.stat().st_size == 0: - raise RuntimeError("Missing runtime notices; copy LICENSE to " + str(notice)) - platform = sysconfig.get_platform().replace("-", "_").replace(".", "_") - if sys.platform == "darwin" and "universal2" in platform: - raise RuntimeError( - "Cannot tag an unverified external runtime as universal2. " - "Set _PYTHON_HOST_PLATFORM to a single-architecture macOS platform " - "matching the staged runtime's architecture and deployment target " - "(for example, macosx-11.0-arm64)." - ) - build_data["pure_python"] = False - build_data["tag"] = f"py3-none-{platform}" diff --git a/python/pyproject.toml b/python/pyproject.toml index 4ea3122..bbd3670 100644 --- a/python/pyproject.toml +++ b/python/pyproject.toml @@ -1,7 +1,7 @@ # Generated from pyproject.toml.j2; render for cpu, cuda, rocm or vulkan. [build-system] -requires = ["hatchling>=1.27"] -build-backend = "hatchling.build" +requires = ["scikit-build-core>=1.1.2"] +build-backend = "scikit_build_core.build" [project] name = "clap-cpp" @@ -19,20 +19,22 @@ cuda = ["clap-cpp-cuda==0.1.0"] rocm = ["clap-cpp-rocm==0.1.0"] vulkan = ["clap-cpp-vulkan==0.1.0"] -[tool.hatch.build] -exclude = ["**/__pycache__", "**/*.pyc", "**/.pytest_cache"] - -[tool.hatch.build.targets.wheel] -packages = ["clap_cpp"] -# The explicitly copied native payload is ignored by Git, not by the wheel. -ignore-vcs = true - -[tool.hatch.build.targets.wheel.hooks.custom] -path = "hatch_build.py" - -[tool.hatch.build.targets.sdist] +[tool.scikit-build] +minimum-version = "1.1.2" +wheel.cmake = false +wheel.platlib = true +wheel.py-api = "py3" +wheel.packages = [] +wheel.exclude = ["**/__pycache__/**", "**/*.pyc", "**/.pytest_cache/**", "**/tests/**"] +sdist.cmake = false +sdist.exclude = ["**/__pycache__/**", "**/*.pyc", "**/.pytest_cache/**", "**/tests/**"] +sdist.inclusion-mode = "explicit" # Never ship staged binaries, companion data or runtime notices in source archives. -include = ["/pyproject.toml", "/pyproject.toml.j2", "/README.md", "/LICENSE", "/hatch_build.py", "/clap_cpp/*.py", "/clap_cpp/*.pyi", "/clap_cpp/py.typed", "/tests/**"] +sdist.include = ["/pyproject.toml", "/pyproject.toml.j2", "/README.md", "/LICENSE", "/clap_cpp/*.py", "/clap_cpp/*.pyi", "/clap_cpp/py.typed"] + +[tool.scikit-build.wheel.force-include] +# Include Git-ignored external payloads without broadening the source archive. +"clap_cpp" = "clap_cpp" [tool.pytest.ini_options] testpaths = ["tests"] diff --git a/python/pyproject.toml.j2 b/python/pyproject.toml.j2 index 3e868f7..4ecd4a2 100644 --- a/python/pyproject.toml.j2 +++ b/python/pyproject.toml.j2 @@ -3,8 +3,8 @@ {% set package = 'clap_cpp' ~ ('_' ~ backend if backend != 'cpu' else '') -%} # Generated from pyproject.toml.j2; render for cpu, cuda, rocm or vulkan. [build-system] -requires = ["hatchling>=1.27"] -build-backend = "hatchling.build" +requires = ["scikit-build-core>=1.1.2"] +build-backend = "scikit_build_core.build" [project] name = "{{ package | replace('_', '-') }}" @@ -22,20 +22,22 @@ cuda = ["clap-cpp-cuda=={{ version }}"] rocm = ["clap-cpp-rocm=={{ version }}"] vulkan = ["clap-cpp-vulkan=={{ version }}"] {% endif %} -[tool.hatch.build] -exclude = ["**/__pycache__", "**/*.pyc", "**/.pytest_cache"] - -[tool.hatch.build.targets.wheel] -packages = ["{{ package }}"] -# The explicitly copied native payload is ignored by Git, not by the wheel. -ignore-vcs = true - -[tool.hatch.build.targets.wheel.hooks.custom] -path = "hatch_build.py" - -[tool.hatch.build.targets.sdist] +[tool.scikit-build] +minimum-version = "1.1.2" +wheel.cmake = false +wheel.platlib = true +wheel.py-api = "py3" +wheel.packages = [] +wheel.exclude = ["**/__pycache__/**", "**/*.pyc", "**/.pytest_cache/**", "**/tests/**"] +sdist.cmake = false +sdist.exclude = ["**/__pycache__/**", "**/*.pyc", "**/.pytest_cache/**", "**/tests/**"] +sdist.inclusion-mode = "explicit" # Never ship staged binaries, companion data or runtime notices in source archives. -include = ["/pyproject.toml", "/pyproject.toml.j2", "/README.md", "/LICENSE", "/hatch_build.py", "/{{ package }}/*.py", "/{{ package }}/*.pyi", "/{{ package }}/py.typed"{% if backend == 'cpu' %}, "/tests/**"{% endif %}] +sdist.include = ["/pyproject.toml", "/pyproject.toml.j2", "/README.md", "/LICENSE", "/{{ package }}/*.py", "/{{ package }}/*.pyi", "/{{ package }}/py.typed"] + +[tool.scikit-build.wheel.force-include] +# Include Git-ignored external payloads without broadening the source archive. +"{{ package }}" = "{{ package }}" {% if backend == 'cpu' %} [tool.pytest.ini_options] testpaths = ["tests"] diff --git a/python/tests/test_packaging.py b/python/tests/test_packaging.py index acba722..844a37b 100644 --- a/python/tests/test_packaging.py +++ b/python/tests/test_packaging.py @@ -1,13 +1,11 @@ -"""Package-only Hatchling wheels consume an explicitly copied external runtime.""" +"""Package-only scikit-build-core wheels consume an explicitly copied external runtime.""" import os from pathlib import Path -import runpy import shutil import subprocess import sys import tarfile -from types import SimpleNamespace import zipfile from jinja2 import Environment, StrictUndefined @@ -33,7 +31,7 @@ def render(source, destination, backend="cpu", version=None): """Mirror the workflow's Jinja rendering and plain wrapper copies.""" destination.mkdir(parents=True) package = "clap_cpp" + (f"_{backend}" if backend != "cpu" else "") - for name in ("README.md", "LICENSE", "hatch_build.py", "pyproject.toml.j2"): + for name in ("README.md", "LICENSE", "pyproject.toml.j2"): shutil.copy2(source / name, destination / name) shutil.copytree(source / "clap_cpp", destination / package, ignore=shutil.ignore_patterns("__pycache__", "*.so", "*.dylib", "*.dll", "licenses")) @@ -56,57 +54,32 @@ def build(project, out, kind, env=None): ) -@pytest.fixture -def macos_hook(tmp_path, monkeypatch): - module = runpy.run_path(str(PROJECT / "hatch_build.py")) - package = tmp_path / "clap_cpp" - (package / "licenses").mkdir(parents=True) - (package / "libclap.dylib").write_bytes(b"single-architecture packaging fixture") - (package / "licenses" / "clap-cpp.txt").write_text("Runtime notice") - hook = module["CustomBuildHook"]( - str(tmp_path), {}, None, SimpleNamespace(name="clap-cpp"), str(tmp_path), "wheel") - monkeypatch.setattr(module["sys"], "platform", "darwin") - # Model a universal2 interpreter, including sysconfig's explicit host override. - monkeypatch.setattr(module["sysconfig"], "get_platform", lambda: os.environ.get( - "_PYTHON_HOST_PLATFORM", "macosx-11.0-universal2")) - monkeypatch.delenv("_PYTHON_HOST_PLATFORM", raising=False) - return hook - - -@pytest.mark.parametrize("explicit", [False, True]) -def test_macos_universal2_tag_rejected(macos_hook, monkeypatch, explicit): - if explicit: - monkeypatch.setenv("_PYTHON_HOST_PLATFORM", "macosx-11.0-universal2") - build_data = {} - with pytest.raises(RuntimeError, match="universal2") as error: - macos_hook.initialize("standard", build_data) - assert "_PYTHON_HOST_PLATFORM" in str(error.value) - assert "single-architecture" in str(error.value) - assert "staged runtime" in str(error.value) - assert "deployment target" in str(error.value) - assert "tag" not in build_data - - -def test_macos_explicit_arm64_runtime_tag(macos_hook, monkeypatch): - monkeypatch.setenv("_PYTHON_HOST_PLATFORM", "macosx-11.0-arm64") - build_data = {} - macos_hook.initialize("standard", build_data) - assert build_data == {"pure_python": False, "tag": "py3-none-macosx_11_0_arm64"} - - def test_template_matches_checked_in_cpu_metadata(tmp_path): project, _ = render(PROJECT, tmp_path / "cpu") assert (project / "pyproject.toml").read_bytes() == (PROJECT / "pyproject.toml").read_bytes() config = metadata(project) - assert config["build-system"] == {"requires": ["hatchling>=1.27"], "build-backend": "hatchling.build"} - assert "scikit-build" not in config["tool"] - assert config["tool"]["hatch"]["build"]["targets"]["wheel"]["hooks"]["custom"]["path"] == "hatch_build.py" + assert config["build-system"] == {"requires": ["scikit-build-core>=1.1.2"], "build-backend": "scikit_build_core.build"} + settings = config["tool"]["scikit-build"] + assert settings["wheel"]["cmake"] is False + assert settings["wheel"]["platlib"] is True + assert settings["wheel"]["py-api"] == "py3" + assert settings["sdist"]["cmake"] is False + assert settings["sdist"]["inclusion-mode"] == "explicit" + assert settings["wheel"]["force-include"] == {"clap_cpp": "clap_cpp"} @pytest.mark.parametrize("backend", ["cpu", "cuda", "rocm", "vulkan"]) def test_wheel_and_wrapper_only_sdist(tmp_path, backend): project, package = render(PROJECT, tmp_path / "project", backend, "2.3.4rc1") config = metadata(project) + (project / ".gitignore").write_text("*.so\n*.dll\n*.dylib\ncompanions/\nlicenses/\n") + (project / "unlisted.txt").write_text("Must not enter explicit sdists") + (project / "tests").mkdir(exist_ok=True) + (project / "tests/test_not_for_distribution.py").write_text("# Source-only test fixture") + (project / package / "tests").mkdir() + (project / package / "tests/test_not_for_distribution.py").write_text("# Must not leak via force inclusion") + (project / package / "__pycache__").mkdir() + (project / package / "__pycache__/clap.pyc").write_bytes(b"cache fixture") assert config["project"]["name"] == package.replace("_", "-") assert config["project"]["version"] == "2.3.4rc1" assert config["project"]["dependencies"] == [] @@ -119,7 +92,7 @@ def test_wheel_and_wrapper_only_sdist(tmp_path, backend): tools = tmp_path / "tools" tools.mkdir() marker = tmp_path / "compiler-called" - for tool in ("cmake", "ninja", "cc", "c++"): + for tool in ("cmake", "ninja", "cc", "c++", "gcc", "g++", "clang", "clang++", "cl"): executable = tools / tool executable.write_text(f'#!/bin/sh\nprintf called > "{marker}"\nexit 91\n') executable.chmod(0o755) @@ -128,22 +101,20 @@ def test_wheel_and_wrapper_only_sdist(tmp_path, backend): metadata_dir = project / "metadata" metadata_dir.mkdir() result = subprocess.run( - [sys.executable, "-c", "from hatchling.build import prepare_metadata_for_build_wheel; " + [sys.executable, "-c", "from scikit_build_core.build import prepare_metadata_for_build_wheel; " "prepare_metadata_for_build_wheel('metadata')"], cwd=project, env=env, capture_output=True, text=True, ) assert result.returncode == 0, result.stdout + result.stderr assert list(metadata_dir.glob("*.dist-info/METADATA")) - result = build(project, out, "wheel", env) - assert result.returncode != 0 and "Missing staged external runtime" in result.stdout, result.stdout - assert not list(out.glob("*.whl")) + # Validation belongs to Actions, not a replacement Python backend/hook. + # Prefer an actual externally compiled runtime; fixture-only jobs still test copying. native = project / package / RUNTIME_NAME - native.touch() - result = build(project, out, "wheel", env) - assert result.returncode != 0 and "Missing staged external runtime" in result.stdout, result.stdout - native.write_bytes(b"packaging fixture only, NOT an executable runtime") - result = build(project, out, "wheel", env) - assert result.returncode != 0 and "Missing runtime notices" in result.stdout, result.stdout + external = os.environ.get("CLAP_TEST_EXTERNAL_RUNTIME") + if external: + shutil.copy2(Path(external) / RUNTIME_NAME, native) + else: + native.write_bytes(b"packaging fixture only, NOT an executable runtime") notices = project / package / "licenses" notices.mkdir() shutil.copy2(project / "LICENSE", notices / "clap-cpp.txt") @@ -165,21 +136,22 @@ def test_wheel_and_wrapper_only_sdist(tmp_path, backend): assert f"{package}/licenses/dependency.txt" in entries for name in ("__init__.py", "clap.py", "clap.pyi", "py.typed"): assert f"{package}/{name}" in entries - assert not any("__pycache__" in n for n in entries) + assert not any("__pycache__" in n or "tests" in Path(n).parts for n in entries) result = build(project, out, "sdist", env) assert result.returncode == 0, result.stdout with tarfile.open(next(out.glob("*.tar.gz"))) as archive: entries = {n.split("/", 1)[1] for n in archive.getnames() if "/" in n} - required = {"pyproject.toml", "pyproject.toml.j2", "README.md", "LICENSE", "hatch_build.py", + required = {"pyproject.toml", "pyproject.toml.j2", "README.md", "LICENSE", *(f"{package}/{name}" for name in ("__init__.py", "clap.py", "clap.pyi", "py.typed"))} assert required <= entries assert not any(n.startswith("native/") or "licenses/" in n or "companions/" in n or n.endswith((".so", ".dll", ".dylib", ".cpp")) for n in entries) - assert not {"stage_runtime.py", "prepare_backend.py", "_build_backend.py"} & entries + assert not any("__pycache__" in n or n.endswith(".pyc") for n in entries) + assert "unlisted.txt" not in entries + assert not any("tests" in Path(n).parts for n in entries) + assert not {"stage_runtime.py", "prepare_backend.py", "_build_backend.py", "hatch_build.py"} & entries archive.extractall(tmp_path / "extracted", filter="data") source = next((tmp_path / "extracted").iterdir()) - result = build(source, tmp_path / "from-sdist", "wheel", env) - assert result.returncode != 0 and "Missing staged external runtime" in result.stdout, result.stdout shutil.copytree(project / package, source / package, dirs_exist_ok=True) result = build(source, tmp_path / "from-sdist", "wheel", env) assert result.returncode == 0, result.stdout @@ -188,11 +160,22 @@ def test_wheel_and_wrapper_only_sdist(tmp_path, backend): assert not marker.exists() -def test_editable_requires_runtime(tmp_path): - project, package = render(PROJECT, tmp_path / "project") - command = [sys.executable, "-c", "from hatchling.build import build_editable; build_editable('editable')"] - result = subprocess.run(command, cwd=project, capture_output=True, text=True) - assert result.returncode != 0 and "Missing staged external runtime" in result.stderr - (project / package / RUNTIME_NAME).write_bytes(b"editable fixture") - result = subprocess.run(command, cwd=project, capture_output=True, text=True) - assert result.returncode != 0 and "Missing runtime notices" in result.stderr +def test_editable_is_platform_tagged_without_compilation(tmp_path): + project, _ = render(PROJECT, tmp_path / "project") + result = subprocess.run( + [sys.executable, "-c", "from scikit_build_core.build import build_editable; build_editable('editable')"], + cwd=project, capture_output=True, text=True) + assert result.returncode == 0, result.stdout + result.stderr + wheel = next((project / "editable").glob("*.whl")) + assert "-py3-none-" in wheel.name and not wheel.name.endswith("-any.whl") + + +def test_macos_native_tag_and_minimum(monkeypatch): + from scikit_build_core.builder import wheel_tag + import sysconfig + sysconfig.get_config_vars() # Initialize real host configuration before mocking Darwin. + monkeypatch.setattr(wheel_tag.sys, "platform", "darwin") + # packaging uses the executing architecture, not a universal2 interpreter tag. + monkeypatch.setattr(wheel_tag.packaging.tags.platform, "mac_ver", lambda: ("15.0", ("", "", ""), "arm64")) + tag = wheel_tag.WheelTag.compute_best([], "py3", env={"MACOSX_DEPLOYMENT_TARGET": "11.0"}) + assert str(tag) == "py3-none-macosx_11_0_arm64" From ffa1e9c5514f094cf16617d137b27a965923e579 Mon Sep 17 00:00:00 2001 From: zhenghaoz Date: Sat, 7 Nov 2026 03:30:41 +0800 Subject: [PATCH 4/4] fix(python): exclude Jinja templates from published archives --- .github/RELEASING.md | 15 +++++++----- .github/scripts/test_python_release_assets.py | 21 ++++++++++++++--- .github/workflows/release.yml | 19 ++++++++++----- python/README.md | 23 +++++++++++-------- python/pyproject.toml | 6 ++--- python/pyproject.toml.j2 | 6 ++--- python/tests/test_packaging.py | 17 ++++++++++---- 7 files changed, 72 insertions(+), 35 deletions(-) diff --git a/.github/RELEASING.md b/.github/RELEASING.md index 04baa6d..e8516ad 100644 --- a/.github/RELEASING.md +++ b/.github/RELEASING.md @@ -32,11 +32,13 @@ validates archive contents, platform constraints and versions before publishing. Node inference tests run against the checkout and its platform workspace in CI. No Python wheel extraction is involved. All native packages publish before the wrapper. -Python publishes canonical wrapper-only source archives including the shared -`pyproject.toml.j2` template and direct scikit-build-core metadata; -neither sdists nor wheels contain tests. Source archives contain neither native -sources nor compiled libraries. Release jobs render -the template with Jinja2 for CPU, CUDA, ROCm and Vulkan, preserving exact versions. +Python publishes canonical wrapper-only source archives with rendered direct +scikit-build-core metadata; neither sdists nor wheels contain tests or `.j2` +templates. Source archives contain neither native sources nor compiled libraries. +Release jobs render the repository's shared `pyproject.toml.j2` template with +Jinja2 for CPU, CUDA, ROCm and Vulkan from the exact `${{ github.sha }}` checkout. +GPU projects reuse wrapper bytes and the version from the canonical CPU sdist; +the template is read from the checkout, not from that archive. CPU wheel jobs extract the base archive and directly copy the downloaded `clap_cpp/` runtime contents and license into its package. scikit-build-core never compiles native code (`wheel.cmake=false`, `sdist.cmake=false`); no custom hook/backend @@ -44,7 +46,8 @@ is used. Actions checks runtime binary headers and nonempty runtime notices befo assembly and checks the archived payload again before publication. The direct backend alone does not reject unstaged projects; never publish their unusable runtime-free wheels. The wheel job checks out `${{ github.sha }}` and copies tests -separately into its working source directory (not into published sdists/wheels). +and the template separately into its working source directory for packaging tests +(not into published sdists/wheels). cibuildwheel runs installed-wheel tests for Python 3.9, 3.10, 3.11, 3.12, 3.13 and 3.14. ABI-independent `py3-none` wheels are reused across these interpreters; Linux wheels retain auditwheel-repaired manylinux tags. diff --git a/.github/scripts/test_python_release_assets.py b/.github/scripts/test_python_release_assets.py index 56c5b9b..d69b9cf 100644 --- a/.github/scripts/test_python_release_assets.py +++ b/.github/scripts/test_python_release_assets.py @@ -45,6 +45,8 @@ def unit_wheel(root, distribution="clap_cpp", platform=PLATFORMS[0], interpreter archive.writestr(f"{distribution}/licenses/clap-cpp.txt", "" if payload == "empty-notice" else "UNIT FIXTURE notice") if payload == "tests": archive.writestr(f"{distribution}/tests/test_leaked.py", "# TEST FIXTURE") + if payload == "template": + archive.writestr(f"{distribution}/unused.j2", "UNIT FIXTURE template") return path @@ -65,7 +67,7 @@ def setUp(self): def unit_sdist(self, source, distribution, omitted=None, extra=()): """Write wrapper-only UNIT FIXTURE source archives for the real heredoc.""" with tarfile.open(source, "w:gz") as archive: - for name in ("pyproject.toml", "pyproject.toml.j2", "README.md", "LICENSE", + for name in ("pyproject.toml", "README.md", "LICENSE", *(f"{distribution}/{name}" for name in ("__init__.py", "clap.py", "clap.pyi", "py.typed")), *extra): if name == omitted: continue @@ -74,9 +76,9 @@ def unit_sdist(self, source, distribution, omitted=None, extra=()): entry.size = len(data) archive.addfile(entry, io.BytesIO(data)) - def test_unit_fixture_sdist_requires_wrapper_metadata_and_template(self): + def test_unit_fixture_sdist_requires_wrapper_and_rendered_metadata(self): for distribution in DISTRIBUTIONS: - for required in ("pyproject.toml", "pyproject.toml.j2", "README.md", "LICENSE", + for required in ("pyproject.toml", "README.md", "LICENSE", *(f"{distribution}/{name}" for name in ("__init__.py", "clap.py", "clap.pyi", "py.typed"))): with self.subTest(distribution=distribution, required=required): source = self.artifacts / f"{distribution}-0.1.0.tar.gz" @@ -102,6 +104,19 @@ def test_published_archives_reject_tests(self): self.assert_rejected("Wheel contains tests") unit_wheel(self.artifacts, distribution=distribution) + def test_published_archives_reject_jinja_templates(self): + for distribution in DISTRIBUTIONS: + for template in ("pyproject.toml.j2", f"{distribution}/unused.j2"): + with self.subTest(distribution=distribution, template=template): + source = self.artifacts / f"{distribution}-0.1.0.tar.gz" + self.unit_sdist(source, distribution, extra=(template,)) + self.assert_rejected("Sdist contains Jinja template") + self.unit_sdist(source, distribution) + with self.subTest(distribution=distribution, archive="wheel"): + unit_wheel(self.artifacts, distribution=distribution, payload="template") + self.assert_rejected("Wheel contains Jinja template") + unit_wheel(self.artifacts, distribution=distribution) + def run_validator(self): shutil.rmtree(self.root / "dist", ignore_errors=True) return subprocess.run([sys.executable, "-c", validator_source()], cwd=self.root, capture_output=True, text=True) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 0451940..6605cde 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -15,6 +15,7 @@ jobs: steps: - uses: actions/checkout@v4 with: + ref: ${{ github.sha }} submodules: recursive - uses: actions/setup-python@v5 with: @@ -50,7 +51,7 @@ jobs: destination="$RUNNER_TEMP/clap-$backend" package="clap_cpp_$backend" mkdir -p "$destination/$package" - cp python-dist/{README.md,LICENSE,pyproject.toml.j2} "$destination/" + cp python-dist/{README.md,LICENSE} "$destination/" cp python-dist/clap_cpp/{clap.py,clap.pyi,py.typed} "$destination/$package/" printf '"""Native %s runtime; import clap_cpp for the public API."""\n' "$backend" > "$destination/$package/__init__.py" BACKEND="$backend" DESTINATION="$destination" python - <<'PY' @@ -61,7 +62,7 @@ jobs: source = Path('python-dist') version = tomllib.loads((source / 'pyproject.toml').read_text(encoding='utf-8'))['project']['version'] template = Environment(undefined=StrictUndefined, keep_trailing_newline=True).from_string( - (source / 'pyproject.toml.j2').read_text(encoding='utf-8')) + Path('python/pyproject.toml.j2').read_text(encoding='utf-8')) (Path(os.environ['DESTINATION']) / 'pyproject.toml').write_text( template.render(backend=os.environ['BACKEND'], version=version), encoding='utf-8') PY @@ -183,9 +184,11 @@ jobs: if not notice.is_file() or not notice.read_text(encoding="utf-8").strip(): raise SystemExit(f"Missing runtime notices: {notice}") PY - - name: Supply installed-wheel tests from the exact release checkout + - name: Supply installed-wheel tests and template from the exact release checkout shell: bash - run: cp -R python/tests python-dist/tests + run: | + cp -R python/tests python-dist/tests + cp python/pyproject.toml.j2 python-dist/pyproject.toml.j2 - name: Package and test CPU wheels from canonical wrapper sdist and shared runtime uses: pypa/cibuildwheel@v4.3.4 with: @@ -611,6 +614,8 @@ jobs: with zipfile.ZipFile(wheel) as archive: package = name.replace('-', '_') entries = set(archive.namelist()) + if any(entry.endswith('.j2') for entry in entries): + raise SystemExit(f'Wheel contains Jinja template: {wheel.name}') if any('tests' in pathlib.PurePosixPath(entry).parts for entry in entries): raise SystemExit(f'Wheel contains tests: {wheel.name}') if f'{package}/clap.py' not in entries: @@ -644,7 +649,9 @@ jobs: package = name.replace('-', '_') if any('tests' in pathlib.PurePosixPath(entry).parts for entry in entries): raise SystemExit(f'Sdist contains tests: {source.name}') - required = {'pyproject.toml', 'pyproject.toml.j2', 'README.md', 'LICENSE', + if any(entry.endswith('.j2') for entry in entries): + raise SystemExit(f'Sdist contains Jinja template: {source.name}') + required = {'pyproject.toml', 'README.md', 'LICENSE', f'{package}/__init__.py', f'{package}/clap.py', f'{package}/clap.pyi', f'{package}/py.typed'} if not required <= entries: raise SystemExit(f'Sdist missing wrapper or packaging input: {source.name}: {required - entries}') @@ -682,7 +689,7 @@ jobs: ROCm GPU targets: gfx908,gfx90a,gfx942,gfx1030,gfx1100,gfx1101,gfx1200,gfx1201. Linux wheels are auditwheel-repaired manylinux wheels; the external GPU runtimes above remain deliberate exceptions, not portability promises. - Source archives contain wrappers, a Jinja metadata template and direct backend metadata, not native sources. + Source archives contain wrappers and rendered direct backend metadata, not Jinja templates or native sources. Compile native runtimes separately from a recursive checkout, then stage them before packaging; Python wheel builds do not run CMake. CPU wheels pass installed-wheel tests on Python 3.9-3.14; GPU execution is not tested here. diff --git a/python/README.md b/python/README.md index fed28dd..cec0bbe 100644 --- a/python/README.md +++ b/python/README.md @@ -149,9 +149,12 @@ never label a single-architecture runtime universal2. `pyproject.toml.j2` is the shared CPU/CUDA/ROCm/Vulkan metadata template. The checked-in `pyproject.toml` is its CPU rendering for ordinary local builds. +The template stays in the repository; wheels and sdists contain only rendered +metadata, never `.j2` files. Render with `backend=cpu|cuda|rocm|vulkan` and one PEP 440 `version` (default `0.1.0`); CPU extras pin that exact version of each backend distribution. -For example, using the canonical CPU sdist extracted into `python-dist`: +For example, using the canonical CPU sdist extracted into `python-dist` and +`python/pyproject.toml.j2` from the exact checkout that produced that archive: ```sh python -m pip install jinja2 @@ -159,7 +162,7 @@ backend=cuda # or rocm, vulkan project="build/python-$backend" package="clap_cpp_$backend" mkdir -p "$project/$package" -cp python-dist/{README.md,LICENSE,pyproject.toml.j2} "$project/" +cp python-dist/{README.md,LICENSE} "$project/" cp python-dist/clap_cpp/{clap.py,clap.pyi,py.typed} "$project/$package/" printf '"""Native %s runtime; import clap_cpp for the public API."""\n' "$backend" > "$project/$package/__init__.py" BACKEND="$backend" PROJECT="$project" python - <<'PY' @@ -173,7 +176,7 @@ except ImportError: source = Path('python-dist') version = tomllib.loads((source / 'pyproject.toml').read_text())['project']['version'] template = Environment(undefined=StrictUndefined, keep_trailing_newline=True).from_string( - (source / 'pyproject.toml.j2').read_text()) + Path('python/pyproject.toml.j2').read_text()) (Path(os.environ['PROJECT']) / 'pyproject.toml').write_text( template.render(backend=os.environ['BACKEND'], version=version)) PY @@ -200,14 +203,16 @@ python -m pip install pytest mypy typing_extensions build 'scikit-build-core>=1. python -m pytest python/tests ``` -Source distributions contain wrapper/typing files, `LICENSE` and -metadata/template files, not native C++ snapshots, staged binaries, build hooks -or native build definitions. Tests are excluded from both wheels and sdists; release -CI supplies them separately from the exact release checkout and tests the installed -wheel on each supported interpreter. Plain `python -m build python` builds from an +Source distributions contain wrapper/typing files, `LICENSE` and rendered +metadata, not Jinja templates, native C++ snapshots, staged binaries, build hooks +or native build definitions. Tests and `.j2` files are excluded from both wheels +and sdists; release CI supplies tests and the metadata template separately from +the exact release checkout and tests the installed wheel on each supported +interpreter. Plain `python -m build python` builds from an unstaged sdist and therefore produces an unusable runtime-free wheel; do not publish it. Use the explicit `--sdist`, unpack, stage, `--wheel` sequence above. -Backend projects can also be rendered from the extracted CPU sdist before staging +Backend projects reuse wrappers and the version from the extracted CPU sdist, +rendering metadata with the template from that same checkout before staging their runtimes. No NumPy, Python development headers or pybind11 are required. ## Releases diff --git a/python/pyproject.toml b/python/pyproject.toml index bbd3670..97e52a9 100644 --- a/python/pyproject.toml +++ b/python/pyproject.toml @@ -25,12 +25,12 @@ wheel.cmake = false wheel.platlib = true wheel.py-api = "py3" wheel.packages = [] -wheel.exclude = ["**/__pycache__/**", "**/*.pyc", "**/.pytest_cache/**", "**/tests/**"] +wheel.exclude = ["**/__pycache__/**", "**/*.pyc", "**/.pytest_cache/**", "**/tests/**", "**/*.j2"] sdist.cmake = false -sdist.exclude = ["**/__pycache__/**", "**/*.pyc", "**/.pytest_cache/**", "**/tests/**"] +sdist.exclude = ["**/__pycache__/**", "**/*.pyc", "**/.pytest_cache/**", "**/tests/**", "**/*.j2"] sdist.inclusion-mode = "explicit" # Never ship staged binaries, companion data or runtime notices in source archives. -sdist.include = ["/pyproject.toml", "/pyproject.toml.j2", "/README.md", "/LICENSE", "/clap_cpp/*.py", "/clap_cpp/*.pyi", "/clap_cpp/py.typed"] +sdist.include = ["/pyproject.toml", "/README.md", "/LICENSE", "/clap_cpp/*.py", "/clap_cpp/*.pyi", "/clap_cpp/py.typed"] [tool.scikit-build.wheel.force-include] # Include Git-ignored external payloads without broadening the source archive. diff --git a/python/pyproject.toml.j2 b/python/pyproject.toml.j2 index 4ecd4a2..b60f194 100644 --- a/python/pyproject.toml.j2 +++ b/python/pyproject.toml.j2 @@ -28,12 +28,12 @@ wheel.cmake = false wheel.platlib = true wheel.py-api = "py3" wheel.packages = [] -wheel.exclude = ["**/__pycache__/**", "**/*.pyc", "**/.pytest_cache/**", "**/tests/**"] +wheel.exclude = ["**/__pycache__/**", "**/*.pyc", "**/.pytest_cache/**", "**/tests/**", "**/*.j2"] sdist.cmake = false -sdist.exclude = ["**/__pycache__/**", "**/*.pyc", "**/.pytest_cache/**", "**/tests/**"] +sdist.exclude = ["**/__pycache__/**", "**/*.pyc", "**/.pytest_cache/**", "**/tests/**", "**/*.j2"] sdist.inclusion-mode = "explicit" # Never ship staged binaries, companion data or runtime notices in source archives. -sdist.include = ["/pyproject.toml", "/pyproject.toml.j2", "/README.md", "/LICENSE", "/{{ package }}/*.py", "/{{ package }}/*.pyi", "/{{ package }}/py.typed"] +sdist.include = ["/pyproject.toml", "/README.md", "/LICENSE", "/{{ package }}/*.py", "/{{ package }}/*.pyi", "/{{ package }}/py.typed"] [tool.scikit-build.wheel.force-include] # Include Git-ignored external payloads without broadening the source archive. diff --git a/python/tests/test_packaging.py b/python/tests/test_packaging.py index 844a37b..e61db96 100644 --- a/python/tests/test_packaging.py +++ b/python/tests/test_packaging.py @@ -27,12 +27,13 @@ def metadata(path=PROJECT): return tomllib.loads((path / "pyproject.toml").read_text(encoding="utf-8")) -def render(source, destination, backend="cpu", version=None): +def render(source, destination, backend="cpu", version=None, template_source=PROJECT): """Mirror the workflow's Jinja rendering and plain wrapper copies.""" destination.mkdir(parents=True) package = "clap_cpp" + (f"_{backend}" if backend != "cpu" else "") - for name in ("README.md", "LICENSE", "pyproject.toml.j2"): + for name in ("README.md", "LICENSE"): shutil.copy2(source / name, destination / name) + shutil.copy2(template_source / "pyproject.toml.j2", destination / "pyproject.toml.j2") shutil.copytree(source / "clap_cpp", destination / package, ignore=shutil.ignore_patterns("__pycache__", "*.so", "*.dylib", "*.dll", "licenses")) if backend != "cpu": @@ -41,7 +42,7 @@ def render(source, destination, backend="cpu", version=None): else: shutil.copytree(source / "tests", destination / "tests") template = Environment(undefined=StrictUndefined, keep_trailing_newline=True).from_string( - (source / "pyproject.toml.j2").read_text(encoding="utf-8")) + (template_source / "pyproject.toml.j2").read_text(encoding="utf-8")) (destination / "pyproject.toml").write_text(template.render( backend=backend, version=version or metadata(source)["project"]["version"]), encoding="utf-8") return destination, package @@ -78,6 +79,7 @@ def test_wheel_and_wrapper_only_sdist(tmp_path, backend): (project / "tests/test_not_for_distribution.py").write_text("# Source-only test fixture") (project / package / "tests").mkdir() (project / package / "tests/test_not_for_distribution.py").write_text("# Must not leak via force inclusion") + (project / package / "unused.j2").write_text("Must not leak via force inclusion") (project / package / "__pycache__").mkdir() (project / package / "__pycache__/clap.pyc").write_bytes(b"cache fixture") assert config["project"]["name"] == package.replace("_", "-") @@ -137,11 +139,12 @@ def test_wheel_and_wrapper_only_sdist(tmp_path, backend): for name in ("__init__.py", "clap.py", "clap.pyi", "py.typed"): assert f"{package}/{name}" in entries assert not any("__pycache__" in n or "tests" in Path(n).parts for n in entries) + assert not any(n.endswith(".j2") for n in entries) result = build(project, out, "sdist", env) assert result.returncode == 0, result.stdout with tarfile.open(next(out.glob("*.tar.gz"))) as archive: entries = {n.split("/", 1)[1] for n in archive.getnames() if "/" in n} - required = {"pyproject.toml", "pyproject.toml.j2", "README.md", "LICENSE", + required = {"pyproject.toml", "README.md", "LICENSE", *(f"{package}/{name}" for name in ("__init__.py", "clap.py", "clap.pyi", "py.typed"))} assert required <= entries assert not any(n.startswith("native/") or "licenses/" in n or "companions/" in n or @@ -149,6 +152,7 @@ def test_wheel_and_wrapper_only_sdist(tmp_path, backend): assert not any("__pycache__" in n or n.endswith(".pyc") for n in entries) assert "unlisted.txt" not in entries assert not any("tests" in Path(n).parts for n in entries) + assert not any(n.endswith(".j2") for n in entries) assert not {"stage_runtime.py", "prepare_backend.py", "_build_backend.py", "hatch_build.py"} & entries archive.extractall(tmp_path / "extracted", filter="data") source = next((tmp_path / "extracted").iterdir()) @@ -156,7 +160,10 @@ def test_wheel_and_wrapper_only_sdist(tmp_path, backend): result = build(source, tmp_path / "from-sdist", "wheel", env) assert result.returncode == 0, result.stdout if backend == "cpu": - render(source, tmp_path / "cuda-from-cpu-sdist", "cuda") + assert not (source / "pyproject.toml.j2").exists() + gpu, gpu_package = render(source, tmp_path / "cuda-from-cpu-sdist", "cuda", template_source=PROJECT) + for name in ("clap.py", "clap.pyi", "py.typed"): + assert (gpu / gpu_package / name).read_bytes() == (source / "clap_cpp" / name).read_bytes() assert not marker.exists()