diff --git a/.github/workflows/pygosqlx-release.yml b/.github/workflows/pygosqlx-release.yml new file mode 100644 index 00000000..de93e3d6 --- /dev/null +++ b/.github/workflows/pygosqlx-release.yml @@ -0,0 +1,306 @@ +# Build, test and publish pygosqlx (python/) wheels to PyPI. +# +# What runs when: +# pull_request (python/**, pkg/**, go.mod, go.sum, this file) +# -> build + test wheels and sdist only. Nothing is published. +# pkg/** because pkg/cbinding links the whole parser via pkg/gosqlx. +# push of a tag matching 'pygosqlx-v*' (e.g. pygosqlx-v0.3.0) +# -> build + test, then publish to PyPI. The tag must match the version +# in python/pyproject.toml and must point at a commit on main. +# This is the ONLY path to PyPI, so every published version has a tag. +# workflow_dispatch +# -> build + test, then publish to the index chosen by the 'target' +# input: 'testpypi' (default) or 'none' (build only). Publishing +# from a dispatch is only allowed on main. TestPyPI skips files that +# already exist, so repeated dry runs of one version do not fail. +# +# Tag isolation: the main GoSQLX 'v*' release tags never trigger this +# workflow, and pygosqlx-v* tags must not disturb the Go release: +# release.yml pins GoReleaser's current/previous tag to the 'v*' series, and +# release.yml, vscode-publish.yml and bench-compare.yml filter on 'v*'. +# +# Wheels: one py3-none- wheel per OS/arch (the bundled libgosqlx is +# loaded through ctypes and does not depend on the CPython ABI), each tested +# on CPython 3.10-3.14 against the installed wheel: +# linux x86_64, aarch64 (built in manylinux_2_28, native runners; +# auditwheel repair pins manylinux_2_17) +# macOS x86_64, arm64 (macOS 13+) +# windows AMD64 +# +# Publishing uses PyPI Trusted Publishing (OIDC); there is no API token. +# ONE-TIME MAINTAINER SETUP (not automated, do this before the first tag): +# 1. On https://pypi.org/manage/account/publishing/ add a *pending* trusted +# publisher: PyPI project name 'pygosqlx', owner 'ajitpratap0', +# repository 'GoSQLX', workflow 'pygosqlx-release.yml', environment +# 'pypi'. +# 2. In GitHub repo Settings -> Environments create an environment named +# 'pypi' (recommended: deployment policy limited to tags 'pygosqlx-v*' +# and a required reviewer). Also recommended: a tag ruleset restricting +# who may create 'pygosqlx-v*' tags. +# 3. Optional, for dry runs: repeat 1 on https://test.pypi.org with +# environment 'testpypi', and create a 'testpypi' GitHub environment. +# Then release with: git tag -a pygosqlx-vX.Y.Z && git push origin pygosqlx-vX.Y.Z + +name: pygosqlx Release + +on: + pull_request: + branches: [ main ] + paths: + - 'python/**' + - 'pkg/**' + - 'go.mod' + - 'go.sum' + - '.github/workflows/pygosqlx-release.yml' + push: + tags: + - 'pygosqlx-v*' + workflow_dispatch: + inputs: + target: + description: 'Where to publish the built distributions' + type: choice + options: + - testpypi + - none + default: testpypi + +permissions: + contents: read + +concurrency: + group: pygosqlx-release-${{ github.ref }} + cancel-in-progress: ${{ github.event_name == 'pull_request' }} + +env: + # Go toolchain for the macOS and Windows runners. Must equal GO_VERSION in + # python/scripts/install-go-linux.sh (used in manylinux); the sdist job + # fails if they differ. + GO_VERSION: '1.26.10' + +jobs: + wheels: + name: wheels (${{ matrix.name }}) + runs-on: ${{ matrix.os }} + timeout-minutes: 45 + strategy: + fail-fast: false + matrix: + include: + - name: linux-x86_64 + os: ubuntu-24.04 + arch: x86_64 + - name: linux-aarch64 + os: ubuntu-24.04-arm + arch: aarch64 + - name: macos-x86_64 + os: macos-15-intel + - name: macos-arm64 + os: macos-15 + - name: windows-amd64 + os: windows-2025 + + steps: + - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4.4.0 + with: + persist-credentials: false + + # Linux builds run inside the manylinux container and install Go there + # (CIBW_BEFORE_ALL_LINUX); macOS and Windows build on the runner. + - name: Set up Go + if: runner.os != 'Linux' + uses: actions/setup-go@40f1582b2485089dde7abd97c1529aa768e1baff # v5.6.0 + with: + go-version: ${{ env.GO_VERSION }} + cache: false + + - name: Build and test wheels + uses: pypa/cibuildwheel@adba99c781d3746d38b3a09b1cf797965bcb7fe6 # v4.3.0 + with: + package-dir: python + output-dir: wheelhouse + env: + # The wheel is py3-none-: cibuildwheel builds it once with + # the first Python and reuses it (test only) for the others. + CIBW_BUILD: 'cp310-* cp311-* cp312-* cp313-* cp314-*' + CIBW_SKIP: '*-musllinux_*' + CIBW_ARCHS: native + CIBW_BUILD_VERBOSITY: 1 + + # auditwheel would tag x86_64 as old as manylinux_2_5 (the Go + # library uses only ancient glibc symbols), claiming distros that + # were never tested and that Go 1.26 (kernel 3.2+) may not run on. + # Pin the floor to manylinux_2_17 (manylinux2014) on both arches. + # cibuildwheel does not expand an {arch} placeholder here, so the + # architecture comes from the matrix (empty on macOS/Windows, where + # this Linux-only command is unused). + CIBW_REPAIR_WHEEL_COMMAND_LINUX: auditwheel repair --plat manylinux_2_17_${{ matrix.arch }} -w {dest_dir} {wheel} + CIBW_BEFORE_ALL_LINUX: bash {project}/python/scripts/install-go-linux.sh + CIBW_BEFORE_BUILD_LINUX: cd {project}/pkg/cbinding && bash build.sh + CIBW_BEFORE_BUILD_MACOS: cd {project}/pkg/cbinding && bash build.sh + CIBW_BEFORE_BUILD_WINDOWS: >- + cd /d "{project}\pkg\cbinding" && + go build -buildmode=c-shared -o ..\..\python\pygosqlx\lib\libgosqlx.dll . + + # GOFLAGS -trimpath (all platforms): keep absolute build and module + # cache paths out of the published binaries. + # -buildvcs=false: the container's copy of the checkout is owned by a + # different user, which makes Go's VCS stamping fail. + CIBW_ENVIRONMENT_LINUX: >- + PATH="/usr/local/go/bin:$PATH" + GOTOOLCHAIN=local + GOFLAGS="-buildvcs=false -trimpath" + CGO_ENABLED=1 + PYGOSQLX_TEST_INSTALLED=1 + # Go 1.26 targets macOS 12+, and cgo objects must not be built for a + # newer macOS than the wheel tag claims. + CIBW_ENVIRONMENT_MACOS: >- + MACOSX_DEPLOYMENT_TARGET=13.0 + CGO_CFLAGS="-O2 -g -mmacosx-version-min=13.0" + CGO_LDFLAGS="-O2 -g -mmacosx-version-min=13.0" + GOTOOLCHAIN=local + GOFLAGS=-trimpath + CGO_ENABLED=1 + PYGOSQLX_TEST_INSTALLED=1 + CIBW_ENVIRONMENT_WINDOWS: >- + GOTOOLCHAIN=local + GOFLAGS=-trimpath + CGO_ENABLED=1 + PYGOSQLX_TEST_INSTALLED=1 + + # Run the suite against the installed wheel, not the checkout: + # PYGOSQLX_TEST_INSTALLED=1 keeps python/ off sys.path and enables + # tests/test_installed_wheel.py, which fails if the bundled library + # is missing or the checkout was imported. + CIBW_TEST_REQUIRES: pytest + CIBW_TEST_COMMAND: >- + python -m pytest --import-mode=importlib -p no:cacheprovider + {project}/python/tests -v -rs + + - name: Upload wheels + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2 + with: + name: pygosqlx-wheels-${{ matrix.name }} + path: wheelhouse/*.whl + if-no-files-found: error + + sdist: + name: sdist + runs-on: ubuntu-24.04 + timeout-minutes: 15 + steps: + - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4.4.0 + with: + persist-credentials: false + # Full history so a release tag can be checked against main. + fetch-depth: 0 + + - name: Check Go version pins agree + run: | + script_go="$(sed -n 's/^GO_VERSION="\(.*\)"$/\1/p' python/scripts/install-go-linux.sh)" + if [ -z "$script_go" ] || [ "$script_go" != "$GO_VERSION" ]; then + echo "::error::GO_VERSION mismatch: workflow '$GO_VERSION' vs install-go-linux.sh '$script_go'" + exit 1 + fi + + # Defense in depth: never publish a tag whose commit is not on main. + - name: Check tagged commit is on main + if: startsWith(github.ref, 'refs/tags/pygosqlx-v') + run: | + if ! git merge-base --is-ancestor "$GITHUB_SHA" origin/main; then + echo "::error::$GITHUB_SHA ($GITHUB_REF_NAME) is not an ancestor of origin/main" + exit 1 + fi + + - name: Set up Python + uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0 + with: + python-version: '3.13' + + - name: Check tag matches package version + if: startsWith(github.ref, 'refs/tags/pygosqlx-v') + env: + TAG: ${{ github.ref_name }} + run: | + pkg_version="$(python -c 'import tomllib; print(tomllib.load(open("python/pyproject.toml", "rb"))["project"]["version"])')" + if [ "${TAG#pygosqlx-v}" != "$pkg_version" ]; then + echo "::error::tag $TAG does not match python/pyproject.toml version $pkg_version" + exit 1 + fi + + - name: Build sdist + run: | + python -m pip install --upgrade build twine + python -m build --sdist --outdir dist python + python -m twine check --strict dist/* + + # The sdist carries only the Python sources. It must never ship a + # prebuilt library for some other platform. + - name: Check sdist has no native binaries + run: | + if tar tzf dist/*.tar.gz | grep -E '\.(so|dylib|dll)$'; then + echo "::error::sdist contains a native library" + exit 1 + fi + + - name: Upload sdist + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2 + with: + name: pygosqlx-sdist + path: dist/*.tar.gz + if-no-files-found: error + + publish-pypi: + name: publish to PyPI + needs: [ wheels, sdist ] + # Only a tag *push* publishes to PyPI. A workflow_dispatch run on a tag + # ref also has github.ref == refs/tags/pygosqlx-v*, so the event check is + # what keeps 'target: none' / 'testpypi' dispatches off real PyPI. + if: github.event_name == 'push' && startsWith(github.ref, 'refs/tags/pygosqlx-v') + runs-on: ubuntu-24.04 + timeout-minutes: 15 + environment: + name: pypi + url: https://pypi.org/p/pygosqlx + permissions: + id-token: write # PyPI Trusted Publishing (OIDC) + steps: + - name: Download distributions + uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4.3.0 + with: + pattern: pygosqlx-* + path: dist + merge-multiple: true + + - name: Publish to PyPI + uses: pypa/gh-action-pypi-publish@dc37677b2e1c63e2034f94d8a5b11f265b73ba33 # v1.14.2 + + publish-testpypi: + name: publish to TestPyPI + needs: [ wheels, sdist ] + if: >- + github.event_name == 'workflow_dispatch' && + inputs.target == 'testpypi' && + github.ref == 'refs/heads/main' + runs-on: ubuntu-24.04 + timeout-minutes: 15 + environment: + name: testpypi + url: https://test.pypi.org/p/pygosqlx + permissions: + id-token: write # TestPyPI Trusted Publishing (OIDC) + steps: + - name: Download distributions + uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4.3.0 + with: + pattern: pygosqlx-* + path: dist + merge-multiple: true + + - name: Publish to TestPyPI + uses: pypa/gh-action-pypi-publish@dc37677b2e1c63e2034f94d8a5b11f265b73ba33 # v1.14.2 + with: + repository-url: https://test.pypi.org/legacy/ + # The version only changes per release, so repeated dry runs would + # otherwise fail on files already uploaded. TestPyPI only. + skip-existing: true diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index b1cac69e..31f4564f 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -40,6 +40,22 @@ jobs: CGO_ENABLED: '1' run: task test:cbinding + # pygosqlx-v* tags (Python package releases, see pygosqlx-release.yml) + # live on main too. GoReleaser OSS derives the previous tag with + # `git describe --tags`, which would pick a pygosqlx tag and produce a + # truncated changelog and a wrong compare link. Pin both tags to the + # 'v*' series explicitly (git.ignore_tags in .goreleaser.yml is Pro-only). + - name: Resolve release tags + env: + TAG: ${{ github.ref_name }} + run: | + prev="$(git describe --tags --abbrev=0 --match 'v[0-9]*' "${TAG}^" 2>/dev/null || true)" + echo "GORELEASER_CURRENT_TAG=${TAG}" >> "$GITHUB_ENV" + if [ -n "$prev" ]; then + echo "GORELEASER_PREVIOUS_TAG=${prev}" >> "$GITHUB_ENV" + fi + echo "current=${TAG} previous=${prev:-}" + - name: Run GoReleaser uses: goreleaser/goreleaser-action@e435ccd777264be153ace6237001ef4d979d3a7a # v6.4.0 with: diff --git a/python/LICENSE b/python/LICENSE new file mode 100644 index 00000000..d6456956 --- /dev/null +++ b/python/LICENSE @@ -0,0 +1,202 @@ + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/python/MANIFEST.in b/python/MANIFEST.in index 3c5022cf..67216b63 100644 --- a/python/MANIFEST.in +++ b/python/MANIFEST.in @@ -3,3 +3,4 @@ include README.md include pyproject.toml recursive-include pygosqlx *.py *.pyi py.typed recursive-include pygosqlx/lib *.so *.dylib *.dll *.h +recursive-include tests *.py diff --git a/python/README.md b/python/README.md index 5347de6f..a2b39777 100644 --- a/python/README.md +++ b/python/README.md @@ -6,37 +6,47 @@ Powered by Go's compiled performance, PyGoSQLX is 100x+ faster than pure Python ## Installation -### Step 1: Build the shared library - -Requires a Go toolchain matching the repository's `go.mod` (Go 1.26+) with cgo enabled. - ```bash -cd pkg/cbinding -./build.sh +pip install pygosqlx ``` -This compiles the Go code into a shared library (`libgosqlx.so` on Linux, `libgosqlx.dylib` on macOS, `libgosqlx.dll` on Windows) and places it in `python/pygosqlx/lib/`. +Prebuilt wheels bundle the native `libgosqlx` library, so no Go toolchain is +needed. They are published to PyPI from the `pygosqlx-v*` release tags; the +command above works once the first such tag (`pygosqlx-v0.3.0`) has been +published. Until then, build from source as shown below. -You can customize the output directory: +Wheels are available for Python 3.10-3.14 on: -```bash -OUTPUT_DIR=/path/to/output ./build.sh -``` +| Platform | Architectures | Wheel tag | +|---|---|---| +| Linux (glibc) | x86_64, aarch64 | `manylinux_2_17` (manylinux2014; glibc 2.17+, built on manylinux_2_28) | +| macOS 13+ | x86_64, arm64 | `macosx_13_0` | +| Windows | AMD64 | `win_amd64` | -### Step 2: Install the Python package +There are no wheels for musl-based Linux (Alpine) or 32-bit platforms. On +those, pip falls back to the source distribution, which installs the Python +package without the native library; build it as described below and point +`GOSQLX_LIB_PATH` at it. -```bash -cd python -pip install . -``` +### Building from source -For development: +Requires a Go toolchain matching the repository's `go.mod` (Go 1.26+) with cgo +enabled (a C compiler on `PATH`). ```bash -pip install -e ".[dev]" +git clone https://github.com/ajitpratap0/GoSQLX.git +cd GoSQLX/pkg/cbinding +./build.sh # writes python/pygosqlx/lib/libgosqlx.{so,dylib} +cd ../../python +pip install . # or: pip install -e ".[dev]" for development ``` -If the library is not bundled in the package, set the environment variable: +`build.sh` compiles the Go code into a shared library (`libgosqlx.so` on +Linux, `libgosqlx.dylib` on macOS, `libgosqlx.dll` on Windows via MSYS2/MinGW) +and places it in `python/pygosqlx/lib/`, where `pip install .` bundles it. Use +`OUTPUT_DIR=/path/to/output ./build.sh` to write it elsewhere. + +To use a library that is not bundled in the package, set: ```bash export GOSQLX_LIB_PATH=/path/to/libgosqlx.so @@ -250,11 +260,12 @@ All exceptions have `message`, `code`, `line`, and `column` attributes. ## Platform Support -| Platform | Library Extension | Status | -|---|---|---| -| Linux (x86_64, arm64) | `.so` | Supported | -| macOS (x86_64, arm64) | `.dylib` | Supported | -| Windows (via MSYS2/MinGW) | `.dll` | Supported | +| Platform | Library | Prebuilt wheel | From source | +|---|---|---|---| +| Linux x86_64, aarch64 (glibc) | `.so` | Yes (glibc 2.17+) | Yes | +| macOS x86_64, arm64 | `.dylib` | Yes (macOS 13+) | Yes | +| Windows AMD64 | `.dll` | Yes | Yes (MSYS2/MinGW) | +| Other (musl, 32-bit, ...) | - | No | Where Go supports `-buildmode=c-shared` | ## Performance @@ -269,25 +280,6 @@ This makes PyGoSQLX suitable for high-throughput use cases such as SQL proxies, ## Development -### Building from source - -```bash -# Clone the repository -git clone https://github.com/ajitpratap0/GoSQLX.git -cd GoSQLX - -# Build the shared library -cd pkg/cbinding -./build.sh -cd ../../python - -# Install in development mode -pip install -e ".[dev]" - -# Run tests -pytest -``` - ### Running tests ```bash @@ -298,6 +290,23 @@ pytest tests/test_integration.py # Integration tests (requires shared library) pytest --cov=pygosqlx # With coverage ``` +To run the suite against an installed wheel instead of the checkout (as the +release CI does): + +```bash +pip install pygosqlx pytest # or a locally built wheel +cd /tmp && PYGOSQLX_TEST_INSTALLED=1 python -m pytest --import-mode=importlib /path/to/GoSQLX/python/tests +``` + +### Releasing + +Wheels for every supported platform and an sdist are built and tested by +`.github/workflows/pygosqlx-release.yml` on each pull request that touches +`python/` or `pkg/`. Pushing a tag `pygosqlx-vX.Y.Z` (matching the version in +`pyproject.toml`, on a commit already on `main`) publishes them to PyPI +through Trusted Publishing; this is the only path to PyPI. A manual dispatch +can publish a dry run to TestPyPI. See the workflow header for the one-time PyPI setup. + ### Project structure ``` @@ -313,10 +322,13 @@ python/ test_package_version.py # Version metadata consistency test_integration.py # Integration tests with shared library test_dialect_and_analysis.py # dialect=, lint, security_scan, transpile, dialects + test_installed_wheel.py # Installed-wheel checks (PYGOSQLX_TEST_INSTALLED=1) + scripts/ + install-go-linux.sh # Pinned, checksum-verified Go install for manylinux CI pyproject.toml # Package metadata and build config - setup.py # Legacy build support + setup.py # Shim that tags wheels per platform (no metadata) ``` ## License -Apache-2.0. See the [LICENSE](../LICENSE) file in the repository root. +Apache-2.0. See the [LICENSE](https://github.com/ajitpratap0/GoSQLX/blob/main/LICENSE) file. diff --git a/python/pygosqlx/core.py b/python/pygosqlx/core.py index b37b8e1f..05eeeba1 100644 --- a/python/pygosqlx/core.py +++ b/python/pygosqlx/core.py @@ -209,7 +209,10 @@ def _get_lib_path() -> str: raise FileNotFoundError( f"GoSQLX shared library not found. " f"Searched: GOSQLX_LIB_PATH env var, {lib_path}. " - f"Build it with: cd pkg/cbinding && ./build.sh" + f"No prebuilt wheel bundles it for this platform; build it from " + f"source (cd pkg/cbinding && ./build.sh) and reinstall, or set " + f"GOSQLX_LIB_PATH. See 'Building from source' in " + f"https://github.com/ajitpratap0/GoSQLX/blob/main/python/README.md" ) diff --git a/python/pyproject.toml b/python/pyproject.toml index 6457a5a4..d6f83d83 100644 --- a/python/pyproject.toml +++ b/python/pyproject.toml @@ -1,5 +1,7 @@ [build-system] -requires = ["setuptools>=64", "wheel"] +# setuptools>=77: PEP 639 SPDX license metadata; setup.py only customizes +# bdist_wheel so the wheel carries a platform tag (see setup.py). +requires = ["setuptools>=77"] build-backend = "setuptools.build_meta" [project] @@ -7,28 +9,31 @@ name = "pygosqlx" version = "0.3.0" description = "Python bindings for GoSQLX - High-performance SQL parser" readme = "README.md" -requires-python = ">=3.8" -license = {text = "Apache-2.0"} +requires-python = ">=3.10" +license = "Apache-2.0" +license-files = ["LICENSE"] authors = [ {name = "GoSQLX Contributors"} ] classifiers = [ "Development Status :: 3 - Alpha", "Intended Audience :: Developers", - "License :: OSI Approved :: Apache Software License", "Programming Language :: Python :: 3", - "Programming Language :: Python :: 3.8", - "Programming Language :: Python :: 3.9", "Programming Language :: Python :: 3.10", "Programming Language :: Python :: 3.11", "Programming Language :: Python :: 3.12", "Programming Language :: Python :: 3.13", + "Programming Language :: Python :: 3.14", + "Operating System :: MacOS", + "Operating System :: Microsoft :: Windows", + "Operating System :: POSIX :: Linux", "Topic :: Database", "Topic :: Software Development :: Libraries", "Typing :: Typed", ] [project.urls] +Documentation = "https://github.com/ajitpratap0/GoSQLX/tree/main/python#readme" Homepage = "https://github.com/ajitpratap0/GoSQLX" Repository = "https://github.com/ajitpratap0/GoSQLX" Issues = "https://github.com/ajitpratap0/GoSQLX/issues" @@ -40,4 +45,4 @@ dev = ["pytest>=7.0", "pytest-cov"] include = ["pygosqlx*"] [tool.setuptools.package-data] -pygosqlx = ["lib/*", "py.typed"] +pygosqlx = ["lib/libgosqlx.*", "py.typed"] diff --git a/python/scripts/install-go-linux.sh b/python/scripts/install-go-linux.sh new file mode 100755 index 00000000..0ce34489 --- /dev/null +++ b/python/scripts/install-go-linux.sh @@ -0,0 +1,31 @@ +#!/usr/bin/env bash +# Install a pinned Go toolchain inside a cibuildwheel manylinux container. +# +# Used as CIBW_BEFORE_ALL_LINUX by .github/workflows/pygosqlx-release.yml. +# The tarball is downloaded from go.dev and verified against the SHA-256 +# pinned below before it is extracted. To bump Go, update GO_VERSION and both +# checksums from https://go.dev/dl/?mode=json (the "sha256" of the +# linux-amd64 and linux-arm64 "archive" files). +set -euo pipefail + +GO_VERSION="1.26.10" +GO_SHA256_AMD64="ccfe7ae822d2c2f41f4fe7c956199967a4dac22d0ee0d2b2bfdb440aab3b20c5" +GO_SHA256_ARM64="3f4227ce7b206a513e794f1393adb473a07b46e94d18bd15d1c684bbb7319797" + +case "$(uname -m)" in + x86_64) arch=amd64; sha="$GO_SHA256_AMD64" ;; + aarch64 | arm64) arch=arm64; sha="$GO_SHA256_ARM64" ;; + *) echo "install-go-linux.sh: unsupported architecture $(uname -m)" >&2; exit 1 ;; +esac + +tarball="go${GO_VERSION}.linux-${arch}.tar.gz" +workdir="$(mktemp -d)" +trap 'rm -rf "$workdir"' EXIT + +curl -fsSL --retry 3 --proto '=https' --tlsv1.2 \ + -o "$workdir/$tarball" "https://go.dev/dl/$tarball" +echo "$sha $workdir/$tarball" | sha256sum -c - + +rm -rf /usr/local/go +tar -C /usr/local -xzf "$workdir/$tarball" +/usr/local/go/bin/go version diff --git a/python/setup.py b/python/setup.py index b4dfb07d..0883513b 100644 --- a/python/setup.py +++ b/python/setup.py @@ -1,37 +1,57 @@ -"""Setup script for pygosqlx - Python bindings for GoSQLX SQL parser.""" - -import os -from setuptools import setup, find_packages - -long_description = "" -readme_path = os.path.join(os.path.dirname(__file__), "README.md") -if os.path.exists(readme_path): - with open(readme_path, encoding="utf-8") as f: - long_description = f.read() - -setup( - name="pygosqlx", - version="0.3.0", - description="Python bindings for GoSQLX - High-performance SQL parser", - long_description=long_description, - long_description_content_type="text/markdown", - author="GoSQLX Contributors", - url="https://github.com/ajitpratap0/GoSQLX", - packages=find_packages(), - package_data={"pygosqlx": ["lib/*", "py.typed"]}, - python_requires=">=3.8", - classifiers=[ - "Development Status :: 3 - Alpha", - "Intended Audience :: Developers", - "License :: OSI Approved :: Apache Software License", - "Programming Language :: Python :: 3", - "Programming Language :: Python :: 3.8", - "Programming Language :: Python :: 3.9", - "Programming Language :: Python :: 3.10", - "Programming Language :: Python :: 3.11", - "Programming Language :: Python :: 3.12", - "Programming Language :: Python :: 3.13", - "Topic :: Database", - "Topic :: Software Development :: Libraries", - ], -) +"""Wheel-tagging shim for pygosqlx. + +All package metadata lives in pyproject.toml. This file exists only because +pyproject.toml cannot express one thing: pygosqlx bundles a native shared +library (pygosqlx/lib/libgosqlx.{so,dylib,dll}) that it loads through ctypes, +so its wheels must be platform-specific. Without this shim setuptools would +see no extension modules and tag the wheel ``py3-none-any``, which pip would +then install on platforms the bundled library cannot run on. + +The library is not a CPython extension module, so it does not depend on the +Python version or ABI. The wheel is therefore tagged ``py3-none-``: +one wheel per OS/architecture serves every supported Python version. +""" + +import sys +from pathlib import Path + +from setuptools import Distribution, setup +from setuptools.command.bdist_wheel import bdist_wheel + +_LIB_DIR = Path(__file__).resolve().parent / "pygosqlx" / "lib" + + +class BinaryDistribution(Distribution): + """Distribution that ships a native library, so it is never pure.""" + + def has_ext_modules(self) -> bool: + return True + + +class PlatformWheel(bdist_wheel): + """bdist_wheel that produces a ``py3-none-`` wheel.""" + + def get_tag(self) -> tuple: + _, _, platform_tag = super().get_tag() + return "py3", "none", platform_tag + + def run(self) -> None: + # Building from the sdist on a platform with no prebuilt wheel yields + # a wheel without the native library; it installs fine and only fails + # at first use. Say so loudly at build time. + if not any( + (_LIB_DIR / f"libgosqlx.{ext}").is_file() for ext in ("so", "dylib", "dll") + ): + print( + "WARNING: pygosqlx/lib/libgosqlx.{so,dylib,dll} is missing; " + "this wheel will not include the native library and pygosqlx " + "will raise FileNotFoundError at first use. Build it with " + "pkg/cbinding/build.sh before installing, or set " + "GOSQLX_LIB_PATH at runtime. See 'Building from source' in " + "python/README.md.", + file=sys.stderr, + ) + super().run() + + +setup(distclass=BinaryDistribution, cmdclass={"bdist_wheel": PlatformWheel}) diff --git a/python/tests/conftest.py b/python/tests/conftest.py new file mode 100644 index 00000000..9137f24e --- /dev/null +++ b/python/tests/conftest.py @@ -0,0 +1,22 @@ +"""Shared pytest configuration for the pygosqlx test suite. + +By default the tests import ``pygosqlx`` from this source checkout, so they +run without installing the package (the shared library is then looked up in +``pygosqlx/lib/`` or via ``GOSQLX_LIB_PATH``). + +Set ``PYGOSQLX_TEST_INSTALLED=1`` to test an installed distribution instead, +for example a wheel built by cibuildwheel. The source checkout is then kept +off ``sys.path`` and ``test_installed_wheel.py`` checks that the installed +package, not the checkout, was imported and that it bundles a loadable +library. Run pytest with ``--import-mode=importlib`` in that mode so pytest +does not put the checkout on ``sys.path`` itself. +""" + +import os +import sys + +SOURCE_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) +TEST_INSTALLED = os.environ.get("PYGOSQLX_TEST_INSTALLED") == "1" + +if not TEST_INSTALLED and SOURCE_DIR not in sys.path: + sys.path.insert(0, SOURCE_DIR) diff --git a/python/tests/test_dialect_and_analysis.py b/python/tests/test_dialect_and_analysis.py index fc208a30..0c8a1b1f 100644 --- a/python/tests/test_dialect_and_analysis.py +++ b/python/tests/test_dialect_and_analysis.py @@ -6,15 +6,10 @@ when it is not built (run: cd pkg/cbinding && ./build.sh). """ -import os -import platform -import sys import threading import pytest -sys.path.insert(0, os.path.join(os.path.dirname(__file__), "..")) - import pygosqlx from pygosqlx import core from pygosqlx import ( @@ -34,18 +29,16 @@ def _lib_available() -> bool: - env_path = os.environ.get("GOSQLX_LIB_PATH") - if env_path and os.path.exists(env_path): - return True - ext = {"linux": "so", "darwin": "dylib", "windows": "dll"}.get( - platform.system().lower() - ) - if ext is None: + """Check if pygosqlx can locate the GoSQLX shared library. + + Uses the package's own lookup (GOSQLX_LIB_PATH, then the bundled + pygosqlx/lib/), so it also works against an installed wheel. + """ + try: + core._get_lib_path() + except OSError: # FileNotFoundError or unsupported platform return False - lib_path = os.path.join( - os.path.dirname(__file__), "..", "pygosqlx", "lib", f"libgosqlx.{ext}" - ) - return os.path.exists(lib_path) + return True requires_lib = pytest.mark.skipif( diff --git a/python/tests/test_installed_wheel.py b/python/tests/test_installed_wheel.py new file mode 100644 index 00000000..02f36f74 --- /dev/null +++ b/python/tests/test_installed_wheel.py @@ -0,0 +1,58 @@ +"""Checks that only run against an installed pygosqlx distribution. + +Enabled with ``PYGOSQLX_TEST_INSTALLED=1`` (the wheel CI sets it, see +.github/workflows/pygosqlx-release.yml). They fail, rather than skip, if the +wheel does not bundle a loadable shared library, so a broken wheel cannot +pass CI by having every library-dependent test skipped. +""" + +import os +import platform + +import pytest + +import pygosqlx +from pygosqlx import core + +pytestmark = pytest.mark.skipif( + os.environ.get("PYGOSQLX_TEST_INSTALLED") != "1", + reason="set PYGOSQLX_TEST_INSTALLED=1 to test an installed wheel", +) + +_SOURCE_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) +_LIB_NAME = { + "linux": "libgosqlx.so", + "darwin": "libgosqlx.dylib", + "windows": "libgosqlx.dll", +}.get(platform.system().lower()) + + +def test_imported_from_installed_package_not_checkout(): + pkg_dir = os.path.dirname(os.path.abspath(pygosqlx.__file__)) + try: + inside_checkout = os.path.commonpath([pkg_dir, _SOURCE_DIR]) == _SOURCE_DIR + except ValueError: # different drives on Windows: cannot be nested + inside_checkout = False + assert not inside_checkout, ( + f"pygosqlx was imported from the source checkout ({pkg_dir}), " + "not from the installed wheel" + ) + + +def test_wheel_bundles_shared_library(): + assert "GOSQLX_LIB_PATH" not in os.environ, ( + "GOSQLX_LIB_PATH must be unset so the bundled library is used" + ) + assert _LIB_NAME is not None, f"unsupported platform {platform.system()}" + bundled = os.path.join( + os.path.dirname(os.path.abspath(core.__file__)), "lib", _LIB_NAME + ) + assert os.path.isfile(bundled), f"wheel does not bundle {bundled}" + assert os.path.samefile(core._get_lib_path(), bundled) + + +def test_bundled_library_loads_and_works(): + assert pygosqlx.version() + assert pygosqlx.validate("SELECT 1") + assert pygosqlx.extract_tables("SELECT * FROM users") == ["users"] + assert "postgresql" in pygosqlx.dialects() diff --git a/python/tests/test_integration.py b/python/tests/test_integration.py index 521f69d7..9041214d 100644 --- a/python/tests/test_integration.py +++ b/python/tests/test_integration.py @@ -11,33 +11,24 @@ GOSQLX_LIB_PATH=/path/to/libgosqlx.dylib python -m pytest tests/test_integration.py -v """ -import os -import platform -import sys import threading import pytest -sys.path.insert(0, os.path.join(os.path.dirname(__file__), "..")) - def _lib_available() -> bool: - """Check if the GoSQLX shared library can be found.""" - # Check environment variable - env_path = os.environ.get("GOSQLX_LIB_PATH") - if env_path and os.path.exists(env_path): - return True - - # Check bundled location - system = platform.system().lower() - ext_map = {"linux": "so", "darwin": "dylib", "windows": "dll"} - ext = ext_map.get(system) - if ext is None: - return False + """Check if pygosqlx can locate the GoSQLX shared library. - pkg_dir = os.path.join(os.path.dirname(__file__), "..", "pygosqlx", "lib") - lib_path = os.path.join(pkg_dir, f"libgosqlx.{ext}") - return os.path.exists(lib_path) + Uses the package's own lookup (GOSQLX_LIB_PATH, then the bundled + pygosqlx/lib/), so it also works against an installed wheel. + """ + from pygosqlx import core + + try: + core._get_lib_path() + except OSError: # FileNotFoundError or unsupported platform + return False + return True # Skip all tests in this module if the library is not built @@ -54,7 +45,6 @@ def _lib_available() -> bool: from pygosqlx.core import ( Metadata, ParseResult, - QualifiedName, ValidationResult, extract_columns, extract_functions, diff --git a/python/tests/test_package_version.py b/python/tests/test_package_version.py index 8f75364c..0034bda9 100644 --- a/python/tests/test_package_version.py +++ b/python/tests/test_package_version.py @@ -1,15 +1,13 @@ """Package metadata consistency tests (no shared library required). -pygosqlx declares its version in three places: pyproject.toml (the source of -truth used by pip/PyPI), setup.py, and ``pygosqlx.__version__``. These tests -keep them from drifting apart. +pygosqlx declares its version in two places: pyproject.toml (the source of +truth used by pip/PyPI) and ``pygosqlx.__version__``. These tests keep them +from drifting apart. (setup.py is only a wheel-tagging shim and carries no +metadata.) """ import os import re -import sys - -sys.path.insert(0, os.path.join(os.path.dirname(__file__), "..")) import pygosqlx @@ -41,7 +39,8 @@ def test_dunder_version_matches_pyproject(): assert pygosqlx.__version__ == _pyproject_version() -def test_setup_py_version_matches_pyproject(): - match = re.search(r'version\s*=\s*"([^"]+)"', _read("setup.py")) - assert match, "setup.py has no version= argument" - assert match.group(1) == _pyproject_version() +def test_setup_py_carries_no_metadata(): + """setup.py must not reintroduce a second, drifting copy of the metadata.""" + text = _read("setup.py") + for field in ("version=", "name=", "python_requires=", "classifiers="): + assert field not in text, f"setup.py should not set {field!r}" diff --git a/python/tests/test_types.py b/python/tests/test_types.py index 460a9c00..190dd5e3 100644 --- a/python/tests/test_types.py +++ b/python/tests/test_types.py @@ -5,11 +5,6 @@ Python with no shared library dependency. """ -import sys -import os - -sys.path.insert(0, os.path.join(os.path.dirname(__file__), "..")) - from pygosqlx.core import ( Metadata, ParseResult,