diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 00000000..ba5a9f2c --- /dev/null +++ b/.dockerignore @@ -0,0 +1,9 @@ +.git +.github +.pytest_cache +dist +tests +docs +utils +*.egg-info +.venv diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 7acecf51..fd71a581 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -18,7 +18,7 @@ jobs: runs-on: ubuntu-latest strategy: matrix: - python-version: ["3.9", "3.11", "3.13"] + python-version: ["3.9", "3.11", "3.13", "3.14"] steps: - uses: actions/checkout@v3 - name: Set up Python ${{ matrix.python-version }} diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 3c1da016..fda56348 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -8,13 +8,36 @@ permissions: contents: write # to create/push git tags with GITHUB_TOKEN id-token: write # to request an OIDC token for AWS STS +# Releases run one at a time. The tag check below and the tag push at the end +# of the job are separated by the whole build/scan/publish sequence, so without +# this two pushes to main could both see the tag missing, build different +# images and push them under the same version tag — and an older run finishing +# last could drag :latest backwards. Queuing (cancel-in-progress: false) means +# the second run's check job re-evaluates the tag only after the first run has +# finished, by which point the tag exists and it skips. +concurrency: + group: release-${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: false + env: QUAY_REPO: quay.io/lifebitaiorg/cloudos-cli PYTHON_VERSION: "3.11" jobs: - release: + # -------- VERSION & RELEASE GATE -------- + # This workflow runs on every push to main, but only a push that bumps + # __version__ is a release. Without this gate an unrelated push (a docs fix, + # a follow-up commit) rebuilds the current version and pushes it over the + # image already on Quay under the same tag — and the SECURITY_REFRESH + # build-arg alone guarantees the rebuild is a different image, so a released + # artefact would silently change. The git tag is the record of what has + # shipped: if v$VERSION already exists, the release job is skipped. + check: runs-on: ubuntu-latest + outputs: + version: ${{ steps.version.outputs.VERSION }} + git_tag: ${{ steps.version.outputs.GIT_TAG }} + release: ${{ steps.released.outputs.RELEASE }} steps: - name: Checkout (full history) @@ -22,7 +45,6 @@ jobs: with: fetch-depth: 0 - # -------- VERSION & TAGGING -------- - name: Extract version from cloudos_cli/_version.py id: version run: | @@ -33,28 +55,36 @@ jobs: echo "VERSION=$VER" >> "$GITHUB_OUTPUT" echo "GIT_TAG=v$VER" >> "$GITHUB_OUTPUT" - - name: Create and push Git tag if missing + - name: Decide whether this version still needs releasing + id: released run: | set -euo pipefail TAG="${{ steps.version.outputs.GIT_TAG }}" - git config user.name "github-actions[bot]" - git config user.email "41898282+github-actions[bot]@users.noreply.github.com" - if git rev-parse "$TAG" >/dev/null 2>&1; then - echo "Tag $TAG already exists; skipping." + if git rev-parse -q --verify "refs/tags/$TAG" >/dev/null; then + echo "$TAG already exists: this version has shipped, nothing to do." + echo "Bump __version__ in cloudos_cli/_version.py to cut a new release." + echo "RELEASE=false" >> "$GITHUB_OUTPUT" else - git tag -a "$TAG" -m "Release $TAG" - git push origin "$TAG" + echo "$TAG does not exist yet; proceeding with the release." + echo "RELEASE=true" >> "$GITHUB_OUTPUT" fi - # -------- GITHUB RELEASE -------- - - name: Create GitHub Release with auto-generated notes - uses: softprops/action-gh-release@v1 + release: + needs: check + if: needs.check.outputs.release == 'true' + runs-on: ubuntu-latest + + steps: + - name: Checkout (full history) + uses: actions/checkout@v4 with: - tag_name: ${{ steps.version.outputs.GIT_TAG }} - name: Release ${{ steps.version.outputs.GIT_TAG }} - draft: false - prerelease: ${{ contains(steps.version.outputs.VERSION, 'a') || contains(steps.version.outputs.VERSION, 'b') || contains(steps.version.outputs.VERSION, 'rc') }} - generate_release_notes: true + fetch-depth: 0 + + # NOTE: the Git tag and the GitHub Release are created at the END of this + # job, after the image has passed the vulnerability gate and both + # artefacts have been published. Creating them here meant a failed scan + # or a failed upload left behind a pushed tag and a published release + # pointing at artefacts that never shipped. # -------- PYTHON SETUP -------- - name: Set up Python ${{ env.PYTHON_VERSION }} @@ -66,6 +96,15 @@ jobs: run: | python -m pip install --upgrade pip build twine + # Built before anything is published: this needs no secrets and no + # registry, so a packaging error should surface while the release is + # still entirely undone, not after the image is public. + - name: Build distribution (sdist + wheel) + run: | + set -euo pipefail + echo "Building Python distribution packages..." + python -m build + # -------- QUAY (no helper actions) -------- - name: Test Quay.io login id: quay @@ -74,25 +113,82 @@ jobs: registry: quay.io username: ${{ secrets.QUAY_LOGIN_ID }} password: ${{ secrets.QUAY_LOGIN_PWD }} - - name: Build & push image to Quay (version + latest) + - name: Build image run: | set -euo pipefail - VER="${{ steps.version.outputs.VERSION }}" + VER="${{ needs.check.outputs.version }}" echo "Building Docker image for version: $VER" - docker build -f Dockerfile -t "${QUAY_REPO}:v${VER}" . - docker push "${QUAY_REPO}:v${VER}" - echo "Pushed ${QUAY_REPO}:v${VER}" + # SECURITY_REFRESH busts the cache on the Dockerfile's final + # `apt-get upgrade` layer. Without it, once this job uses a build + # cache, a cached upgrade silently stops picking up new patches. + docker build -f Dockerfile \ + --build-arg SECURITY_REFRESH="$(date +%s)" \ + -t "${QUAY_REPO}:v${VER}" . - docker tag "${QUAY_REPO}:v${VER}" "${QUAY_REPO}:latest" - docker push "${QUAY_REPO}:latest" + # Trivy runs as a pinned container, so there is nothing to install: + # Docker is already present on the runner (the build step above needs + # it). --exit-code 1 fails the step on any CRITICAL/HIGH, and unfixed + # CVEs are counted (no --ignore-unfixed), which mirrors the Harbor gate. + # A Trivy error exits 2, so a broken scanner also fails the step rather + # than silently passing the release. + - name: Scan image with Trivy (blocks the release on CRITICAL/HIGH) + run: | + set -euo pipefail + VER="${{ needs.check.outputs.version }}" + docker run --rm \ + -v /var/run/docker.sock:/var/run/docker.sock \ + -v "${HOME}/.cache/trivy:/root/.cache/trivy" \ + aquasec/trivy:0.74.0 image \ + --image-src docker \ + --scanners vuln \ + --severity CRITICAL,HIGH \ + --exit-code 1 \ + "${QUAY_REPO}:v${VER}" - # -------- PYPI (no helper actions) -------- - - name: Build distribution (sdist + wheel) + # Last check before anything leaves the runner. The concurrency group + # above already serializes release runs, but a manual re-run of an old + # workflow run, or a run started before the group was in place, is not + # covered by it. origin is authoritative here: the local clone was made + # before the build started. + - name: Verify the version tag is still unclaimed on the remote run: | set -euo pipefail - echo "Building Python distribution packages..." - python -m build + TAG="${{ needs.check.outputs.git_tag }}" + if git ls-remote --exit-code --tags origin "refs/tags/$TAG" >/dev/null 2>&1; then + echo "::error::$TAG appeared on origin while this job was building." + echo "Another run has already released this version; refusing to" + echo "overwrite ${QUAY_REPO}:$TAG with a different image." + exit 1 + fi + echo "$TAG is still unclaimed; publishing." + + # git says what we intended to release; only the registry says what is + # actually published, and the two can disagree — a run that pushed the + # image and then failed before tagging leaves v$VER on Quay with no git + # tag, and the next push to main would rebuild it (a different image, + # because of SECURITY_REFRESH) and silently overwrite the released + # artefact. Ask the registry, which is the thing we are about to mutate. + # Uses the credentials from the docker/login-action step above. + - name: Verify the image tag is unclaimed in the registry + run: | + set -euo pipefail + VER="${{ needs.check.outputs.version }}" + if docker manifest inspect "${QUAY_REPO}:v${VER}" >/dev/null 2>&1; then + echo "::error::${QUAY_REPO}:v${VER} is already published." + echo "Refusing to overwrite it with a different build." + echo "Bump __version__ to cut a new release." + exit 1 + fi + echo "${QUAY_REPO}:v${VER} is unclaimed; publishing." + + - name: Push image to Quay (version tag) + run: | + set -euo pipefail + VER="${{ needs.check.outputs.version }}" + docker push "${QUAY_REPO}:v${VER}" + echo "Pushed ${QUAY_REPO}:v${VER}" + # -------- PYPI (no helper actions) -------- - name: Upload to PyPI with twine env: TWINE_USERNAME: __token__ @@ -102,3 +198,55 @@ jobs: echo "Uploading to PyPI..." python -m twine upload --non-interactive dist/* --skip-existing echo "Successfully uploaded to PyPI" + + # -------- TAG & RELEASE -------- + # The tag lands here: after the immutable publishes (the version image + # tag and PyPI, neither of which can be replaced), and before the + # mutable one (:latest). That way the tag is the record of a release + # whose artefacts are already final, and the guards above see it. + - name: Create and push Git tag if missing + run: | + set -euo pipefail + TAG="${{ needs.check.outputs.git_tag }}" + git config user.name "github-actions[bot]" + git config user.email "41898282+github-actions[bot]@users.noreply.github.com" + if git rev-parse "$TAG" >/dev/null 2>&1; then + echo "Tag $TAG already exists; skipping." + else + git tag -a "$TAG" -m "Release $TAG" + git push origin "$TAG" + fi + + # Only ever move :latest forward, and never onto a pre-release. A patch + # cut from an older branch would otherwise drag :latest backwards. + # This version's tag was created by the step above, so NEWEST includes + # it: the comparison passes when this is the newest release and fails + # when a later one already exists. + - name: Tag and push :latest (newest final release only) + run: | + set -euo pipefail + VER="${{ needs.check.outputs.version }}" + case "$VER" in + *a*|*b*|*rc*) + echo "v$VER is a pre-release; leaving :latest unchanged." + exit 0 + ;; + esac + NEWEST="$(git tag --list 'v*' | sed 's/^v//' | sort -V | tail -1)" + if [ -n "$NEWEST" ] \ + && [ "$(printf '%s\n%s\n' "$NEWEST" "$VER" | sort -V | tail -1)" != "$VER" ]; then + echo "v$VER is older than the newest tag (v$NEWEST); leaving :latest unchanged." + exit 0 + fi + docker tag "${QUAY_REPO}:v${VER}" "${QUAY_REPO}:latest" + docker push "${QUAY_REPO}:latest" + echo "Pushed ${QUAY_REPO}:latest" + + - name: Create GitHub Release with auto-generated notes + uses: softprops/action-gh-release@v1 + with: + tag_name: ${{ needs.check.outputs.git_tag }} + name: Release ${{ needs.check.outputs.git_tag }} + draft: false + prerelease: ${{ contains(needs.check.outputs.version, 'a') || contains(needs.check.outputs.version, 'b') || contains(needs.check.outputs.version, 'rc') }} + generate_release_notes: true diff --git a/CHANGELOG.md b/CHANGELOG.md index 6572fe8c..b61ad807 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,15 @@ ## lifebit-ai/cloudos-cli: changelog +## v2.96.0 (2026-09-09) + +### Feat: + +- Changed dockerfile and release.yml to address harbor issues +- The release workflow asks the registry, not git, whether a version image is already published, and creates the Git tag after the immutable artefacts (version image tag, PyPI) and before the mutable one (`:latest`) +- CI unit tests now also run on Python 3.14, the version shipped by the container's base image +- Adds a `.dockerignore`, keeping `.git` (and its full history), `tests/`, `docs/` and build artefacts out of the published image +- The image cleanup now also removes the `conda_rattler_solver` plugin and the `conda-meta` entries for the rattler packages, so the scanner's inventory matches what is actually installed + ## v2.95.1 (2026-08-20) ### Fix: diff --git a/Dockerfile b/Dockerfile index 6ebfc28d..c1dbbef3 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,32 +1,78 @@ -FROM continuumio/miniconda3:26.3.2 +# Base: condaforge/miniforge3:26.7.2-0 — the official conda-forge image +# (Ubuntu 24.04 LTS, versioned tags). Replaces continuumio/miniconda3, which is +# Debian: perl-base is an essential package there, cannot be removed, and has +# no fix for CVE-2026-13221 / CVE-2026-42496 / CVE-2026-8376 — a floor of 3 +# unfixable CRITICALs at every miniconda tag. miniforge3 also keeps the +# environment at /opt/conda, so nothing that hardcodes a conda path changes. +FROM condaforge/miniforge3:26.7.2-0 + LABEL name="quay.io/lifebitaiorg/cloudos-cli" \ description="The cloudos-cli docker container" \ maintainer="David Pineyro " -# Use the base conda env to not be reliant on conda activate when using pip +# `apt-get upgrade` matters: the base image lags the security archive, so +# without it you inherit whatever was current when the base was cut. +# procps is required by Nextflow for `ps` task tracing. +RUN apt-get update -y \ + && apt-get upgrade -y \ + && apt-get install -y --no-install-recommends procps \ + && rm -rf /var/lib/apt/lists/* + +# mamba ships in this base, so the old `conda install mamba` step is gone. ARG ENV_NAME="base" +COPY environment.yml / +RUN conda env update --quiet --name ${ENV_NAME} --file /environment.yml \ + && conda clean -a -y -# Install mamba for faster installation in the subsequent step -RUN conda install -c conda-forge mamba -y +ENV PATH=/opt/conda/envs/${ENV_NAME}/bin:$PATH -# Install the conda environment -COPY environment.yml / -RUN mamba env update -y --name ${ENV_NAME} --file /environment.yml -vv && \ - mamba clean -afy +COPY . / +ENV PATH=/cloudos_cli:$PATH +RUN pip install --no-cache-dir -e / \ + && chmod ugo+x /cloudos_cli/*py /cloudos_cli/jobs/*py /cloudos_cli/utils/*py \ + && cloudos --help >/dev/null \ + && python -c "import cloudos_cli" -# Add conda installation dir to PATH (instead of doing 'conda activate') -ENV PATH /opt/conda/envs/${ENV_NAME}/bin:$PATH +# Two cleanups are what take a miniforge3 image to zero. +# +# 1. pip vendors its own dependency manifests (pip/_vendor/vendor.txt and +# bom.cdx.json). Trivy reads them as installed inventory and reports the +# versions listed there. Metadata only; pip does not read them at runtime. +# +# 2. py-rattler is conda's Rust-based resolver. It ships a CycloneDX SBOM +# declaring the Rust crates linked into its binary, two of which carry HIGH +# advisories (pyo3, quinn-proto). These are NOT false positives — the code +# really is in the .so. The environment is fully built by this point and +# nothing solves dependencies at runtime, so removing the resolver removes +# real attack surface rather than silencing the scanner. +# +# The resolver is four things, not one: the py_rattler/rattler modules, the +# conda_rattler_solver plugin that dispatches to them, and the conda-meta +# entries for both. Removing only the modules leaves Trivy's conda-meta +# analyzer still reporting py-rattler as installed (so a future advisory +# would fail the gate on code that is no longer here) and leaves the plugin +# registered against a module that is gone, which turns +# `conda --solver rattler` into an ImportError. The default solver is +# libmamba, so nothing routine depends on it. +# +# && rather than ;: with ; the layer's exit status is rm's, which is always 0, +# so a find that errored (a missing /opt/conda, an unreadable tree) was +# discarded and the build carried on with an image that was never cleaned. +# Note find still exits 0 when a pattern simply matches nothing, so this +# guards against a broken tree, not against a base that renames these paths. +RUN find /opt/conda \( -path "*/pip/_vendor/vendor.txt" \ + -o -path "*/pip/_vendor/bom.cdx.json" \) -delete \ + && rm -rf /opt/conda/lib/python*/site-packages/py_rattler* \ + /opt/conda/lib/python*/site-packages/rattler* \ + /opt/conda/lib/python*/site-packages/conda_rattler_solver* \ + /opt/conda/conda-meta/py-rattler-*.json \ + /opt/conda/conda-meta/conda-rattler-solver-*.json -# Dump the details of the installed packages to a file for posterity -RUN mamba env export --name ${ENV_NAME} > ${ENV_NAME}_exported.yml +# Keep the security update last and in its own layer: earlier layers get cached +# across rebuilds and a cached update silently stops picking up new patches. +# The release workflow forces it with --build-arg SECURITY_REFRESH=$(date +%s). +ARG SECURITY_REFRESH=1 +RUN apt-get update -y && apt-get upgrade -y && rm -rf /var/lib/apt/lists/* -# Copy local package files to be able to install -COPY . / -# Add the created folder to PATH so that tools are accessible -ENV PATH /cloudos:$PATH -# Install from local files, -e / points to where the setup.py file is located -RUN pip install -e / -# Make the python files executable from anyone (user, group, owner) -RUN chmod ugo+x /cloudos_cli/*py -RUN chmod ugo+x /cloudos_cli/jobs/*py -RUN chmod ugo+x /cloudos_cli/utils/*py +CMD ["/bin/bash"] +ENTRYPOINT [] diff --git a/cloudos_cli/_version.py b/cloudos_cli/_version.py index fa202cff..dcaaaece 100644 --- a/cloudos_cli/_version.py +++ b/cloudos_cli/_version.py @@ -1 +1 @@ -__version__ = '2.95.1' +__version__ = '2.96.0' diff --git a/tests/test_jobs/test_accelerate_saving_results_clone_resume.py b/tests/test_jobs/test_accelerate_saving_results_clone_resume.py index 20822951..941b7b15 100644 --- a/tests/test_jobs/test_accelerate_saving_results_clone_resume.py +++ b/tests/test_jobs/test_accelerate_saving_results_clone_resume.py @@ -3,6 +3,7 @@ This test file provides testing for the --accelerate-saving-results flag functionality in the job resume command of CloudOS CLI. """ +import click from cloudos_cli.jobs.cli import clone_resume @@ -20,7 +21,11 @@ def test_resume_accelerate_saving_results_flag_is_boolean(): assert accelerate_saving_results_option is not None assert accelerate_saving_results_option.is_flag is True - assert accelerate_saving_results_option.default is False + # The effective default, not the `default` attribute: click >= 8.5 leaves + # an unset flag's attribute as a sentinel, while the value the command + # actually receives is False in every version. + ctx = click.Context(clone_resume) + assert accelerate_saving_results_option.get_default(ctx) is False def test_resume_accelerate_saving_results_flag_definition(): diff --git a/tests/test_jobs/test_accelerate_saving_results_run.py b/tests/test_jobs/test_accelerate_saving_results_run.py index 1d43337a..0e059f12 100644 --- a/tests/test_jobs/test_accelerate_saving_results_run.py +++ b/tests/test_jobs/test_accelerate_saving_results_run.py @@ -3,6 +3,7 @@ This test file provides testing for the --accelerate-saving-results flag functionality in the job run command of CloudOS CLI. """ +import click from click.testing import CliRunner from cloudos_cli.jobs.cli import run @@ -21,7 +22,11 @@ def test_run_accelerate_saving_results_flag_is_boolean(): assert accelerate_saving_results_option is not None assert accelerate_saving_results_option.is_flag is True - assert accelerate_saving_results_option.default is False + # The effective default, not the `default` attribute: click >= 8.5 leaves + # an unset flag's attribute as a sentinel, while the value the command + # actually receives is False in every version. + ctx = click.Context(run) + assert accelerate_saving_results_option.get_default(ctx) is False def test_run_accelerate_saving_results_help_text():