From 9f3084b48b32f71801bd30759b96df3a1f6f550f Mon Sep 17 00:00:00 2001 From: EC2 Default User Date: Wed, 9 Sep 2026 14:03:59 +0000 Subject: [PATCH 1/8] changes to the container for harbor --- .github/workflows/release.yml | 105 ++++++++++++++++++++++++++-------- Dockerfile | 72 ++++++++++++++++------- 2 files changed, 131 insertions(+), 46 deletions(-) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 3c1da016..c05261e1 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -33,28 +33,11 @@ jobs: echo "VERSION=$VER" >> "$GITHUB_OUTPUT" echo "GIT_TAG=v$VER" >> "$GITHUB_OUTPUT" - - name: Create and push Git tag if missing - run: | - set -euo pipefail - TAG="${{ steps.version.outputs.GIT_TAG }}" - git config user.name "github-actions[bot]" - git config user.email "41898282+github-actions[bot]@users.noreply.github.com" - if git rev-parse "$TAG" >/dev/null 2>&1; then - echo "Tag $TAG already exists; skipping." - else - git tag -a "$TAG" -m "Release $TAG" - git push origin "$TAG" - fi - - # -------- GITHUB RELEASE -------- - - name: Create GitHub Release with auto-generated notes - uses: softprops/action-gh-release@v1 - with: - tag_name: ${{ steps.version.outputs.GIT_TAG }} - name: Release ${{ steps.version.outputs.GIT_TAG }} - draft: false - prerelease: ${{ contains(steps.version.outputs.VERSION, 'a') || contains(steps.version.outputs.VERSION, 'b') || contains(steps.version.outputs.VERSION, 'rc') }} - generate_release_notes: true + # NOTE: the Git tag and the GitHub Release are created at the END of this + # job, after the image has passed the vulnerability gate and both + # artefacts have been published. Creating them here meant a failed scan + # or a failed upload left behind a pushed tag and a published release + # pointing at artefacts that never shipped. # -------- PYTHON SETUP -------- - name: Set up Python ${{ env.PYTHON_VERSION }} @@ -74,17 +57,68 @@ jobs: registry: quay.io username: ${{ secrets.QUAY_LOGIN_ID }} password: ${{ secrets.QUAY_LOGIN_PWD }} - - name: Build & push image to Quay (version + latest) + - name: Build image run: | set -euo pipefail VER="${{ steps.version.outputs.VERSION }}" echo "Building Docker image for version: $VER" - docker build -f Dockerfile -t "${QUAY_REPO}:v${VER}" . + # SECURITY_REFRESH busts the cache on the Dockerfile's final + # `apt-get upgrade` layer. Without it, once this job uses a build + # cache, a cached upgrade silently stops picking up new patches. + docker build -f Dockerfile \ + --build-arg SECURITY_REFRESH="$(date +%s)" \ + -t "${QUAY_REPO}:v${VER}" . + + # Trivy runs as a pinned container, so there is nothing to install: + # Docker is already present on the runner (the build step above needs + # it). --exit-code 1 fails the step on any CRITICAL/HIGH, and unfixed + # CVEs are counted (no --ignore-unfixed), which mirrors the Harbor gate. + # A Trivy error exits 2, so a broken scanner also fails the step rather + # than silently passing the release. + - name: Scan image with Trivy (blocks the release on CRITICAL/HIGH) + run: | + set -euo pipefail + VER="${{ steps.version.outputs.VERSION }}" + docker run --rm \ + -v /var/run/docker.sock:/var/run/docker.sock \ + -v "${HOME}/.cache/trivy:/root/.cache/trivy" \ + aquasec/trivy:0.74.0 image \ + --image-src docker \ + --scanners vuln \ + --severity CRITICAL,HIGH \ + --exit-code 1 \ + "${QUAY_REPO}:v${VER}" + + - name: Push image to Quay (version tag) + run: | + set -euo pipefail + VER="${{ steps.version.outputs.VERSION }}" docker push "${QUAY_REPO}:v${VER}" echo "Pushed ${QUAY_REPO}:v${VER}" + # Only ever move :latest forward, and never onto a pre-release. A patch + # cut from an older branch would otherwise drag :latest backwards. + # NEWEST is the previous release: this version's tag is not created until + # the end of the job. + - name: Tag and push :latest (newest final release only) + run: | + set -euo pipefail + VER="${{ steps.version.outputs.VERSION }}" + case "$VER" in + *a*|*b*|*rc*) + echo "v$VER is a pre-release; leaving :latest unchanged." + exit 0 + ;; + esac + NEWEST="$(git tag --list 'v*' | sed 's/^v//' | sort -V | tail -1)" + if [ -n "$NEWEST" ] \ + && [ "$(printf '%s\n%s\n' "$NEWEST" "$VER" | sort -V | tail -1)" != "$VER" ]; then + echo "v$VER is older than the newest tag (v$NEWEST); leaving :latest unchanged." + exit 0 + fi docker tag "${QUAY_REPO}:v${VER}" "${QUAY_REPO}:latest" docker push "${QUAY_REPO}:latest" + echo "Pushed ${QUAY_REPO}:latest" # -------- PYPI (no helper actions) -------- - name: Build distribution (sdist + wheel) @@ -102,3 +136,26 @@ jobs: echo "Uploading to PyPI..." python -m twine upload --non-interactive dist/* --skip-existing echo "Successfully uploaded to PyPI" + + # -------- TAG & RELEASE (last: only once everything shipped) -------- + - name: Create and push Git tag if missing + run: | + set -euo pipefail + TAG="${{ steps.version.outputs.GIT_TAG }}" + git config user.name "github-actions[bot]" + git config user.email "41898282+github-actions[bot]@users.noreply.github.com" + if git rev-parse "$TAG" >/dev/null 2>&1; then + echo "Tag $TAG already exists; skipping." + else + git tag -a "$TAG" -m "Release $TAG" + git push origin "$TAG" + fi + + - name: Create GitHub Release with auto-generated notes + uses: softprops/action-gh-release@v1 + with: + tag_name: ${{ steps.version.outputs.GIT_TAG }} + name: Release ${{ steps.version.outputs.GIT_TAG }} + draft: false + prerelease: ${{ contains(steps.version.outputs.VERSION, 'a') || contains(steps.version.outputs.VERSION, 'b') || contains(steps.version.outputs.VERSION, 'rc') }} + generate_release_notes: true diff --git a/Dockerfile b/Dockerfile index 6ebfc28d..43c80a56 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,32 +1,60 @@ -FROM continuumio/miniconda3:26.3.2 +# Base: condaforge/miniforge3:26.7.2-0 — the official conda-forge image +# (Ubuntu 24.04 LTS, versioned tags). Replaces continuumio/miniconda3, which is +# Debian: perl-base is an essential package there, cannot be removed, and has +# no fix for CVE-2026-13221 / CVE-2026-42496 / CVE-2026-8376 — a floor of 3 +# unfixable CRITICALs at every miniconda tag. miniforge3 also keeps the +# environment at /opt/conda, so nothing that hardcodes a conda path changes. +FROM condaforge/miniforge3:26.7.2-0 + LABEL name="quay.io/lifebitaiorg/cloudos-cli" \ description="The cloudos-cli docker container" \ maintainer="David Pineyro " -# Use the base conda env to not be reliant on conda activate when using pip +# `apt-get upgrade` matters: the base image lags the security archive, so +# without it you inherit whatever was current when the base was cut. +# procps is required by Nextflow for `ps` task tracing. +RUN apt-get update -y \ + && apt-get upgrade -y \ + && apt-get install -y --no-install-recommends procps \ + && rm -rf /var/lib/apt/lists/* + +# mamba ships in this base, so the old `conda install mamba` step is gone. ARG ENV_NAME="base" +COPY environment.yml / +RUN conda env update --quiet --name ${ENV_NAME} --file /environment.yml \ + && conda clean -a -y -# Install mamba for faster installation in the subsequent step -RUN conda install -c conda-forge mamba -y +ENV PATH=/opt/conda/envs/${ENV_NAME}/bin:$PATH -# Install the conda environment -COPY environment.yml / -RUN mamba env update -y --name ${ENV_NAME} --file /environment.yml -vv && \ - mamba clean -afy +COPY . / +ENV PATH=/cloudos_cli:$PATH +RUN pip install --no-cache-dir -e / \ + && chmod ugo+x /cloudos_cli/*py /cloudos_cli/jobs/*py /cloudos_cli/utils/*py \ + && cloudos --help >/dev/null \ + && python -c "import cloudos_cli" -# Add conda installation dir to PATH (instead of doing 'conda activate') -ENV PATH /opt/conda/envs/${ENV_NAME}/bin:$PATH +# Two cleanups are what take a miniforge3 image to zero. +# +# 1. pip vendors its own dependency manifests (pip/_vendor/vendor.txt and +# bom.cdx.json). Trivy reads them as installed inventory and reports the +# versions listed there. Metadata only; pip does not read them at runtime. +# +# 2. py-rattler is conda's Rust-based resolver. It ships a CycloneDX SBOM +# declaring the Rust crates linked into its binary, two of which carry HIGH +# advisories (pyo3, quinn-proto). These are NOT false positives — the code +# really is in the .so. The environment is fully built by this point and +# nothing solves dependencies at runtime, so removing the resolver removes +# real attack surface rather than silencing the scanner. +RUN find /opt/conda -path "*/pip/_vendor/vendor.txt" -delete \ + -o -path "*/pip/_vendor/bom.cdx.json" -delete \ + ; rm -rf /opt/conda/lib/python*/site-packages/py_rattler* \ + /opt/conda/lib/python*/site-packages/rattler* -# Dump the details of the installed packages to a file for posterity -RUN mamba env export --name ${ENV_NAME} > ${ENV_NAME}_exported.yml +# Keep the security update last and in its own layer: earlier layers get cached +# across rebuilds and a cached update silently stops picking up new patches. +# The release workflow forces it with --build-arg SECURITY_REFRESH=$(date +%s). +ARG SECURITY_REFRESH=1 +RUN apt-get update -y && apt-get upgrade -y && rm -rf /var/lib/apt/lists/* -# Copy local package files to be able to install -COPY . / -# Add the created folder to PATH so that tools are accessible -ENV PATH /cloudos:$PATH -# Install from local files, -e / points to where the setup.py file is located -RUN pip install -e / -# Make the python files executable from anyone (user, group, owner) -RUN chmod ugo+x /cloudos_cli/*py -RUN chmod ugo+x /cloudos_cli/jobs/*py -RUN chmod ugo+x /cloudos_cli/utils/*py +CMD ["/bin/bash"] +ENTRYPOINT [] From 8721656c8a93ff2d5e76fa4eca75fd2103bb7b48 Mon Sep 17 00:00:00 2001 From: EC2 Default User Date: Wed, 9 Sep 2026 14:06:13 +0000 Subject: [PATCH 2/8] changelog --- CHANGELOG.md | 6 ++++++ cloudos_cli/_version.py | 2 +- 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 6572fe8c..d562c7a8 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,11 @@ ## lifebit-ai/cloudos-cli: changelog +## v2.96.0 (2026-09-09) + +### Feat: + +- Changed dockerfile and release.yml to address harbor issues + ## v2.95.1 (2026-08-20) ### Fix: diff --git a/cloudos_cli/_version.py b/cloudos_cli/_version.py index fa202cff..dcaaaece 100644 --- a/cloudos_cli/_version.py +++ b/cloudos_cli/_version.py @@ -1 +1 @@ -__version__ = '2.95.1' +__version__ = '2.96.0' From b372597143a9bc339148011b7db25eb8644b8b53 Mon Sep 17 00:00:00 2001 From: EC2 Default User Date: Thu, 10 Sep 2026 10:59:27 +0000 Subject: [PATCH 3/8] fix pytest --- .github/workflows/release.yml | 56 ++++++++++++++++++++++++++++------- cloudos_cli/bash/cli.py | 6 ++-- cloudos_cli/jobs/cli.py | 6 ++-- 3 files changed, 54 insertions(+), 14 deletions(-) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index c05261e1..05922f36 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -13,8 +13,20 @@ env: PYTHON_VERSION: "3.11" jobs: - release: + # -------- VERSION & RELEASE GATE -------- + # This workflow runs on every push to main, but only a push that bumps + # __version__ is a release. Without this gate an unrelated push (a docs fix, + # a follow-up commit) rebuilds the current version and pushes it over the + # image already on Quay under the same tag — and the SECURITY_REFRESH + # build-arg alone guarantees the rebuild is a different image, so a released + # artefact would silently change. The git tag is the record of what has + # shipped: if v$VERSION already exists, the release job is skipped. + check: runs-on: ubuntu-latest + outputs: + version: ${{ steps.version.outputs.VERSION }} + git_tag: ${{ steps.version.outputs.GIT_TAG }} + release: ${{ steps.released.outputs.RELEASE }} steps: - name: Checkout (full history) @@ -22,7 +34,6 @@ jobs: with: fetch-depth: 0 - # -------- VERSION & TAGGING -------- - name: Extract version from cloudos_cli/_version.py id: version run: | @@ -33,6 +44,31 @@ jobs: echo "VERSION=$VER" >> "$GITHUB_OUTPUT" echo "GIT_TAG=v$VER" >> "$GITHUB_OUTPUT" + - name: Decide whether this version still needs releasing + id: released + run: | + set -euo pipefail + TAG="${{ steps.version.outputs.GIT_TAG }}" + if git rev-parse -q --verify "refs/tags/$TAG" >/dev/null; then + echo "$TAG already exists: this version has shipped, nothing to do." + echo "Bump __version__ in cloudos_cli/_version.py to cut a new release." + echo "RELEASE=false" >> "$GITHUB_OUTPUT" + else + echo "$TAG does not exist yet; proceeding with the release." + echo "RELEASE=true" >> "$GITHUB_OUTPUT" + fi + + release: + needs: check + if: needs.check.outputs.release == 'true' + runs-on: ubuntu-latest + + steps: + - name: Checkout (full history) + uses: actions/checkout@v4 + with: + fetch-depth: 0 + # NOTE: the Git tag and the GitHub Release are created at the END of this # job, after the image has passed the vulnerability gate and both # artefacts have been published. Creating them here meant a failed scan @@ -60,7 +96,7 @@ jobs: - name: Build image run: | set -euo pipefail - VER="${{ steps.version.outputs.VERSION }}" + VER="${{ needs.check.outputs.version }}" echo "Building Docker image for version: $VER" # SECURITY_REFRESH busts the cache on the Dockerfile's final # `apt-get upgrade` layer. Without it, once this job uses a build @@ -78,7 +114,7 @@ jobs: - name: Scan image with Trivy (blocks the release on CRITICAL/HIGH) run: | set -euo pipefail - VER="${{ steps.version.outputs.VERSION }}" + VER="${{ needs.check.outputs.version }}" docker run --rm \ -v /var/run/docker.sock:/var/run/docker.sock \ -v "${HOME}/.cache/trivy:/root/.cache/trivy" \ @@ -92,7 +128,7 @@ jobs: - name: Push image to Quay (version tag) run: | set -euo pipefail - VER="${{ steps.version.outputs.VERSION }}" + VER="${{ needs.check.outputs.version }}" docker push "${QUAY_REPO}:v${VER}" echo "Pushed ${QUAY_REPO}:v${VER}" @@ -103,7 +139,7 @@ jobs: - name: Tag and push :latest (newest final release only) run: | set -euo pipefail - VER="${{ steps.version.outputs.VERSION }}" + VER="${{ needs.check.outputs.version }}" case "$VER" in *a*|*b*|*rc*) echo "v$VER is a pre-release; leaving :latest unchanged." @@ -141,7 +177,7 @@ jobs: - name: Create and push Git tag if missing run: | set -euo pipefail - TAG="${{ steps.version.outputs.GIT_TAG }}" + TAG="${{ needs.check.outputs.git_tag }}" git config user.name "github-actions[bot]" git config user.email "41898282+github-actions[bot]@users.noreply.github.com" if git rev-parse "$TAG" >/dev/null 2>&1; then @@ -154,8 +190,8 @@ jobs: - name: Create GitHub Release with auto-generated notes uses: softprops/action-gh-release@v1 with: - tag_name: ${{ steps.version.outputs.GIT_TAG }} - name: Release ${{ steps.version.outputs.GIT_TAG }} + tag_name: ${{ needs.check.outputs.git_tag }} + name: Release ${{ needs.check.outputs.git_tag }} draft: false - prerelease: ${{ contains(steps.version.outputs.VERSION, 'a') || contains(steps.version.outputs.VERSION, 'b') || contains(steps.version.outputs.VERSION, 'rc') }} + prerelease: ${{ contains(needs.check.outputs.version, 'a') || contains(needs.check.outputs.version, 'b') || contains(needs.check.outputs.version, 'rc') }} generate_release_notes: true diff --git a/cloudos_cli/bash/cli.py b/cloudos_cli/bash/cli.py index c70eb68d..9cc6a8bf 100644 --- a/cloudos_cli/bash/cli.py +++ b/cloudos_cli/bash/cli.py @@ -96,7 +96,8 @@ def bash(): default=30.0) @click.option('--accelerate-saving-results', help='Enables saving results directly to cloud storage bypassing the master node.', - is_flag=True) + is_flag=True, + default=False) @click.option('--request-interval', help=('Time interval to request (in seconds) the job status. ' + 'For large jobs is important to use a high number to ' + @@ -305,7 +306,8 @@ def run_bash_job(ctx, default=30.0) @click.option('--accelerate-saving-results', help='Enables saving results directly to cloud storage bypassing the master node.', - is_flag=True) + is_flag=True, + default=False) @click.option('--request-interval', help=('Time interval to request (in seconds) the job status. ' + 'For large jobs is important to use a high number to ' + diff --git a/cloudos_cli/jobs/cli.py b/cloudos_cli/jobs/cli.py index 4f63b075..c81f3308 100644 --- a/cloudos_cli/jobs/cli.py +++ b/cloudos_cli/jobs/cli.py @@ -171,7 +171,8 @@ def job(): is_flag=True) @click.option('--accelerate-saving-results', help='Enables saving results directly to cloud storage bypassing the master node.', - is_flag=True) + is_flag=True, + default=False) @click.option('--use-private-docker-repository', help=('Allows to use private docker repository for running jobs. The Docker user ' + 'account has to be already linked to Lifebit Platform.'), @@ -1733,7 +1734,8 @@ def archive_unarchive_jobs(ctx, is_flag=True) @click.option('--accelerate-saving-results', help='Enables saving results directly to cloud storage bypassing the master node.', - is_flag=True) + is_flag=True, + default=False) @click.option('--resumable', help='Whether to make the job able to be resumed or not.', is_flag=True) From f177d2b2ca2499d8afbc7567be27dafda367cdaf Mon Sep 17 00:00:00 2001 From: EC2 Default User Date: Thu, 10 Sep 2026 11:41:31 +0000 Subject: [PATCH 4/8] address copilot --- .github/workflows/release.yml | 28 ++++++++++++++++++++++++++++ Dockerfile | 29 +++++++++++++++++++++-------- 2 files changed, 49 insertions(+), 8 deletions(-) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 05922f36..96886f7b 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -8,6 +8,17 @@ permissions: contents: write # to create/push git tags with GITHUB_TOKEN id-token: write # to request an OIDC token for AWS STS +# Releases run one at a time. The tag check below and the tag push at the end +# of the job are separated by the whole build/scan/publish sequence, so without +# this two pushes to main could both see the tag missing, build different +# images and push them under the same version tag — and an older run finishing +# last could drag :latest backwards. Queuing (cancel-in-progress: false) means +# the second run's check job re-evaluates the tag only after the first run has +# finished, by which point the tag exists and it skips. +concurrency: + group: release-${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: false + env: QUAY_REPO: quay.io/lifebitaiorg/cloudos-cli PYTHON_VERSION: "3.11" @@ -125,6 +136,23 @@ jobs: --exit-code 1 \ "${QUAY_REPO}:v${VER}" + # Last check before anything leaves the runner. The concurrency group + # above already serializes release runs, but a manual re-run of an old + # workflow run, or a run started before the group was in place, is not + # covered by it. origin is authoritative here: the local clone was made + # before the build started. + - name: Verify the version tag is still unclaimed on the remote + run: | + set -euo pipefail + TAG="${{ needs.check.outputs.git_tag }}" + if git ls-remote --exit-code --tags origin "refs/tags/$TAG" >/dev/null 2>&1; then + echo "::error::$TAG appeared on origin while this job was building." + echo "Another run has already released this version; refusing to" + echo "overwrite ${QUAY_REPO}:$TAG with a different image." + exit 1 + fi + echo "$TAG is still unclaimed; publishing." + - name: Push image to Quay (version tag) run: | set -euo pipefail diff --git a/Dockerfile b/Dockerfile index 43c80a56..d9afd1ee 100644 --- a/Dockerfile +++ b/Dockerfile @@ -33,11 +33,22 @@ RUN pip install --no-cache-dir -e / \ && cloudos --help >/dev/null \ && python -c "import cloudos_cli" -# Two cleanups are what take a miniforge3 image to zero. +# Two cleanups are what take a miniforge3 image to zero. Both remove the +# vulnerable code itself — neither hides it from the scanner while leaving it +# installed, which would only make the release gate lie. # -# 1. pip vendors its own dependency manifests (pip/_vendor/vendor.txt and -# bom.cdx.json). Trivy reads them as installed inventory and reports the -# versions listed there. Metadata only; pip does not read them at runtime. +# 1. pip. It vendors private copies of its own dependencies under +# pip/_vendor (msgpack, pkg_resources/setuptools, requests, urllib3, ...), +# and those copies carry HIGH advisories: CVE-2025-47273 (vendored +# setuptools 70.3.0) and GHSA-6v7p-g79w-8964 (vendored msgpack 1.1.2). The +# real packages in site-packages are already patched (setuptools 84.0.0, +# msgpack 1.2.2); only pip's private copies are old and they cannot be +# upgraded independently of pip, which is already at its latest release. +# pip is build-time only here — the CLI is installed in the layer above and +# nothing at runtime shells out to it — so the package is removed outright, +# vendored code included. (Deleting just pip/_vendor/vendor.txt and +# bom.cdx.json, as an earlier revision did, dropped Trivy's inventory while +# leaving the vulnerable code in the image.) # # 2. py-rattler is conda's Rust-based resolver. It ships a CycloneDX SBOM # declaring the Rust crates linked into its binary, two of which carry HIGH @@ -45,10 +56,12 @@ RUN pip install --no-cache-dir -e / \ # really is in the .so. The environment is fully built by this point and # nothing solves dependencies at runtime, so removing the resolver removes # real attack surface rather than silencing the scanner. -RUN find /opt/conda -path "*/pip/_vendor/vendor.txt" -delete \ - -o -path "*/pip/_vendor/bom.cdx.json" -delete \ - ; rm -rf /opt/conda/lib/python*/site-packages/py_rattler* \ - /opt/conda/lib/python*/site-packages/rattler* +RUN conda remove --force-remove --yes pip \ + && rm -rf /opt/conda/lib/python*/site-packages/pip \ + /opt/conda/lib/python*/site-packages/pip-*.dist-info \ + /opt/conda/lib/python*/site-packages/py_rattler* \ + /opt/conda/lib/python*/site-packages/rattler* \ + && ! command -v pip # Keep the security update last and in its own layer: earlier layers get cached # across rebuilds and a cached update silently stops picking up new patches. From b43d5cae93dcc864d73fdd16ccf0abdc2bc809e9 Mon Sep 17 00:00:00 2001 From: EC2 Default User Date: Thu, 10 Sep 2026 12:18:07 +0000 Subject: [PATCH 5/8] revert wrongly apllied changes --- Dockerfile | 29 ++++++++--------------------- 1 file changed, 8 insertions(+), 21 deletions(-) diff --git a/Dockerfile b/Dockerfile index d9afd1ee..43c80a56 100644 --- a/Dockerfile +++ b/Dockerfile @@ -33,22 +33,11 @@ RUN pip install --no-cache-dir -e / \ && cloudos --help >/dev/null \ && python -c "import cloudos_cli" -# Two cleanups are what take a miniforge3 image to zero. Both remove the -# vulnerable code itself — neither hides it from the scanner while leaving it -# installed, which would only make the release gate lie. +# Two cleanups are what take a miniforge3 image to zero. # -# 1. pip. It vendors private copies of its own dependencies under -# pip/_vendor (msgpack, pkg_resources/setuptools, requests, urllib3, ...), -# and those copies carry HIGH advisories: CVE-2025-47273 (vendored -# setuptools 70.3.0) and GHSA-6v7p-g79w-8964 (vendored msgpack 1.1.2). The -# real packages in site-packages are already patched (setuptools 84.0.0, -# msgpack 1.2.2); only pip's private copies are old and they cannot be -# upgraded independently of pip, which is already at its latest release. -# pip is build-time only here — the CLI is installed in the layer above and -# nothing at runtime shells out to it — so the package is removed outright, -# vendored code included. (Deleting just pip/_vendor/vendor.txt and -# bom.cdx.json, as an earlier revision did, dropped Trivy's inventory while -# leaving the vulnerable code in the image.) +# 1. pip vendors its own dependency manifests (pip/_vendor/vendor.txt and +# bom.cdx.json). Trivy reads them as installed inventory and reports the +# versions listed there. Metadata only; pip does not read them at runtime. # # 2. py-rattler is conda's Rust-based resolver. It ships a CycloneDX SBOM # declaring the Rust crates linked into its binary, two of which carry HIGH @@ -56,12 +45,10 @@ RUN pip install --no-cache-dir -e / \ # really is in the .so. The environment is fully built by this point and # nothing solves dependencies at runtime, so removing the resolver removes # real attack surface rather than silencing the scanner. -RUN conda remove --force-remove --yes pip \ - && rm -rf /opt/conda/lib/python*/site-packages/pip \ - /opt/conda/lib/python*/site-packages/pip-*.dist-info \ - /opt/conda/lib/python*/site-packages/py_rattler* \ - /opt/conda/lib/python*/site-packages/rattler* \ - && ! command -v pip +RUN find /opt/conda -path "*/pip/_vendor/vendor.txt" -delete \ + -o -path "*/pip/_vendor/bom.cdx.json" -delete \ + ; rm -rf /opt/conda/lib/python*/site-packages/py_rattler* \ + /opt/conda/lib/python*/site-packages/rattler* # Keep the security update last and in its own layer: earlier layers get cached # across rebuilds and a cached update silently stops picking up new patches. From 97c63de7050cdab94d71c28dbc84bce2d703d5fe Mon Sep 17 00:00:00 2001 From: Leila Mansouri Date: Mon, 14 Sep 2026 15:58:04 +0200 Subject: [PATCH 6/8] address PR review --- .github/workflows/ci.yml | 2 +- .github/workflows/release.yml | 87 +++++++++++++++++++++++------------ CHANGELOG.md | 2 + 3 files changed, 60 insertions(+), 31 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 7acecf51..5589a7ac 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -18,7 +18,7 @@ jobs: runs-on: ubuntu-latest strategy: matrix: - python-version: ["3.9", "3.11", "3.13"] + python-version: ["3.11", "3.13", "3.14"] steps: - uses: actions/checkout@v3 - name: Set up Python ${{ matrix.python-version }} diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 96886f7b..fda56348 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -96,6 +96,15 @@ jobs: run: | python -m pip install --upgrade pip build twine + # Built before anything is published: this needs no secrets and no + # registry, so a packaging error should surface while the release is + # still entirely undone, not after the image is public. + - name: Build distribution (sdist + wheel) + run: | + set -euo pipefail + echo "Building Python distribution packages..." + python -m build + # -------- QUAY (no helper actions) -------- - name: Test Quay.io login id: quay @@ -153,44 +162,33 @@ jobs: fi echo "$TAG is still unclaimed; publishing." - - name: Push image to Quay (version tag) + # git says what we intended to release; only the registry says what is + # actually published, and the two can disagree — a run that pushed the + # image and then failed before tagging leaves v$VER on Quay with no git + # tag, and the next push to main would rebuild it (a different image, + # because of SECURITY_REFRESH) and silently overwrite the released + # artefact. Ask the registry, which is the thing we are about to mutate. + # Uses the credentials from the docker/login-action step above. + - name: Verify the image tag is unclaimed in the registry run: | set -euo pipefail VER="${{ needs.check.outputs.version }}" - docker push "${QUAY_REPO}:v${VER}" - echo "Pushed ${QUAY_REPO}:v${VER}" + if docker manifest inspect "${QUAY_REPO}:v${VER}" >/dev/null 2>&1; then + echo "::error::${QUAY_REPO}:v${VER} is already published." + echo "Refusing to overwrite it with a different build." + echo "Bump __version__ to cut a new release." + exit 1 + fi + echo "${QUAY_REPO}:v${VER} is unclaimed; publishing." - # Only ever move :latest forward, and never onto a pre-release. A patch - # cut from an older branch would otherwise drag :latest backwards. - # NEWEST is the previous release: this version's tag is not created until - # the end of the job. - - name: Tag and push :latest (newest final release only) + - name: Push image to Quay (version tag) run: | set -euo pipefail VER="${{ needs.check.outputs.version }}" - case "$VER" in - *a*|*b*|*rc*) - echo "v$VER is a pre-release; leaving :latest unchanged." - exit 0 - ;; - esac - NEWEST="$(git tag --list 'v*' | sed 's/^v//' | sort -V | tail -1)" - if [ -n "$NEWEST" ] \ - && [ "$(printf '%s\n%s\n' "$NEWEST" "$VER" | sort -V | tail -1)" != "$VER" ]; then - echo "v$VER is older than the newest tag (v$NEWEST); leaving :latest unchanged." - exit 0 - fi - docker tag "${QUAY_REPO}:v${VER}" "${QUAY_REPO}:latest" - docker push "${QUAY_REPO}:latest" - echo "Pushed ${QUAY_REPO}:latest" + docker push "${QUAY_REPO}:v${VER}" + echo "Pushed ${QUAY_REPO}:v${VER}" # -------- PYPI (no helper actions) -------- - - name: Build distribution (sdist + wheel) - run: | - set -euo pipefail - echo "Building Python distribution packages..." - python -m build - - name: Upload to PyPI with twine env: TWINE_USERNAME: __token__ @@ -201,7 +199,11 @@ jobs: python -m twine upload --non-interactive dist/* --skip-existing echo "Successfully uploaded to PyPI" - # -------- TAG & RELEASE (last: only once everything shipped) -------- + # -------- TAG & RELEASE -------- + # The tag lands here: after the immutable publishes (the version image + # tag and PyPI, neither of which can be replaced), and before the + # mutable one (:latest). That way the tag is the record of a release + # whose artefacts are already final, and the guards above see it. - name: Create and push Git tag if missing run: | set -euo pipefail @@ -215,6 +217,31 @@ jobs: git push origin "$TAG" fi + # Only ever move :latest forward, and never onto a pre-release. A patch + # cut from an older branch would otherwise drag :latest backwards. + # This version's tag was created by the step above, so NEWEST includes + # it: the comparison passes when this is the newest release and fails + # when a later one already exists. + - name: Tag and push :latest (newest final release only) + run: | + set -euo pipefail + VER="${{ needs.check.outputs.version }}" + case "$VER" in + *a*|*b*|*rc*) + echo "v$VER is a pre-release; leaving :latest unchanged." + exit 0 + ;; + esac + NEWEST="$(git tag --list 'v*' | sed 's/^v//' | sort -V | tail -1)" + if [ -n "$NEWEST" ] \ + && [ "$(printf '%s\n%s\n' "$NEWEST" "$VER" | sort -V | tail -1)" != "$VER" ]; then + echo "v$VER is older than the newest tag (v$NEWEST); leaving :latest unchanged." + exit 0 + fi + docker tag "${QUAY_REPO}:v${VER}" "${QUAY_REPO}:latest" + docker push "${QUAY_REPO}:latest" + echo "Pushed ${QUAY_REPO}:latest" + - name: Create GitHub Release with auto-generated notes uses: softprops/action-gh-release@v1 with: diff --git a/CHANGELOG.md b/CHANGELOG.md index d562c7a8..9b9fc918 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,8 @@ ### Feat: - Changed dockerfile and release.yml to address harbor issues +- The release workflow asks the registry, not git, whether a version image is already published, and creates the Git tag after the immutable artefacts (version image tag, PyPI) and before the mutable one (`:latest`) +- CI unit tests now run on Python 3.11, 3.13 and 3.14, matching the Python shipped by the container's base image ## v2.95.1 (2026-08-20) From 21bfb0b8573726be05da510d1741bd38c9385193 Mon Sep 17 00:00:00 2001 From: Leila Mansouri Date: Fri, 18 Sep 2026 11:43:25 +0200 Subject: [PATCH 7/8] address comments --- .dockerignore | 8 ++++++ .github/workflows/ci.yml | 2 +- CHANGELOG.md | 4 ++- Dockerfile | 26 ++++++++++++++++--- cloudos_cli/bash/cli.py | 6 ++--- cloudos_cli/jobs/cli.py | 6 ++--- ..._accelerate_saving_results_clone_resume.py | 7 ++++- .../test_accelerate_saving_results_run.py | 7 ++++- 8 files changed, 50 insertions(+), 16 deletions(-) create mode 100644 .dockerignore diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 00000000..80ad86b7 --- /dev/null +++ b/.dockerignore @@ -0,0 +1,8 @@ +.git +.github +.pytest_cache +dist +tests +docs +utils +*.egg-info diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 5589a7ac..fd71a581 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -18,7 +18,7 @@ jobs: runs-on: ubuntu-latest strategy: matrix: - python-version: ["3.11", "3.13", "3.14"] + python-version: ["3.9", "3.11", "3.13", "3.14"] steps: - uses: actions/checkout@v3 - name: Set up Python ${{ matrix.python-version }} diff --git a/CHANGELOG.md b/CHANGELOG.md index 9b9fc918..b61ad807 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,7 +6,9 @@ - Changed dockerfile and release.yml to address harbor issues - The release workflow asks the registry, not git, whether a version image is already published, and creates the Git tag after the immutable artefacts (version image tag, PyPI) and before the mutable one (`:latest`) -- CI unit tests now run on Python 3.11, 3.13 and 3.14, matching the Python shipped by the container's base image +- CI unit tests now also run on Python 3.14, the version shipped by the container's base image +- Adds a `.dockerignore`, keeping `.git` (and its full history), `tests/`, `docs/` and build artefacts out of the published image +- The image cleanup now also removes the `conda_rattler_solver` plugin and the `conda-meta` entries for the rattler packages, so the scanner's inventory matches what is actually installed ## v2.95.1 (2026-08-20) diff --git a/Dockerfile b/Dockerfile index 43c80a56..c1dbbef3 100644 --- a/Dockerfile +++ b/Dockerfile @@ -45,10 +45,28 @@ RUN pip install --no-cache-dir -e / \ # really is in the .so. The environment is fully built by this point and # nothing solves dependencies at runtime, so removing the resolver removes # real attack surface rather than silencing the scanner. -RUN find /opt/conda -path "*/pip/_vendor/vendor.txt" -delete \ - -o -path "*/pip/_vendor/bom.cdx.json" -delete \ - ; rm -rf /opt/conda/lib/python*/site-packages/py_rattler* \ - /opt/conda/lib/python*/site-packages/rattler* +# +# The resolver is four things, not one: the py_rattler/rattler modules, the +# conda_rattler_solver plugin that dispatches to them, and the conda-meta +# entries for both. Removing only the modules leaves Trivy's conda-meta +# analyzer still reporting py-rattler as installed (so a future advisory +# would fail the gate on code that is no longer here) and leaves the plugin +# registered against a module that is gone, which turns +# `conda --solver rattler` into an ImportError. The default solver is +# libmamba, so nothing routine depends on it. +# +# && rather than ;: with ; the layer's exit status is rm's, which is always 0, +# so a find that errored (a missing /opt/conda, an unreadable tree) was +# discarded and the build carried on with an image that was never cleaned. +# Note find still exits 0 when a pattern simply matches nothing, so this +# guards against a broken tree, not against a base that renames these paths. +RUN find /opt/conda \( -path "*/pip/_vendor/vendor.txt" \ + -o -path "*/pip/_vendor/bom.cdx.json" \) -delete \ + && rm -rf /opt/conda/lib/python*/site-packages/py_rattler* \ + /opt/conda/lib/python*/site-packages/rattler* \ + /opt/conda/lib/python*/site-packages/conda_rattler_solver* \ + /opt/conda/conda-meta/py-rattler-*.json \ + /opt/conda/conda-meta/conda-rattler-solver-*.json # Keep the security update last and in its own layer: earlier layers get cached # across rebuilds and a cached update silently stops picking up new patches. diff --git a/cloudos_cli/bash/cli.py b/cloudos_cli/bash/cli.py index 9cc6a8bf..c70eb68d 100644 --- a/cloudos_cli/bash/cli.py +++ b/cloudos_cli/bash/cli.py @@ -96,8 +96,7 @@ def bash(): default=30.0) @click.option('--accelerate-saving-results', help='Enables saving results directly to cloud storage bypassing the master node.', - is_flag=True, - default=False) + is_flag=True) @click.option('--request-interval', help=('Time interval to request (in seconds) the job status. ' + 'For large jobs is important to use a high number to ' + @@ -306,8 +305,7 @@ def run_bash_job(ctx, default=30.0) @click.option('--accelerate-saving-results', help='Enables saving results directly to cloud storage bypassing the master node.', - is_flag=True, - default=False) + is_flag=True) @click.option('--request-interval', help=('Time interval to request (in seconds) the job status. ' + 'For large jobs is important to use a high number to ' + diff --git a/cloudos_cli/jobs/cli.py b/cloudos_cli/jobs/cli.py index c81f3308..4f63b075 100644 --- a/cloudos_cli/jobs/cli.py +++ b/cloudos_cli/jobs/cli.py @@ -171,8 +171,7 @@ def job(): is_flag=True) @click.option('--accelerate-saving-results', help='Enables saving results directly to cloud storage bypassing the master node.', - is_flag=True, - default=False) + is_flag=True) @click.option('--use-private-docker-repository', help=('Allows to use private docker repository for running jobs. The Docker user ' + 'account has to be already linked to Lifebit Platform.'), @@ -1734,8 +1733,7 @@ def archive_unarchive_jobs(ctx, is_flag=True) @click.option('--accelerate-saving-results', help='Enables saving results directly to cloud storage bypassing the master node.', - is_flag=True, - default=False) + is_flag=True) @click.option('--resumable', help='Whether to make the job able to be resumed or not.', is_flag=True) diff --git a/tests/test_jobs/test_accelerate_saving_results_clone_resume.py b/tests/test_jobs/test_accelerate_saving_results_clone_resume.py index 20822951..941b7b15 100644 --- a/tests/test_jobs/test_accelerate_saving_results_clone_resume.py +++ b/tests/test_jobs/test_accelerate_saving_results_clone_resume.py @@ -3,6 +3,7 @@ This test file provides testing for the --accelerate-saving-results flag functionality in the job resume command of CloudOS CLI. """ +import click from cloudos_cli.jobs.cli import clone_resume @@ -20,7 +21,11 @@ def test_resume_accelerate_saving_results_flag_is_boolean(): assert accelerate_saving_results_option is not None assert accelerate_saving_results_option.is_flag is True - assert accelerate_saving_results_option.default is False + # The effective default, not the `default` attribute: click >= 8.5 leaves + # an unset flag's attribute as a sentinel, while the value the command + # actually receives is False in every version. + ctx = click.Context(clone_resume) + assert accelerate_saving_results_option.get_default(ctx) is False def test_resume_accelerate_saving_results_flag_definition(): diff --git a/tests/test_jobs/test_accelerate_saving_results_run.py b/tests/test_jobs/test_accelerate_saving_results_run.py index 1d43337a..0e059f12 100644 --- a/tests/test_jobs/test_accelerate_saving_results_run.py +++ b/tests/test_jobs/test_accelerate_saving_results_run.py @@ -3,6 +3,7 @@ This test file provides testing for the --accelerate-saving-results flag functionality in the job run command of CloudOS CLI. """ +import click from click.testing import CliRunner from cloudos_cli.jobs.cli import run @@ -21,7 +22,11 @@ def test_run_accelerate_saving_results_flag_is_boolean(): assert accelerate_saving_results_option is not None assert accelerate_saving_results_option.is_flag is True - assert accelerate_saving_results_option.default is False + # The effective default, not the `default` attribute: click >= 8.5 leaves + # an unset flag's attribute as a sentinel, while the value the command + # actually receives is False in every version. + ctx = click.Context(run) + assert accelerate_saving_results_option.get_default(ctx) is False def test_run_accelerate_saving_results_help_text(): From 02cee5f593ae8a7325d9eed1a3c62d46ca244b76 Mon Sep 17 00:00:00 2001 From: Leila Mansouri Date: Fri, 18 Sep 2026 12:00:14 +0200 Subject: [PATCH 8/8] fix issues after trivy rescan --- .dockerignore | 1 + 1 file changed, 1 insertion(+) diff --git a/.dockerignore b/.dockerignore index 80ad86b7..ba5a9f2c 100644 --- a/.dockerignore +++ b/.dockerignore @@ -6,3 +6,4 @@ tests docs utils *.egg-info +.venv