From c72b563b74a5533127a09ad5ba6f5185a2e62e77 Mon Sep 17 00:00:00 2001 From: mchekm Date: Thu, 13 Aug 2026 14:11:13 +0300 Subject: [PATCH 01/33] feat: TEM-42 ComfyUI migration --- .github/workflows/comfyui.yml | 147 ++++++++ official-templates/comfyui/Dockerfile | 244 +++++++++++++ official-templates/comfyui/README.md | 43 +++ official-templates/comfyui/docker-bake.hcl | 116 +++++++ .../comfyui/scripts/fetch-hashes.sh | 80 +++++ .../comfyui/scripts/prebake-manager-cache.py | 98 ++++++ official-templates/comfyui/scripts/start.sh | 325 ++++++++++++++++++ 7 files changed, 1053 insertions(+) create mode 100644 .github/workflows/comfyui.yml create mode 100644 official-templates/comfyui/Dockerfile create mode 100644 official-templates/comfyui/README.md create mode 100644 official-templates/comfyui/docker-bake.hcl create mode 100755 official-templates/comfyui/scripts/fetch-hashes.sh create mode 100644 official-templates/comfyui/scripts/prebake-manager-cache.py create mode 100755 official-templates/comfyui/scripts/start.sh diff --git a/.github/workflows/comfyui.yml b/.github/workflows/comfyui.yml new file mode 100644 index 00000000..e1cc84bb --- /dev/null +++ b/.github/workflows/comfyui.yml @@ -0,0 +1,147 @@ +name: ROCm Images Build + +# Reusable building block invoked by the release orchestrator (release.yml), +# which computes the version ONCE and passes it in. Not triggered on its own — +# the orchestrator owns triggers, per-family gating, concurrency and release. +on: + workflow_call: + inputs: + version: + description: "Semantic version without a leading v (e.g. 1.1.0)." + required: true + type: string + suffix: + description: "Tag suffix: empty for releases, -rc. for PRs, -dev otherwise." + required: false + default: "" + type: string + +permissions: + contents: read + +jobs: + build-rocm: + runs-on: blacksmith-8vcpu-ubuntu-2404 + strategy: + fail-fast: false + matrix: + rocm: [rocm644] + permissions: + contents: read + id-token: write + steps: + - name: Checkout + uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 + with: + fetch-depth: 0 + + - name: Setup Docker + uses: ./.github/actions/docker-setup + id: setup + with: + dockerhub-username: ${{ secrets.DOCKERHUB_USERNAME }} + dockerhub-token: ${{ secrets.DOCKERHUB_TOKEN }} + + - name: Build rocm images (${{ matrix.rocm }}) + id: build + uses: docker/bake-action@6614cfa25eff9a0b2b2697efb0b6159e7680d584 + env: + BUILDX_BAKE_ENTITLEMENTS_FS: 0 + RELEASE_VERSION: ${{ inputs.version }} + RELEASE_SUFFIX: ${{ inputs.suffix }} + with: + source: . + files: | + official-templates/shared/versions.hcl + official-templates/rocm/docker-bake.hcl + load: true + push: false + targets: | + ${{ matrix.rocm }} + + - name: Extract image refs + id: refs + uses: ./.github/actions/image-name + with: + bake-metadata: ${{ steps.build.outputs.metadata }} + + - name: Grype scan + uses: ./.github/actions/grype + with: + image-refs: ${{ steps.refs.outputs.refs }} + + - name: Push images + uses: ./.github/actions/docker-push + with: + image-refs: ${{ steps.refs.outputs.refs }} + ref-digests: ${{ steps.refs.outputs.ref-digests }} + + - name: Install Cosign + uses: sigstore/cosign-installer@6f9f17788090df1f26f669e9d70d6ae9567deba6 + with: + cosign-release: v2.6.0 + + - name: Cosign sign and verify + uses: ./.github/actions/cosign + with: + image-digests: ${{ steps.refs.outputs.digests }} + + - name: Save refs artifact + shell: bash + env: + REFS: ${{ steps.refs.outputs.refs }} + run: | + mkdir -p /tmp/rocm-refs + printf '%s\n' "$REFS" > /tmp/rocm-refs/refs.json + + - name: Upload refs artifact + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a + with: + name: rocm-refs-${{ matrix.rocm }} + path: /tmp/rocm-refs/refs.json + retention-days: 1 + if-no-files-found: error + + test-rocm: + runs-on: blacksmith-4vcpu-ubuntu-2404 + needs: build-rocm + if: needs.build-rocm.result == 'success' + strategy: + fail-fast: false + matrix: + rocm: [rocm644] + steps: + - name: Checkout + uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 + with: + fetch-depth: 0 + + - name: Download refs for ${{ matrix.rocm }} + uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c + with: + name: rocm-refs-${{ matrix.rocm }} + path: /tmp/rocm-refs + + - name: Read refs + id: refs + shell: bash + run: | + REFS=$(cat /tmp/rocm-refs/refs.json) + { + echo "refs<> "$GITHUB_OUTPUT" + + - name: Smoke test (${{ matrix.rocm }}) + uses: ./.github/actions/smoke-test + with: + image-refs: ${{ steps.refs.outputs.refs }} + profile: gpu + runpod-api-key: ${{ secrets.TESTING_RUNPOD_API_KEY }} + ssh-private-key: ${{ secrets.TESTING_RUNPOD_SSH_PRIVATE_KEY }} + on-skip: "warn" + manufacturer: "AMD" + budget-usd-per-hour: "3.0" + min-vram-gb: "80" + create-timeout: "1200" diff --git a/official-templates/comfyui/Dockerfile b/official-templates/comfyui/Dockerfile new file mode 100644 index 00000000..fe1f3d3c --- /dev/null +++ b/official-templates/comfyui/Dockerfile @@ -0,0 +1,244 @@ +# ============================================================================ +# Stage 1: Builder - Download pinned sources and install all Python packages +# ============================================================================ +FROM ubuntu:24.04 AS builder + +ENV DEBIAN_FRONTEND=noninteractive + +# ---- Version pins (set in docker-bake.hcl) ---- +ARG COMFYUI_VERSION +ARG MANAGER_SHA +ARG KJNODES_SHA +ARG CIVICOMFY_SHA +ARG RUNPODDIRECT_SHA +ARG TORCH_VERSION +ARG TORCHVISION_VERSION +ARG TORCHAUDIO_VERSION + +# ---- CUDA variant (set in docker-bake.hcl per target) ---- +ARG CUDA_VERSION_DASH=12-8 +ARG TORCH_INDEX_SUFFIX=cu128 + +# Install minimal dependencies needed for building +RUN apt-get update && \ + apt-get install -y --no-install-recommends \ + wget \ + curl \ + git \ + ca-certificates \ + python3.12 \ + python3.12-venv \ + python3.12-dev \ + build-essential \ + && wget https://developer.download.nvidia.com/compute/cuda/repos/ubuntu2404/x86_64/cuda-keyring_1.1-1_all.deb \ + && dpkg -i cuda-keyring_1.1-1_all.deb \ + && apt-get update \ + && apt-get install -y --no-install-recommends cuda-minimal-build-${CUDA_VERSION_DASH} libcusparse-dev-${CUDA_VERSION_DASH} \ + && apt-get clean \ + && rm -rf /var/lib/apt/lists/* \ + && rm cuda-keyring_1.1-1_all.deb \ + && rm -f /usr/lib/python3.12/EXTERNALLY-MANAGED + +# Install pip and pip-tools for lock file generation. +RUN curl -sS https://bootstrap.pypa.io/get-pip.py -o get-pip.py && \ + python3.12 get-pip.py "pip==26.1.2" && \ + python3.12 -m pip install --no-cache-dir "pip-tools==7.6.0" && \ + rm get-pip.py + +# Set CUDA environment for building +ENV PATH=/usr/local/cuda/bin:${PATH} +ENV LD_LIBRARY_PATH=/usr/local/cuda/lib64 + +# Download pinned source archives +WORKDIR /tmp/build +RUN curl -fSL "https://github.com/comfyanonymous/ComfyUI/archive/refs/tags/${COMFYUI_VERSION}.tar.gz" -o comfyui.tar.gz && \ + mkdir -p ComfyUI && tar xzf comfyui.tar.gz --strip-components=1 -C ComfyUI && rm comfyui.tar.gz + +WORKDIR /tmp/build/ComfyUI/custom_nodes +RUN curl -fSL "https://github.com/ltdrdata/ComfyUI-Manager/archive/${MANAGER_SHA}.tar.gz" -o manager.tar.gz && \ + mkdir -p ComfyUI-Manager && tar xzf manager.tar.gz --strip-components=1 -C ComfyUI-Manager && rm manager.tar.gz && \ + curl -fSL "https://github.com/kijai/ComfyUI-KJNodes/archive/${KJNODES_SHA}.tar.gz" -o kjnodes.tar.gz && \ + mkdir -p ComfyUI-KJNodes && tar xzf kjnodes.tar.gz --strip-components=1 -C ComfyUI-KJNodes && rm kjnodes.tar.gz && \ + curl -fSL "https://github.com/MoonGoblinDev/Civicomfy/archive/${CIVICOMFY_SHA}.tar.gz" -o civicomfy.tar.gz && \ + mkdir -p Civicomfy && tar xzf civicomfy.tar.gz --strip-components=1 -C Civicomfy && rm civicomfy.tar.gz && \ + curl -fSL "https://github.com/MadiatorLabs/ComfyUI-RunpodDirect/archive/${RUNPODDIRECT_SHA}.tar.gz" -o runpoddirect.tar.gz && \ + mkdir -p ComfyUI-RunpodDirect && tar xzf runpoddirect.tar.gz --strip-components=1 -C ComfyUI-RunpodDirect && rm runpoddirect.tar.gz + +# Init git repos with upstream remotes so ComfyUI-Manager can detect versions +# and users can update via Manager at their own risk +RUN cd /tmp/build/ComfyUI && \ + git init && git add -A && git -c user.name=- -c user.email=- commit -q -m "ComfyUI ${COMFYUI_VERSION}" && git tag "${COMFYUI_VERSION}" && \ + git remote add origin https://github.com/comfyanonymous/ComfyUI.git && \ + cd /tmp/build/ComfyUI/custom_nodes/ComfyUI-Manager && \ + git init && git add -A && git -c user.name=- -c user.email=- commit -q -m "ComfyUI-Manager ${MANAGER_SHA}" && \ + git remote add origin https://github.com/ltdrdata/ComfyUI-Manager.git && \ + cd /tmp/build/ComfyUI/custom_nodes/ComfyUI-KJNodes && \ + git init && git add -A && git -c user.name=- -c user.email=- commit -q -m "ComfyUI-KJNodes ${KJNODES_SHA}" && \ + git remote add origin https://github.com/kijai/ComfyUI-KJNodes.git && \ + cd /tmp/build/ComfyUI/custom_nodes/Civicomfy && \ + git init && git add -A && git -c user.name=- -c user.email=- commit -q -m "Civicomfy ${CIVICOMFY_SHA}" && \ + git remote add origin https://github.com/MoonGoblinDev/Civicomfy.git && \ + cd /tmp/build/ComfyUI/custom_nodes/ComfyUI-RunpodDirect && \ + git init && git add -A && git -c user.name=- -c user.email=- commit -q -m "ComfyUI-RunpodDirect ${RUNPODDIRECT_SHA}" && \ + git remote add origin https://github.com/MadiatorLabs/ComfyUI-RunpodDirect.git + +# Generate lock file from all requirements (including torch pins), then install with hash verification +WORKDIR /tmp/build +RUN cat ComfyUI/requirements.txt > requirements.in && \ + for node_dir in ComfyUI/custom_nodes/*/; do \ + if [ -f "$node_dir/requirements.txt" ]; then \ + cat "$node_dir/requirements.txt" >> requirements.in; \ + fi; \ + done && \ + echo "GitPython" >> requirements.in && \ + echo "opencv-python" >> requirements.in && \ + echo "jupyter" >> requirements.in && \ + echo "jupyter-resource-usage" >> requirements.in && \ + echo "jupyterlab-nvdashboard" >> requirements.in && \ + sed -i -E '/^[[:space:]]*(torch|torchvision|torchaudio)([[:space:]]|[\[<>=!~;#]|$)/d' requirements.in && \ + echo "torch==${TORCH_VERSION}" >> requirements.in && \ + echo "torchvision==${TORCHVISION_VERSION}" >> requirements.in && \ + echo "torchaudio==${TORCHAUDIO_VERSION}" >> requirements.in && \ + echo "pillow>=12.1.1" >> requirements.in && \ + TORCH_INDEX_URL="https://download.pytorch.org/whl/${TORCH_INDEX_SUFFIX}" && \ + PIP_INDEX_URL=https://pypi.org/simple \ + PIP_EXTRA_INDEX_URL="${TORCH_INDEX_URL}" \ + pip-compile --generate-hashes --output-file=requirements.lock --strip-extras --allow-unsafe requirements.in && \ + python3.12 -m pip install --no-cache-dir --ignore-installed --require-hashes \ + --index-url https://pypi.org/simple \ + --extra-index-url "${TORCH_INDEX_URL}" \ + -r requirements.lock && \ + TORCH_VERSION="${TORCH_VERSION}" TORCHVISION_VERSION="${TORCHVISION_VERSION}" TORCHAUDIO_VERSION="${TORCHAUDIO_VERSION}" \ + python3.12 -c 'import importlib.metadata as m, os, sys; expected = {"torch": os.environ["TORCH_VERSION"], "torchvision": os.environ["TORCHVISION_VERSION"], "torchaudio": os.environ["TORCHAUDIO_VERSION"]}; mismatches = [f"{pkg}: expected {version}, got {m.version(pkg)}" for pkg, version in expected.items() if m.version(pkg) != version]; sys.exit("\n".join(mismatches) if mismatches else 0)' + +# Pre-populate ComfyUI-Manager cache so first cold start skips the slow registry fetch +COPY scripts/prebake-manager-cache.py /tmp/prebake-manager-cache.py +RUN python3.12 /tmp/prebake-manager-cache.py /tmp/build/ComfyUI/user/__manager/cache + +# Bake ComfyUI + custom nodes into a known location for runtime copy +RUN printf '%s\n' \ + "COMFYUI_VERSION=${COMFYUI_VERSION}" \ + "MANAGER_SHA=${MANAGER_SHA}" \ + "KJNODES_SHA=${KJNODES_SHA}" \ + "CIVICOMFY_SHA=${CIVICOMFY_SHA}" \ + "RUNPODDIRECT_SHA=${RUNPODDIRECT_SHA}" \ + > /tmp/build/ComfyUI/.runpod-bundle-version && \ + cp -r /tmp/build/ComfyUI /opt/comfyui-baked + +# ============================================================================ +# Stage 2: Runtime - Clean image with pre-installed packages +# ============================================================================ +FROM ubuntu:24.04 + +ENV DEBIAN_FRONTEND=noninteractive +ENV PYTHONUNBUFFERED=1 +ENV IMAGEIO_FFMPEG_EXE=/usr/bin/ffmpeg +ENV FILEBROWSER_CONFIG=/workspace/runpod-slim/.filebrowser.json + +# ---- CUDA variant (re-declared for runtime stage) ---- +ARG CUDA_VERSION_DASH=12-8 +ARG TORCH_VERSION +ARG TORCHVISION_VERSION +ARG TORCHAUDIO_VERSION + +# ---- FileBrowser version pin (set in docker-bake.hcl) ---- +ARG FILEBROWSER_VERSION +ARG FILEBROWSER_SHA256 + +# Keep runtime pip installs aligned with the baked CUDA-specific PyTorch stack. +RUN printf "torch==%s\ntorchvision==%s\ntorchaudio==%s\n" \ + "$TORCH_VERSION" "$TORCHVISION_VERSION" "$TORCHAUDIO_VERSION" \ + > /opt/comfyui-runtime-constraints.txt + +# Update and install runtime dependencies, CUDA, and common tools +RUN apt-get update && \ + apt-get upgrade -y && \ + apt-get install -y --no-install-recommends \ + git \ + python3.12 \ + python3.12-venv \ + python3.12-dev \ + build-essential \ + libssl-dev \ + wget \ + gnupg \ + xz-utils \ + openssh-client \ + openssh-server \ + nano \ + curl \ + htop \ + tmux \ + ca-certificates \ + less \ + net-tools \ + iputils-ping \ + procps \ + openssl \ + ffmpeg \ + rsync \ + && wget https://developer.download.nvidia.com/compute/cuda/repos/ubuntu2404/x86_64/cuda-keyring_1.1-1_all.deb \ + && dpkg -i cuda-keyring_1.1-1_all.deb \ + && apt-get update \ + && apt-get install -y --no-install-recommends cuda-minimal-build-${CUDA_VERSION_DASH} \ + && apt-get clean \ + && rm -rf /var/lib/apt/lists/* \ + && rm cuda-keyring_1.1-1_all.deb \ + && rm -f /usr/lib/python3.12/EXTERNALLY-MANAGED + +# Copy Python packages, executables, and Jupyter data from builder stage +COPY --from=builder /usr/local/lib/python3.12 /usr/local/lib/python3.12 +COPY --from=builder /usr/local/bin /usr/local/bin +COPY --from=builder /usr/local/share/jupyter /usr/local/share/jupyter + +# Register Jupyter extensions (pip --ignore-installed skips post-install hooks) +RUN mkdir -p /usr/local/etc/jupyter/jupyter_server_config.d && \ + echo '{"ServerApp":{"jpserver_extensions":{"jupyter_server_terminals":true,"jupyterlab":true,"jupyter_resource_usage":true,"jupyterlab_nvdashboard":true}}}' \ + > /usr/local/etc/jupyter/jupyter_server_config.d/extensions.json + +# Copy baked ComfyUI + custom nodes from builder stage +COPY --from=builder /opt/comfyui-baked /opt/comfyui-baked + +# Remove uv to force ComfyUI-Manager to use pip (uv doesn't respect --system-site-packages properly) +RUN pip uninstall -y uv 2>/dev/null || true && \ + rm -f /usr/local/bin/uv /usr/local/bin/uvx + +# Install FileBrowser (pinned version with checksum) +RUN curl -fSL "https://github.com/filebrowser/filebrowser/releases/download/${FILEBROWSER_VERSION}/linux-amd64-filebrowser.tar.gz" -o /tmp/fb.tar.gz && \ + echo "${FILEBROWSER_SHA256} /tmp/fb.tar.gz" | sha256sum -c - && \ + tar xzf /tmp/fb.tar.gz -C /usr/local/bin filebrowser && \ + rm /tmp/fb.tar.gz + +# Set CUDA environment variables +ENV PATH=/usr/local/cuda/bin:${PATH} +ENV LD_LIBRARY_PATH=/usr/local/cuda/lib64 + +# Allow container to start on hosts with older CUDA 12.x drivers +ENV NVIDIA_REQUIRE_CUDA="" +ENV NVIDIA_DISABLE_REQUIRE=true +ENV NVIDIA_VISIBLE_DEVICES=all +ENV NVIDIA_DRIVER_CAPABILITIES=compute,utility,video + +# Jupyter is included in the lock file and installed in the builder stage + +# Configure SSH for root login +RUN sed -i 's/#PermitRootLogin prohibit-password/PermitRootLogin yes/' /etc/ssh/sshd_config && \ + sed -i 's/#PasswordAuthentication yes/PasswordAuthentication yes/' /etc/ssh/sshd_config && \ + mkdir -p /run/sshd && \ + rm -f /etc/ssh/ssh_host_* + +# Create workspace directory +RUN mkdir -p /workspace/runpod-slim +WORKDIR /workspace/runpod-slim + +# Expose ports +EXPOSE 8188 22 8888 8080 + +# Copy start script +COPY scripts/start.sh /start.sh + +# Set Python 3.12 as default +RUN update-alternatives --install /usr/bin/python3 python3 /usr/bin/python3.12 1 && \ + update-alternatives --set python3 /usr/bin/python3.12 + +ENTRYPOINT ["/start.sh"] diff --git a/official-templates/comfyui/README.md b/official-templates/comfyui/README.md new file mode 100644 index 00000000..eb0d5746 --- /dev/null +++ b/official-templates/comfyui/README.md @@ -0,0 +1,43 @@ +[![Watch the video](https://i3.ytimg.com/vi/JovhfHhxqdM/hqdefault.jpg)](https://www.youtube.com/watch?v=JovhfHhxqdM) + +Run the latest ComfyUI with CUDA 12.8. All dependencies are pre-installed in the image. On first boot, ComfyUI is copied to your workspace — when you see `[ComfyUI-Manager] All startup tasks have been completed.` in the logs, it's ready to use. + +> **This template is for CUDA 12 only.** It does not support CUDA 13 (Blackwell / RTX 5090). +> If you need CUDA 13, use our [ComfyUI CUDA 13 template](https://console.runpod.io/hub/template/comfyui-cuda-13?id=2lv7ev3wfp) instead. + +## Upgrading from a previous version + +If you have an existing pod created with an older version of this template (CUDA 12.4), a one-time migration is performed automatically on the next boot. ComfyUI and the bundled custom nodes are updated to the versions pinned by the image, while models, inputs, outputs, user settings, and user-installed custom nodes are preserved. The virtual environment is also migrated to CUDA 12.8 compatibility. This may take a few extra minutes on the first start after the update. + +## Access + +- `8188`: ComfyUI web UI +- `8080`: FileBrowser (admin / `FILEBROWSER_PASSWORD`, default: `adminadmin12`) +- `8888`: JupyterLab (token via `JUPYTER_PASSWORD`, root at `/workspace`) +- `22`: SSH (set `PUBLIC_KEY` or check logs for generated root password) + +## Pre-installed custom nodes + +- ComfyUI-Manager +- ComfyUI-KJNodes +- Civicomfy +- ComfyUI-RunpodDirect + +## Source Code + +This is an open source template. Source code available at: [github.com/runpod-workers/comfyui-base](https://github.com/runpod-workers/comfyui-base) + +## Custom Arguments + +Edit `/workspace/runpod-slim/comfyui_args.txt` (one arg per line): + +``` +--max-batch-size 8 +--preview-method auto +``` + +## Directory Structure + +- `/workspace/runpod-slim/ComfyUI`: ComfyUI install +- `/workspace/runpod-slim/comfyui_args.txt`: ComfyUI args +- `/workspace/runpod-slim/filebrowser.db`: FileBrowser DB diff --git a/official-templates/comfyui/docker-bake.hcl b/official-templates/comfyui/docker-bake.hcl new file mode 100644 index 00000000..dde5e5d7 --- /dev/null +++ b/official-templates/comfyui/docker-bake.hcl @@ -0,0 +1,116 @@ +# === Version Pins (single source of truth) === +variable "COMFYUI_VERSION" { + default = "v0.30.0" +} +variable "MANAGER_SHA" { + default = "c352b16bb186" +} +variable "KJNODES_SHA" { + default = "bc8e4ce4254b" +} +variable "CIVICOMFY_SHA" { + default = "555e984bbcb0" +} +variable "RUNPODDIRECT_SHA" { + default = "809065c9d2f3" +} +variable "FILEBROWSER_VERSION" { + default = "v2.59.0" +} +variable "FILEBROWSER_SHA256" { + default = "8cd8c3baecb086028111b912f252a6e3169737fa764b5c510139e81f9da87799" +} + +variable "CUDA_TORCH_COMBINATIONS" { + default = [ + { cuda_version = "12.8", + // torch_index_suffix = "cu128", + // cuda_version_dash = "12-8", + // torch_version = "2.10.0+cu128", + // torchvision_version = "0.25.0+cu128", + // torchaudio_version = "2.10.0+cu128", + torch_version = "2.10.0", + torchvision_version = "0.25.0", + torchaudio_version = "2.10.0" + }, + { cuda_version = "13.0", + // torch_index_suffix = "cu130", + // cuda_version_dash = "13-0", + torch_version = "2.10.0", + torchvision_version = "0.25.0", + torchaudio_version = "2.10.0" + } + ] +} + +variable "COMPATIBLE_BUILDS" { + default = flatten([ + for combination in CUDA_TORCH_COMBINATIONS: + [ + { cuda_version = combination.cuda_version, + cuda_version_code = replace(combination.cuda_version, ".", "") + + }, + ] + ] + ) +} + +group "default" { + targets = [ + for combination in CUDA_TORCH_COMBINATIONS: + "cuda${combination.cuda_version_code}" + ] +} + +group "cuda128" { + targets = [ + for combination in CUDA_TORCH_COMBINATIONS: + "cuda${combination.cuda_version_code}" + if combination.cuda_version == "12.8" + ] +} + +group "cuda13" { + + targets = [ + for combination in CUDA_TORCH_COMBINATIONS: + "cuda${combination.cuda_version_code}" + if combination.cuda_version == "13.0" + ] +} + +# Common settings for all targets (defaults to regular CUDA 12.8 / cu128) +target "common" { + context = "." + dockerfile = "Dockerfile" + platforms = ["linux/amd64"] +} + +target "comfyui-matrix" { + inherits = ["common"] + matrix = { + build = COMPATIBLE_BUILDS + } + + name = "cuda${build.cuda_version_code}" + + args = { + COMFYUI_VERSION = COMFYUI_VERSION + MANAGER_SHA = MANAGER_SHA + KJNODES_SHA = KJNODES_SHA + CIVICOMFY_SHA = CIVICOMFY_SHA + RUNPODDIRECT_SHA = RUNPODDIRECT_SHA + FILEBROWSER_VERSION = FILEBROWSER_VERSION + FILEBROWSER_SHA256 = FILEBROWSER_SHA256 + TORCH_VERSION = TORCH_VERSION_5090 + TORCHVISION_VERSION = TORCHVISION_VERSION_5090 + TORCHAUDIO_VERSION = TORCHAUDIO_VERSION_5090 + CUDA_VERSION_DASH = "13-0" + TORCH_INDEX_SUFFIX = "cu130" + } + + tags = [ + "runpod/comfyui:${RELEASE_VERSION}${RELEASE_SUFFIX}-comfyui${build.comfyui_code}-cuda${build.cuda_code}" + ] +} \ No newline at end of file diff --git a/official-templates/comfyui/scripts/fetch-hashes.sh b/official-templates/comfyui/scripts/fetch-hashes.sh new file mode 100755 index 00000000..78e2cd9e --- /dev/null +++ b/official-templates/comfyui/scripts/fetch-hashes.sh @@ -0,0 +1,80 @@ +#!/usr/bin/env bash +# fetch-hashes.sh — Query GitHub for latest custom node commit SHAs. +# Prints HCL-formatted output for copy-paste into docker-bake.hcl. +# +# Usage: +# ./scripts/fetch-hashes.sh +# GITHUB_TOKEN=ghp_xxx ./scripts/fetch-hashes.sh # higher rate limit +# +# Works on both Linux (bash 4+) and macOS (bash 3.2). + +set -euo pipefail + +NODES=" +ltdrdata/ComfyUI-Manager|MANAGER_SHA +kijai/ComfyUI-KJNodes|KJNODES_SHA +MoonGoblinDev/Civicomfy|CIVICOMFY_SHA +MadiatorLabs/ComfyUI-RunpodDirect|RUNPODDIRECT_SHA +" + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +BAKE_FILE="$SCRIPT_DIR/../docker-bake.hcl" + +if [[ ! -f "$BAKE_FILE" ]]; then + echo "ERROR: docker-bake.hcl not found at $BAKE_FILE" >&2 + exit 1 +fi + +get_current_hash() { + local var_name="$1" + grep -A2 "variable \"${var_name}\"" "$BAKE_FILE" | sed -n 's/.*default *= *"\([^"]*\)".*/\1/p' | head -1 || echo "unknown" +} + +fetch_latest_sha() { + local repo="$1" + local response + local auth_header="" + [[ -n "${GITHUB_TOKEN:-}" ]] && auth_header="Authorization: Bearer $GITHUB_TOKEN" + + if [[ -n "$auth_header" ]]; then + response=$(curl -fsSL -H "$auth_header" \ + -H "Accept: application/vnd.github.v3+json" \ + "https://api.github.com/repos/${repo}/commits?per_page=1" 2>/dev/null) || { echo "ERROR"; return; } + else + response=$(curl -fsSL \ + -H "Accept: application/vnd.github.v3+json" \ + "https://api.github.com/repos/${repo}/commits?per_page=1" 2>/dev/null) || { echo "ERROR"; return; } + fi + echo "$response" | sed -n 's/.*"sha" *: *"\([a-f0-9]\{12\}\).*/\1/p' | head -1 +} + +echo "# Updated custom node hashes ($(date +%Y-%m-%d))" +echo "# Paste these into docker-bake.hcl to update" +echo "" + +while IFS='|' read -r repo var_name; do + [[ -z "$repo" ]] && continue + + current=$(get_current_hash "$var_name") + latest=$(fetch_latest_sha "$repo") + + if [[ "$latest" == "ERROR" ]]; then + echo "# ${var_name}: FAILED to fetch from ${repo}" >&2 + echo "variable \"${var_name}\" {" + echo " default = \"${current}\"" + echo "}" + continue + fi + + if [[ "$current" == "$latest" ]]; then + echo "# ${var_name}: ${current} (unchanged)" + else + echo "# ${var_name}: ${current} -> ${latest} (CHANGED)" + fi + echo "variable \"${var_name}\" {" + echo " default = \"${latest}\"" + echo "}" +done <<< "$NODES" + +echo "" +echo "# ^ Copy the variable blocks above into docker-bake.hcl" diff --git a/official-templates/comfyui/scripts/prebake-manager-cache.py b/official-templates/comfyui/scripts/prebake-manager-cache.py new file mode 100644 index 00000000..a26fbf7b --- /dev/null +++ b/official-templates/comfyui/scripts/prebake-manager-cache.py @@ -0,0 +1,98 @@ +#!/usr/bin/env python3 +"""Pre-populate ComfyUI-Manager cache at Docker build time. + +Downloads the registry JSON files and saves them with the CRC32-prefixed +filenames that ComfyUI-Manager expects, so the first cold start skips +the slow paginated fetch from api.comfy.org (~127 requests). + +Cache expires after 24h; after that Manager re-fetches in the background. +""" + +import json +import sys +import zlib +from pathlib import Path +from urllib.request import urlopen, Request + +CACHE_DIR = Path(sys.argv[1]) if len(sys.argv) > 1 else Path("/opt/comfyui-baked/user/__manager/cache") + +# GitHub-hosted JSON files (fast, single request each) +GITHUB_URLS = [ + "https://raw.githubusercontent.com/ltdrdata/ComfyUI-Manager/main/alter-list.json", + "https://raw.githubusercontent.com/ltdrdata/ComfyUI-Manager/main/model-list.json", + "https://raw.githubusercontent.com/ltdrdata/ComfyUI-Manager/main/github-stats.json", + "https://raw.githubusercontent.com/ltdrdata/ComfyUI-Manager/main/extension-node-map.json", + "https://raw.githubusercontent.com/ltdrdata/ComfyUI-Manager/main/custom-node-list.json", +] + +# Paginated ComfyRegistry API +REGISTRY_URL = "https://api.comfy.org/nodes" +REGISTRY_PAGE_SIZE = 30 + + +def cache_filename(url: str) -> str: + """Compute CRC32-prefixed filename matching ComfyUI-Manager's convention.""" + h = zlib.crc32(url.encode()) & 0xFFFFFFFF + name = url.rsplit("/", 1)[-1] + if not name.endswith(".json"): + name += ".json" + return f"{h}_{name}" + + +def fetch_json(url: str) -> bytes: + """Fetch URL and return raw bytes.""" + req = Request(url, headers={"User-Agent": "ComfyUI-Docker-Build"}) + with urlopen(req, timeout=30) as resp: + return resp.read() + + +def fetch_registry_all() -> list: + """Fetch all pages from the ComfyRegistry API.""" + all_nodes = [] + page = 1 + + # First request to get total pages + url = f"{REGISTRY_URL}?page={page}&limit={REGISTRY_PAGE_SIZE}" + data = json.loads(fetch_json(url)) + total_pages = data["totalPages"] + all_nodes.extend(data["nodes"]) + print(f" Registry: page 1/{total_pages} ({len(data['nodes'])} nodes)") + + for page in range(2, total_pages + 1): + url = f"{REGISTRY_URL}?page={page}&limit={REGISTRY_PAGE_SIZE}" + data = json.loads(fetch_json(url)) + all_nodes.extend(data["nodes"]) + if page % 20 == 0 or page == total_pages: + print(f" Registry: page {page}/{total_pages} ({len(all_nodes)} nodes total)") + + return all_nodes + + +def main(): + CACHE_DIR.mkdir(parents=True, exist_ok=True) + print(f"Pre-populating ComfyUI-Manager cache in {CACHE_DIR}") + + # Fetch GitHub JSON files + for url in GITHUB_URLS: + fname = cache_filename(url) + print(f" Fetching {url.rsplit('/', 1)[-1]}") + data = fetch_json(url) + (CACHE_DIR / fname).write_bytes(data) + + # Fetch paginated registry and save as aggregated JSON + # Non-fatal: if the registry is down, Manager will fetch on first start + print(" Fetching ComfyRegistry (paginated)...") + try: + nodes = fetch_registry_all() + registry_data = json.dumps(nodes, separators=(",", ":")) + fname = cache_filename(REGISTRY_URL) + (CACHE_DIR / fname).write_bytes(registry_data.encode()) + print(f" Cached {len(nodes)} registry nodes") + except Exception as e: + print(f" WARNING: Registry fetch failed ({e}), skipping — Manager will fetch on first start") + + print(f"Done. {len(list(CACHE_DIR.iterdir()))} cache files written.") + + +if __name__ == "__main__": + main() diff --git a/official-templates/comfyui/scripts/start.sh b/official-templates/comfyui/scripts/start.sh new file mode 100755 index 00000000..049f6f7b --- /dev/null +++ b/official-templates/comfyui/scripts/start.sh @@ -0,0 +1,325 @@ +#!/bin/bash +set -e # Exit the script if any statement returns a non-true return value + +COMFYUI_DIR="/workspace/runpod-slim/ComfyUI" +BAKED_COMFYUI_DIR="/opt/comfyui-baked" +BUNDLE_VERSION_FILE=".runpod-bundle-version" +VENV_DIR="$COMFYUI_DIR/.venv-cu128" +OLD_VENV_DIR="$COMFYUI_DIR/.venv" +FILEBROWSER_CONFIG="/root/.config/filebrowser/config.json" +DB_FILE="/workspace/runpod-slim/filebrowser.db" +PIP_CONSTRAINT_FILE="/opt/comfyui-runtime-constraints.txt" +BAKED_NODES=("ComfyUI-Manager" "ComfyUI-KJNodes" "Civicomfy" "ComfyUI-RunpodDirect") + +# ---------------------------------------------------------------------------- # +# Function Definitions # +# ---------------------------------------------------------------------------- # + +# Setup SSH with optional key or random password +setup_ssh() { + mkdir -p ~/.ssh + + if [ ! -f /etc/ssh/ssh_host_ed25519_key ]; then + ssh-keygen -A -q + fi + + # If PUBLIC_KEY is provided, use it + if [[ $PUBLIC_KEY ]]; then + echo "$PUBLIC_KEY" >> ~/.ssh/authorized_keys + chmod 700 -R ~/.ssh + else + # Generate random password if no public key + RANDOM_PASS=$(openssl rand -base64 12) + echo "root:${RANDOM_PASS}" | chpasswd + echo "Generated random SSH password for root: ${RANDOM_PASS}" + fi + + # Configure SSH to preserve environment variables + echo "PermitUserEnvironment yes" >> /etc/ssh/sshd_config + + # Start SSH service + /usr/sbin/sshd +} + +# Export environment variables +export_env_vars() { + echo "Exporting environment variables..." + + # Create environment files + ENV_FILE="/etc/environment" + PAM_ENV_FILE="/etc/security/pam_env.conf" + SSH_ENV_DIR="/root/.ssh/environment" + + # Backup original files + cp "$ENV_FILE" "${ENV_FILE}.bak" 2>/dev/null || true + cp "$PAM_ENV_FILE" "${PAM_ENV_FILE}.bak" 2>/dev/null || true + + # Clear files + > "$ENV_FILE" + > "$PAM_ENV_FILE" + mkdir -p /root/.ssh + > "$SSH_ENV_DIR" + + # Export to multiple locations for maximum compatibility + printenv | grep -E '^RUNPOD_|^PATH=|^_=|^CUDA|^LD_LIBRARY_PATH|^PYTHONPATH|^PIP_CONSTRAINT=' | while read -r line; do + # Get variable name and value + name=$(echo "$line" | cut -d= -f1) + value=$(echo "$line" | cut -d= -f2-) + + # Add to /etc/environment (system-wide) + echo "$name=\"$value\"" >> "$ENV_FILE" + + # Add to PAM environment + echo "$name DEFAULT=\"$value\"" >> "$PAM_ENV_FILE" + + # Add to SSH environment file + echo "$name=\"$value\"" >> "$SSH_ENV_DIR" + + # Add to current shell + echo "export $name=\"$value\"" >> /etc/rp_environment + done + + # Add sourcing to shell startup files + echo 'source /etc/rp_environment' >> ~/.bashrc + echo 'source /etc/rp_environment' >> /etc/bash.bashrc + + # Set permissions + chmod 644 "$ENV_FILE" "$PAM_ENV_FILE" + chmod 600 "$SSH_ENV_DIR" +} + +# Start Jupyter Lab server for remote access +start_jupyter() { + mkdir -p /workspace + echo "Starting Jupyter Lab on port 8888..." + nohup jupyter lab \ + --allow-root \ + --no-browser \ + --port=8888 \ + --ip=0.0.0.0 \ + --FileContentsManager.delete_to_trash=False \ + --FileContentsManager.preferred_dir=/workspace \ + --ServerApp.root_dir=/workspace \ + --ServerApp.terminado_settings='{"shell_command":["/bin/bash"]}' \ + --IdentityProvider.token="${JUPYTER_PASSWORD:-}" \ + --ServerApp.allow_origin=* &> /jupyter.log & + echo "Jupyter Lab started" +} + +# Upgrade the image-managed ComfyUI files while leaving user data on the +# persistent workspace untouched. +upgrade_comfyui_if_needed() { + local baked_manifest="$BAKED_COMFYUI_DIR/$BUNDLE_VERSION_FILE" + local installed_manifest="$COMFYUI_DIR/$BUNDLE_VERSION_FILE" + + # A missing workspace is handled by the first-time setup below. + if [ ! -d "$COMFYUI_DIR" ]; then + return + fi + + if [ ! -f "$baked_manifest" ]; then + echo "WARNING: Baked ComfyUI bundle manifest is missing; skipping upgrade" + return + fi + + if [ -f "$installed_manifest" ] && cmp -s "$baked_manifest" "$installed_manifest"; then + echo "Using existing ComfyUI installation (bundle is current)" + return + fi + + echo "=============================================" + echo " Upgrading ComfyUI workspace from baked bundle" + echo " Preserving models, user data, and custom nodes" + echo "=============================================" + + # Sync ComfyUI core and remove files that no longer exist in the new + # release. Excluded paths belong to the user or are managed separately. + rsync -a --delete \ + --exclude="/$BUNDLE_VERSION_FILE" \ + --exclude="/.venv*" \ + --exclude="/models" \ + --exclude="/input" \ + --exclude="/output" \ + --exclude="/user" \ + --exclude="/custom_nodes" \ + --exclude="/extra_model_paths.yaml" \ + "$BAKED_COMFYUI_DIR/" "$COMFYUI_DIR/" + + mkdir -p "$COMFYUI_DIR/custom_nodes" + + # Update files located directly under custom_nodes without deleting + # user-provided files or directories. + rsync -a --exclude="*/" \ + "$BAKED_COMFYUI_DIR/custom_nodes/" "$COMFYUI_DIR/custom_nodes/" + + # Image-managed nodes are pinned with the image and must be upgraded. + # Other custom-node directories are user-owned and remain untouched. + local node + for node in "${BAKED_NODES[@]}"; do + if [ -d "$BAKED_COMFYUI_DIR/custom_nodes/$node" ]; then + mkdir -p "$COMFYUI_DIR/custom_nodes/$node" + rsync -a --delete \ + "$BAKED_COMFYUI_DIR/custom_nodes/$node/" \ + "$COMFYUI_DIR/custom_nodes/$node/" + fi + done + + # Write the manifest only after every sync succeeds. An interrupted + # migration is retried on the next container start. + cp "$baked_manifest" "${installed_manifest}.tmp" + mv "${installed_manifest}.tmp" "$installed_manifest" + echo "ComfyUI workspace upgraded successfully" +} + +# ---------------------------------------------------------------------------- # +# Main Program # +# ---------------------------------------------------------------------------- # + +# Setup environment +if [ -f "$PIP_CONSTRAINT_FILE" ]; then + export PIP_CONSTRAINT="$PIP_CONSTRAINT_FILE" + echo "Using runtime pip constraints from $PIP_CONSTRAINT_FILE" +fi + +setup_ssh +export_env_vars + +# Initialize FileBrowser if not already done +if [ ! -f "$DB_FILE" ]; then + echo "Initializing FileBrowser..." + filebrowser config init + filebrowser config set --address 0.0.0.0 + filebrowser config set --port 8080 + filebrowser config set --root /workspace + filebrowser config set --auth.method=json + filebrowser users add admin "${FILEBROWSER_PASSWORD:-adminadmin12}" --perm.admin +else + echo "Using existing FileBrowser configuration..." +fi + +# Start FileBrowser +echo "Starting FileBrowser on port 8080..." +nohup filebrowser &> /filebrowser.log & + +start_jupyter + +# Create default comfyui_args.txt if it doesn't exist +ARGS_FILE="/workspace/runpod-slim/comfyui_args.txt" +if [ ! -f "$ARGS_FILE" ]; then + echo "# Add your custom ComfyUI arguments here (one per line)" > "$ARGS_FILE" + echo "Created empty ComfyUI arguments file at $ARGS_FILE" +fi + +upgrade_comfyui_if_needed + +# Migrate old CUDA 12.4 venv to cu128 +if [ -d "$OLD_VENV_DIR" ] && [ ! -d "$VENV_DIR" ]; then + NODE_COUNT=$(find "$COMFYUI_DIR/custom_nodes" -maxdepth 2 -name "requirements.txt" 2>/dev/null | wc -l) + echo "=============================================" + echo " CUDA 12.4 -> 12.8 migration" + echo " Reinstalling deps for $NODE_COUNT custom nodes" + echo " This may take several minutes" + echo "=============================================" + mv "$OLD_VENV_DIR" "${OLD_VENV_DIR}.bak" + cd "$COMFYUI_DIR" + python3.12 -m venv --system-site-packages "$VENV_DIR" + source "$VENV_DIR/bin/activate" + python -m ensurepip + # Skip nodes baked into the image — their deps are in system site-packages + BAKED_NODES="ComfyUI-Manager ComfyUI-KJNodes Civicomfy ComfyUI-RunpodDirect" + CURRENT=0 + INSTALLED=0 + for req in "$COMFYUI_DIR"/custom_nodes/*/requirements.txt; do + if [ -f "$req" ]; then + NODE_NAME=$(basename "$(dirname "$req")") + case " $BAKED_NODES " in + *" $NODE_NAME "*) continue ;; + esac + CURRENT=$((CURRENT + 1)) + echo "[$CURRENT] $NODE_NAME" + pip install -r "$req" 2>&1 | grep -E "^(Successfully|ERROR)" || true + INSTALLED=$((INSTALLED + 1)) + fi + done + echo "Ensuring ComfyUI requirements are present..." + pip install -r "$COMFYUI_DIR/requirements.txt" 2>&1 | grep -E "^(Successfully|ERROR)" || true + echo "Migration complete — $INSTALLED user nodes processed (${NODE_COUNT} total, baked nodes skipped)" + echo "Old venv backed up at ${OLD_VENV_DIR}.bak — delete it to free space:" + echo " rm -rf ${OLD_VENV_DIR}.bak" +fi + +# Setup ComfyUI if needed +if [ ! -d "$COMFYUI_DIR" ] || [ ! -d "$VENV_DIR" ]; then + echo "First time setup: Copying baked ComfyUI to workspace..." + + # Copy baked ComfyUI from image (no git, no network) + if [ ! -d "$COMFYUI_DIR" ]; then + cp -r /opt/comfyui-baked "$COMFYUI_DIR" + echo "ComfyUI copied to workspace" + fi + + # Create venv with access to system packages (torch, numpy, etc. pre-installed in image) + if [ ! -d "$VENV_DIR" ]; then + cd "$COMFYUI_DIR" + python3.12 -m venv --system-site-packages "$VENV_DIR" + source "$VENV_DIR/bin/activate" + + # Ensure pip is available in the venv (needed for ComfyUI-Manager) + python -m ensurepip + + echo "Base packages (torch, numpy, etc.) available from system site-packages" + echo "ComfyUI ready — all dependencies pre-installed in image" + fi +else + # Just activate the existing venv + source "$VENV_DIR/bin/activate" + echo "Using existing ComfyUI installation" +fi + +# Warm up pip so ComfyUI-Manager's 5s timeout check doesn't fail on cold start. +# Log wall time — Manager fails if `python -m pip --version` takes >5s. +echo "Warming up pip (Manager timeout is 5s)..." +time python -m pip --version + +# Start ComfyUI — keep container alive if it crashes so SSH/Jupyter remain accessible +cd $COMFYUI_DIR +FIXED_ARGS="--listen 0.0.0.0 --port 8188 --enable-cors-header" +if [ -s "$ARGS_FILE" ]; then + CUSTOM_ARGS=$(grep -v '^#' "$ARGS_FILE" | tr '\n' ' ') + if [ ! -z "$CUSTOM_ARGS" ]; then + FIXED_ARGS="$FIXED_ARGS $CUSTOM_ARGS" + fi +fi + +echo "Starting ComfyUI with args: $FIXED_ARGS" +python main.py $FIXED_ARGS & +COMFY_PID=$! + +# Distinguish a real ComfyUI crash from the pod being stopped/restarted/ +# terminated (RunPod sends SIGTERM to PID 1, which we forward to ComfyUI — +# without the flag the crash banner would print on every normal shutdown). +SHUTTING_DOWN=0 +trap 'SHUTTING_DOWN=1; kill $COMFY_PID 2>/dev/null' SIGTERM SIGINT + +COMFY_EXIT=0 +wait $COMFY_PID || COMFY_EXIT=$? + +if [ "$SHUTTING_DOWN" = "1" ]; then + echo "Pod is shutting down (stop/restart/terminate) — stopping ComfyUI, Jupyter and FileBrowser." + # Docker only signals PID 1; stop the nohup'd background services too so + # they exit cleanly instead of waiting for SIGKILL. + pkill -TERM -f "jupyter-lab" 2>/dev/null || true + pkill -TERM -x "filebrowser" 2>/dev/null || true + exit 0 +fi + +echo "=============================================" +echo " ComfyUI exited unexpectedly (exit code $COMFY_EXIT)." +echo " Check the logs above for the error/traceback." +echo " SSH and JupyterLab are still available." +echo " To restart after fixing:" +echo " cd $COMFYUI_DIR && source .venv-cu128/bin/activate" +echo " python main.py $FIXED_ARGS" +echo "=============================================" + +sleep infinity From ab14a5eaa06190fd0a5c5f9c88287ee3a1c25992 Mon Sep 17 00:00:00 2001 From: mchekm Date: Tue, 25 Aug 2026 10:54:57 +0300 Subject: [PATCH 02/33] feat: TEM-42 ComfyUI migration --- .github/workflows/comfyui.yml | 39 +++++++++++----------- official-templates/comfyui/docker-bake.hcl | 20 ++++++----- 2 files changed, 31 insertions(+), 28 deletions(-) diff --git a/.github/workflows/comfyui.yml b/.github/workflows/comfyui.yml index e1cc84bb..16e87b63 100644 --- a/.github/workflows/comfyui.yml +++ b/.github/workflows/comfyui.yml @@ -20,12 +20,12 @@ permissions: contents: read jobs: - build-rocm: + build-comfyui: runs-on: blacksmith-8vcpu-ubuntu-2404 strategy: fail-fast: false matrix: - rocm: [rocm644] + comfyui: [cuda128, cuda130] permissions: contents: read id-token: write @@ -42,7 +42,7 @@ jobs: dockerhub-username: ${{ secrets.DOCKERHUB_USERNAME }} dockerhub-token: ${{ secrets.DOCKERHUB_TOKEN }} - - name: Build rocm images (${{ matrix.rocm }}) + - name: Build comfyui images (${{ matrix.comfyui }}) id: build uses: docker/bake-action@6614cfa25eff9a0b2b2697efb0b6159e7680d584 env: @@ -52,12 +52,11 @@ jobs: with: source: . files: | - official-templates/shared/versions.hcl - official-templates/rocm/docker-bake.hcl + official-templates/comfyui/docker-bake.hcl load: true push: false targets: | - ${{ matrix.rocm }} + ${{ matrix.comfyui }} - name: Extract image refs id: refs @@ -91,49 +90,49 @@ jobs: env: REFS: ${{ steps.refs.outputs.refs }} run: | - mkdir -p /tmp/rocm-refs - printf '%s\n' "$REFS" > /tmp/rocm-refs/refs.json + mkdir -p /tmp/comfyui-refs + printf '%s\n' "$REFS" > /tmp/comfyui-refs/refs.json - name: Upload refs artifact uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a with: - name: rocm-refs-${{ matrix.rocm }} - path: /tmp/rocm-refs/refs.json + name: comfyui-refs-${{ matrix.comfyui }} + path: /tmp/comfyui-refs/refs.json retention-days: 1 if-no-files-found: error - test-rocm: + test-comfyui: runs-on: blacksmith-4vcpu-ubuntu-2404 - needs: build-rocm - if: needs.build-rocm.result == 'success' + needs: build-comfyui + if: needs.build-comfyui.result == 'success' strategy: fail-fast: false matrix: - rocm: [rocm644] + comfyui: [cuda128, cuda130] steps: - name: Checkout uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 with: fetch-depth: 0 - - name: Download refs for ${{ matrix.rocm }} + - name: Download refs for ${{ matrix.comfyui }} uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c with: - name: rocm-refs-${{ matrix.rocm }} - path: /tmp/rocm-refs + name: comfyui-refs-${{ matrix.comfyui }} + path: /tmp/comfyui-refs - name: Read refs id: refs shell: bash run: | - REFS=$(cat /tmp/rocm-refs/refs.json) + REFS=$(cat /tmp/comfyui-refs/refs.json) { echo "refs<> "$GITHUB_OUTPUT" - - name: Smoke test (${{ matrix.rocm }}) + - name: Smoke test (${{ matrix.comfyui }}) uses: ./.github/actions/smoke-test with: image-refs: ${{ steps.refs.outputs.refs }} @@ -141,7 +140,7 @@ jobs: runpod-api-key: ${{ secrets.TESTING_RUNPOD_API_KEY }} ssh-private-key: ${{ secrets.TESTING_RUNPOD_SSH_PRIVATE_KEY }} on-skip: "warn" - manufacturer: "AMD" + manufacturer: "NVIDIA" budget-usd-per-hour: "3.0" min-vram-gb: "80" create-timeout: "1200" diff --git a/official-templates/comfyui/docker-bake.hcl b/official-templates/comfyui/docker-bake.hcl index dde5e5d7..566592ef 100644 --- a/official-templates/comfyui/docker-bake.hcl +++ b/official-templates/comfyui/docker-bake.hcl @@ -48,8 +48,12 @@ variable "COMPATIBLE_BUILDS" { for combination in CUDA_TORCH_COMBINATIONS: [ { cuda_version = combination.cuda_version, - cuda_version_code = replace(combination.cuda_version, ".", "") - + cuda_version_code = replace(combination.cuda_version, ".", ""), + cuda_version_dash = replace(combination.cuda_version, ".", "-"), + torch_index_suffix = "cu${combination.cuda_version_code}", + torch_version = "${combination.torch_version}+${combination.torch_index_suffix}", + torchvision_version = "${combination.torchvision_version}+${combination.torch_index_suffix}", + torchaudio_version = "${combination.torchaudio_version}+${combination.torch_index_suffix}", }, ] ] @@ -103,14 +107,14 @@ target "comfyui-matrix" { RUNPODDIRECT_SHA = RUNPODDIRECT_SHA FILEBROWSER_VERSION = FILEBROWSER_VERSION FILEBROWSER_SHA256 = FILEBROWSER_SHA256 - TORCH_VERSION = TORCH_VERSION_5090 - TORCHVISION_VERSION = TORCHVISION_VERSION_5090 - TORCHAUDIO_VERSION = TORCHAUDIO_VERSION_5090 - CUDA_VERSION_DASH = "13-0" - TORCH_INDEX_SUFFIX = "cu130" + TORCH_VERSION = build.torch_version + TORCHVISION_VERSION = build.torchvision_version + TORCHAUDIO_VERSION = build.torchaudio_version + CUDA_VERSION_DASH = build.cuda_version_dash + TORCH_INDEX_SUFFIX = build.torch_index_suffix } tags = [ - "runpod/comfyui:${RELEASE_VERSION}${RELEASE_SUFFIX}-comfyui${build.comfyui_code}-cuda${build.cuda_code}" + "runpod/comfyui:${RELEASE_VERSION}${RELEASE_SUFFIX}-comfyui${COMFYUI_VERSION}-cuda${build.cuda_version}" ] } \ No newline at end of file From b847514766d7c5c0d228a748b70069439de2ad84 Mon Sep 17 00:00:00 2001 From: mchekm Date: Tue, 25 Aug 2026 11:33:33 +0300 Subject: [PATCH 03/33] cI: comfyui --- .github/workflows/release.yml | 13 +++++++++++++ official-templates/comfyui/scripts/start.sh | 2 +- 2 files changed, 14 insertions(+), 1 deletion(-) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 228bacc6..2bf2f483 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -94,6 +94,8 @@ jobs: - 'official-templates/base/**' - 'official-templates/shared/**' - 'container-template/**' + comfyui: + - 'official-templates/comfyui/**' base: needs: [version, changes] @@ -132,6 +134,17 @@ jobs: version: ${{ needs.version.outputs.version }} suffix: ${{ needs.version.outputs.suffix }} + comfyui: + needs: [version, changes] + if: >- + !cancelled() && needs.version.outputs.should-build == 'true' + && (github.event_name != 'pull_request' || needs.changes.outputs.comfyui == 'true') + uses: ./.github/workflows/comfyui.yml + secrets: inherit + with: + version: ${{ needs.version.outputs.version }} + suffix: ${{ needs.version.outputs.suffix }} + # Tag + GitHub Release ONLY on a releasable push to main, and ONLY after every # family built and smoke-tested successfully — so we never publish a release # that's missing an image family or built on a broken one. diff --git a/official-templates/comfyui/scripts/start.sh b/official-templates/comfyui/scripts/start.sh index 049f6f7b..17af85ed 100755 --- a/official-templates/comfyui/scripts/start.sh +++ b/official-templates/comfyui/scripts/start.sh @@ -292,7 +292,7 @@ if [ -s "$ARGS_FILE" ]; then fi echo "Starting ComfyUI with args: $FIXED_ARGS" -python main.py $FIXED_ARGS & +python main.py "$FIXED_ARGS" & COMFY_PID=$! # Distinguish a real ComfyUI crash from the pod being stopped/restarted/ From dfdf2eb42ba5df801d45ba819a14e56e8da9dfc7 Mon Sep 17 00:00:00 2001 From: mchekm Date: Tue, 25 Aug 2026 11:43:15 +0300 Subject: [PATCH 04/33] ci: comfyui changes --- .github/workflows/release.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 2bf2f483..4ca65f99 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -66,6 +66,7 @@ jobs: base: ${{ steps.f.outputs.base_any_changed }} nvidia: ${{ steps.f.outputs.nvidia_any_changed }} rocm: ${{ steps.f.outputs.rocm_any_changed }} + comfyui: ${{ steps.f.outputs.comfyui_any_changed }} steps: - name: Checkout uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 From a8726487ececbdc84eb096dfcd154caba424a6f9 Mon Sep 17 00:00:00 2001 From: mchekm Date: Tue, 25 Aug 2026 12:03:32 +0300 Subject: [PATCH 05/33] fix: shellcheck --- official-templates/comfyui/docker-bake.hcl | 6 +++--- official-templates/comfyui/scripts/start.sh | 13 +++++++------ 2 files changed, 10 insertions(+), 9 deletions(-) diff --git a/official-templates/comfyui/docker-bake.hcl b/official-templates/comfyui/docker-bake.hcl index 566592ef..ad7e2d34 100644 --- a/official-templates/comfyui/docker-bake.hcl +++ b/official-templates/comfyui/docker-bake.hcl @@ -62,14 +62,14 @@ variable "COMPATIBLE_BUILDS" { group "default" { targets = [ - for combination in CUDA_TORCH_COMBINATIONS: + for combination in COMPATIBLE_BUILDS: "cuda${combination.cuda_version_code}" ] } group "cuda128" { targets = [ - for combination in CUDA_TORCH_COMBINATIONS: + for combination in COMPATIBLE_BUILDS: "cuda${combination.cuda_version_code}" if combination.cuda_version == "12.8" ] @@ -78,7 +78,7 @@ group "cuda128" { group "cuda13" { targets = [ - for combination in CUDA_TORCH_COMBINATIONS: + for combination in COMPATIBLE_BUILDS: "cuda${combination.cuda_version_code}" if combination.cuda_version == "13.0" ] diff --git a/official-templates/comfyui/scripts/start.sh b/official-templates/comfyui/scripts/start.sh index 17af85ed..5852ef2e 100755 --- a/official-templates/comfyui/scripts/start.sh +++ b/official-templates/comfyui/scripts/start.sh @@ -6,7 +6,6 @@ BAKED_COMFYUI_DIR="/opt/comfyui-baked" BUNDLE_VERSION_FILE=".runpod-bundle-version" VENV_DIR="$COMFYUI_DIR/.venv-cu128" OLD_VENV_DIR="$COMFYUI_DIR/.venv" -FILEBROWSER_CONFIG="/root/.config/filebrowser/config.json" DB_FILE="/workspace/runpod-slim/filebrowser.db" PIP_CONSTRAINT_FILE="/opt/comfyui-runtime-constraints.txt" BAKED_NODES=("ComfyUI-Manager" "ComfyUI-KJNodes" "Civicomfy" "ComfyUI-RunpodDirect") @@ -55,10 +54,10 @@ export_env_vars() { cp "$PAM_ENV_FILE" "${PAM_ENV_FILE}.bak" 2>/dev/null || true # Clear files - > "$ENV_FILE" - > "$PAM_ENV_FILE" + : > "$ENV_FILE" + : > "$PAM_ENV_FILE" mkdir -p /root/.ssh - > "$SSH_ENV_DIR" + : > "$SSH_ENV_DIR" # Export to multiple locations for maximum compatibility printenv | grep -E '^RUNPOD_|^PATH=|^_=|^CUDA|^LD_LIBRARY_PATH|^PYTHONPATH|^PIP_CONSTRAINT=' | while read -r line; do @@ -223,16 +222,16 @@ if [ -d "$OLD_VENV_DIR" ] && [ ! -d "$VENV_DIR" ]; then mv "$OLD_VENV_DIR" "${OLD_VENV_DIR}.bak" cd "$COMFYUI_DIR" python3.12 -m venv --system-site-packages "$VENV_DIR" + # shellcheck disable=SC1091 source "$VENV_DIR/bin/activate" python -m ensurepip # Skip nodes baked into the image — their deps are in system site-packages - BAKED_NODES="ComfyUI-Manager ComfyUI-KJNodes Civicomfy ComfyUI-RunpodDirect" CURRENT=0 INSTALLED=0 for req in "$COMFYUI_DIR"/custom_nodes/*/requirements.txt; do if [ -f "$req" ]; then NODE_NAME=$(basename "$(dirname "$req")") - case " $BAKED_NODES " in + case " ${BAKED_NODES[*]} " in *" $NODE_NAME "*) continue ;; esac CURRENT=$((CURRENT + 1)) @@ -262,6 +261,7 @@ if [ ! -d "$COMFYUI_DIR" ] || [ ! -d "$VENV_DIR" ]; then if [ ! -d "$VENV_DIR" ]; then cd "$COMFYUI_DIR" python3.12 -m venv --system-site-packages "$VENV_DIR" + # shellcheck disable=SC1091 source "$VENV_DIR/bin/activate" # Ensure pip is available in the venv (needed for ComfyUI-Manager) @@ -272,6 +272,7 @@ if [ ! -d "$COMFYUI_DIR" ] || [ ! -d "$VENV_DIR" ]; then fi else # Just activate the existing venv + # shellcheck disable=SC1091 source "$VENV_DIR/bin/activate" echo "Using existing ComfyUI installation" fi From b1313f1c75639c1e26a2277f0d3239d1272fc1b9 Mon Sep 17 00:00:00 2001 From: mchekm Date: Tue, 25 Aug 2026 12:06:48 +0300 Subject: [PATCH 06/33] fix: comfyui --- official-templates/comfyui/docker-bake.hcl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/official-templates/comfyui/docker-bake.hcl b/official-templates/comfyui/docker-bake.hcl index ad7e2d34..80690ea2 100644 --- a/official-templates/comfyui/docker-bake.hcl +++ b/official-templates/comfyui/docker-bake.hcl @@ -50,7 +50,7 @@ variable "COMPATIBLE_BUILDS" { { cuda_version = combination.cuda_version, cuda_version_code = replace(combination.cuda_version, ".", ""), cuda_version_dash = replace(combination.cuda_version, ".", "-"), - torch_index_suffix = "cu${combination.cuda_version_code}", + torch_index_suffix = "cu" + replace(combination.cuda_version, ".", ""), torch_version = "${combination.torch_version}+${combination.torch_index_suffix}", torchvision_version = "${combination.torchvision_version}+${combination.torch_index_suffix}", torchaudio_version = "${combination.torchaudio_version}+${combination.torch_index_suffix}", From 466f1a4137a88d2fd67ac5dd2efbb1c9dcc85325 Mon Sep 17 00:00:00 2001 From: mchekm Date: Tue, 25 Aug 2026 12:09:42 +0300 Subject: [PATCH 07/33] fix --- official-templates/comfyui/docker-bake.hcl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/official-templates/comfyui/docker-bake.hcl b/official-templates/comfyui/docker-bake.hcl index 80690ea2..54a39e75 100644 --- a/official-templates/comfyui/docker-bake.hcl +++ b/official-templates/comfyui/docker-bake.hcl @@ -50,7 +50,7 @@ variable "COMPATIBLE_BUILDS" { { cuda_version = combination.cuda_version, cuda_version_code = replace(combination.cuda_version, ".", ""), cuda_version_dash = replace(combination.cuda_version, ".", "-"), - torch_index_suffix = "cu" + replace(combination.cuda_version, ".", ""), + torch_index_suffix = "cu${replace(combination.cuda_version, ".", "")}", torch_version = "${combination.torch_version}+${combination.torch_index_suffix}", torchvision_version = "${combination.torchvision_version}+${combination.torch_index_suffix}", torchaudio_version = "${combination.torchaudio_version}+${combination.torch_index_suffix}", From 8a66f19a69e84df35576c5de74b2c30d2de6bee4 Mon Sep 17 00:00:00 2001 From: mchekm Date: Tue, 25 Aug 2026 12:12:04 +0300 Subject: [PATCH 08/33] fix --- official-templates/comfyui/docker-bake.hcl | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/official-templates/comfyui/docker-bake.hcl b/official-templates/comfyui/docker-bake.hcl index 54a39e75..ca713f80 100644 --- a/official-templates/comfyui/docker-bake.hcl +++ b/official-templates/comfyui/docker-bake.hcl @@ -51,9 +51,9 @@ variable "COMPATIBLE_BUILDS" { cuda_version_code = replace(combination.cuda_version, ".", ""), cuda_version_dash = replace(combination.cuda_version, ".", "-"), torch_index_suffix = "cu${replace(combination.cuda_version, ".", "")}", - torch_version = "${combination.torch_version}+${combination.torch_index_suffix}", - torchvision_version = "${combination.torchvision_version}+${combination.torch_index_suffix}", - torchaudio_version = "${combination.torchaudio_version}+${combination.torch_index_suffix}", + torch_version = "${combination.torch_version}+cu${replace(combination.cuda_version, ".", "")}", + torchvision_version = "${combination.torchvision_version}+cu${replace(combination.cuda_version, ".", "")}", + torchaudio_version = "${combination.torchaudio_version}+cu${replace(combination.cuda_version, ".", "")}", }, ] ] From ac22789a66d647da97cf93f88dece314659bb3da Mon Sep 17 00:00:00 2001 From: mchekm Date: Tue, 25 Aug 2026 12:15:42 +0300 Subject: [PATCH 09/33] fix: shared --- .github/workflows/comfyui.yml | 1 + .github/workflows/release.yml | 1 + 2 files changed, 2 insertions(+) diff --git a/.github/workflows/comfyui.yml b/.github/workflows/comfyui.yml index 16e87b63..800d3ab9 100644 --- a/.github/workflows/comfyui.yml +++ b/.github/workflows/comfyui.yml @@ -52,6 +52,7 @@ jobs: with: source: . files: | + official-templates/shared/versions.hcl official-templates/comfyui/docker-bake.hcl load: true push: false diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 4ca65f99..92b6214c 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -97,6 +97,7 @@ jobs: - 'container-template/**' comfyui: - 'official-templates/comfyui/**' + - 'official-templates/shared/**' base: needs: [version, changes] From 26aeecc07d0ac8996c473eb9736fac5e7c8cd960 Mon Sep 17 00:00:00 2001 From: mchekm Date: Tue, 25 Aug 2026 12:18:32 +0300 Subject: [PATCH 10/33] fix: docker context --- official-templates/comfyui/docker-bake.hcl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/official-templates/comfyui/docker-bake.hcl b/official-templates/comfyui/docker-bake.hcl index ca713f80..5301ddac 100644 --- a/official-templates/comfyui/docker-bake.hcl +++ b/official-templates/comfyui/docker-bake.hcl @@ -86,7 +86,7 @@ group "cuda13" { # Common settings for all targets (defaults to regular CUDA 12.8 / cu128) target "common" { - context = "." + context = "official-templates/comfyui" dockerfile = "Dockerfile" platforms = ["linux/amd64"] } From 2ac3e9c383f79c803a5fdf87574f17a0dd935eef Mon Sep 17 00:00:00 2001 From: mchekm Date: Tue, 25 Aug 2026 14:49:58 +0300 Subject: [PATCH 11/33] feat: ComfyUI tests --- .github/actions/smoke-test/action.yml | 76 ++++ .github/scripts/generate_test_manifest.py | 58 ++- .github/workflows/comfyui.yml | 14 +- .github/workflows/release.yml | 4 +- tests/README.md | 87 +++-- tests/comfyui/images.example.yaml | 18 + tests/comfyui/models.json | 8 + .../workflows/gsl_starter_1_1.api.json | 47 +++ tests/runpod_smoke/checks.py | 331 ++++++++++++++++-- tests/runpod_smoke/comfyui.py | 295 ++++++++++++++++ tests/runpod_smoke/config.py | 74 +++- tests/runpod_smoke/instances.py | 43 ++- tests/runpod_smoke/log.py | 9 + tests/runpod_smoke/pod.py | 45 ++- tests/runpod_smoke/runner.py | 203 +++++++++-- tests/test_images.py | 113 ++++-- 16 files changed, 1283 insertions(+), 142 deletions(-) create mode 100644 tests/comfyui/images.example.yaml create mode 100644 tests/comfyui/models.json create mode 100644 tests/comfyui/workflows/gsl_starter_1_1.api.json create mode 100644 tests/runpod_smoke/comfyui.py diff --git a/.github/actions/smoke-test/action.yml b/.github/actions/smoke-test/action.yml index 8d54ec9b..a6f9aacb 100644 --- a/.github/actions/smoke-test/action.yml +++ b/.github/actions/smoke-test/action.yml @@ -55,6 +55,43 @@ inputs: its own entrypoint and would fail the probe. required: false default: "false" + test-ports: + description: | + Newline-separated HTTP ports to expose and probe with the generic + proxy-first reachability check. Accepts plain lines or `- 8080`. + required: false + default: "" + test-comfyui: + description: | + Emit `test_comfyui: true` for each generated group. Exposes :8188 as + 8188/http, verifies public reachability first, and re-probes + /system_stats after dwell. + required: false + default: "false" + test-comfyui-functional: + description: | + Emit `test_comfyui_functional: true` for the end-to-end ComfyUI + generation check (model provisioning, workflow execution, PNG validation). + required: false + default: "false" + save-comfyui-images: + description: "Upload PNGs generated by the ComfyUI functional check." + required: false + default: "false" + comfyui-images-artifact-name: + description: "Artifact name used with save-comfyui-images." + required: false + default: "comfyui-generated-images" + comfyui-images-retention-days: + description: "Retention period for the generated ComfyUI images artifact." + required: false + default: "14" + check-all-gpu: + description: | + Test every GPU matching the vendor/vRAM filters independently instead + of selecting by hourly budget. This can be expensive. + required: false + default: "false" exclude-instances: description: | Newline-separated list of fnmatch-style GPU-display-name patterns to @@ -241,6 +278,10 @@ runs: MIN_VRAM_GB: ${{ inputs.min-vram-gb }} MANUFACTURER: ${{ inputs.manufacturer }} TEST_JUPYTER: ${{ inputs.test-jupyter }} + TEST_PORTS: ${{ inputs.test-ports }} + TEST_COMFYUI: ${{ inputs.test-comfyui }} + TEST_COMFYUI_FUNCTIONAL: ${{ inputs.test-comfyui-functional }} + CHECK_ALL_GPU: ${{ inputs.check-all-gpu }} EXCLUDE_INSTANCES: ${{ inputs.exclude-instances }} MIN_CUDA_VERSION: ${{ inputs.min-cuda-version }} run: | @@ -252,6 +293,27 @@ runs: if [[ "${TEST_JUPYTER,,}" == "true" ]]; then EXTRA_ARGS+=(--test-jupyter) fi + if [[ "${TEST_COMFYUI,,}" == "true" ]]; then + EXTRA_ARGS+=(--test-comfyui) + fi + if [[ "${TEST_COMFYUI_FUNCTIONAL,,}" == "true" ]]; then + EXTRA_ARGS+=(--test-comfyui-functional) + fi + if [[ "${CHECK_ALL_GPU,,}" == "true" ]]; then + EXTRA_ARGS+=(--check-all-gpu) + fi + # Split TEST_PORTS on newlines. Accept both "8080" and "- 8080" + # so workflow input can match the generated manifest syntax. + while IFS= read -r line; do + line="${line#"${line%%[![:space:]]*}"}" + line="${line%"${line##*[![:space:]]}"}" + [[ -z "${line}" || "${line}" == \#* ]] && continue + line="${line#-}" + line="${line#"${line%%[![:space:]]*}"}" + line="${line%"${line##*[![:space:]]}"}" + [[ -z "${line}" ]] && continue + EXTRA_ARGS+=(--test-port "${line}") + done <<< "${TEST_PORTS}" # Only pass --min-cuda-version when the input is non-empty; the # generator treats empty as "no floor" and we want to keep the # argv free of stray empty strings so argparse doesn't choke. @@ -291,6 +353,20 @@ runs: # Forwarded to config.CREATE_TIMEOUT — bumps the SSH-readiness # deadline for slow pulls (mainly multi-GB ROCm base images). CREATE_TIMEOUT: ${{ inputs.create-timeout }} + SAVE_COMFYUI_IMAGES: ${{ inputs.save-comfyui-images }} run: | set -uo pipefail + if [[ "${SAVE_COMFYUI_IMAGES,,}" == "true" ]]; then + export COMFYUI_SAVE_DIR="${RUNNER_TEMP}/comfy-out" + mkdir -p "${COMFYUI_SAVE_DIR}" + fi python3 "${GITHUB_WORKSPACE}/tests/test_images.py" "${MANIFEST_PATH}" + + - name: Upload generated ComfyUI images + if: ${{ always() && inputs.save-comfyui-images == 'true' }} + uses: actions/upload-artifact@v4 + with: + name: ${{ inputs.comfyui-images-artifact-name }} + path: ${{ runner.temp }}/comfy-out + retention-days: ${{ inputs.comfyui-images-retention-days }} + if-no-files-found: ignore diff --git a/.github/scripts/generate_test_manifest.py b/.github/scripts/generate_test_manifest.py index d1ae8724..80ab7a03 100755 --- a/.github/scripts/generate_test_manifest.py +++ b/.github/scripts/generate_test_manifest.py @@ -78,12 +78,19 @@ def render_yaml(groups: dict) -> str: "manufacturer", "min_cuda_version", "test_jupyter", + "check_all_gpu", + "test_comfyui", + "test_comfyui_functional", ): if key in body: val = body[key] if isinstance(val, bool): val = "true" if val else "false" lines.append(f" {key}: {val}") + if body.get("test_ports"): + lines.append(" test_ports:") + for port in body["test_ports"]: + lines.append(f" - {port}") # exclude_instances is a list, emitted at the bottom of the group so # it's visually grouped with other "filter" options. Patterns are # double-quoted to keep glob-leading characters ('*', '?') safe from @@ -103,6 +110,10 @@ def build_groups( min_vram_gb: int, manufacturer: str, test_jupyter: bool = False, + test_ports: list[int] | None = None, + test_comfyui: bool = False, + test_comfyui_functional: bool = False, + check_all_gpu: bool = False, exclude_instances: list[str] | None = None, min_cuda_version: str | None = None, ) -> dict: @@ -131,10 +142,24 @@ def build_groups( CUDA 13.0 and refuses to run on hosts with a 12.x driver. """ exclude_instances = list(exclude_instances or []) + test_ports = list(test_ports or []) - def _decorate(body: dict) -> dict: + def _decorate(body: dict, *, gpu_group: bool) -> dict: + if gpu_group: + if check_all_gpu: + body["check_all_gpu"] = True + else: + body["max_price_per_hour"] = budget + body["min_vram_gb"] = min_vram_gb + body["manufacturer"] = manufacturer if test_jupyter: body["test_jupyter"] = True + if test_ports: + body["test_ports"] = list(test_ports) + if test_comfyui: + body["test_comfyui"] = True + if test_comfyui_functional: + body["test_comfyui_functional"] = True if exclude_instances: body["exclude_instances"] = list(exclude_instances) if min_cuda_version: @@ -152,14 +177,9 @@ def _decorate(body: dict) -> dict: gpu = [r for r in refs if is_gpu_ref(r)] groups: dict = {} if cpu: - groups["base_cpu"] = _decorate({"images": cpu}) + groups["base_cpu"] = _decorate({"images": cpu}, gpu_group=False) if gpu: - groups["base_gpu"] = _decorate({ - "images": gpu, - "max_price_per_hour": budget, - "min_vram_gb": min_vram_gb, - "manufacturer": manufacturer, - }) + groups["base_gpu"] = _decorate({"images": gpu}, gpu_group=True) return groups if profile == "gpu": @@ -168,12 +188,7 @@ def _decorate(body: dict) -> dict: # is picked from the IMAGE REF by runpod_smoke.checks, so the group # name 'base_gpu' is purely conventional here. return { - "base_gpu": _decorate({ - "images": refs, - "max_price_per_hour": budget, - "min_vram_gb": min_vram_gb, - "manufacturer": manufacturer, - }) + "base_gpu": _decorate({"images": refs}, gpu_group=True) } raise ValueError(f"unknown profile: {profile!r}") @@ -221,6 +236,17 @@ def main() -> int: "Off by default — enable per CI step." ), ) + ap.add_argument( + "--test-port", + action="append", + default=[], + type=int, + metavar="PORT", + help="HTTP port to expose and probe. Repeat for multiple ports.", + ) + ap.add_argument("--test-comfyui", action="store_true") + ap.add_argument("--test-comfyui-functional", action="store_true") + ap.add_argument("--check-all-gpu", action="store_true") ap.add_argument( "--exclude-instance", action="append", @@ -265,6 +291,10 @@ def main() -> int: min_vram_gb=args.min_vram_gb, manufacturer=args.manufacturer, test_jupyter=args.test_jupyter, + test_ports=args.test_port, + test_comfyui=args.test_comfyui, + test_comfyui_functional=args.test_comfyui_functional, + check_all_gpu=args.check_all_gpu, exclude_instances=args.exclude_instance, min_cuda_version=(args.min_cuda_version or None), ) diff --git a/.github/workflows/comfyui.yml b/.github/workflows/comfyui.yml index 800d3ab9..1c8fc631 100644 --- a/.github/workflows/comfyui.yml +++ b/.github/workflows/comfyui.yml @@ -1,4 +1,4 @@ -name: ROCm Images Build +name: ComfyUI Images Build # Reusable building block invoked by the release orchestrator (release.yml), # which computes the version ONCE and passes it in. Not triggered on its own — @@ -15,6 +15,11 @@ on: required: false default: "" type: string + run-functional-tests: + description: "Run ComfyUI model-generation validation and upload its PNG." + required: false + default: false + type: boolean permissions: contents: read @@ -145,3 +150,10 @@ jobs: budget-usd-per-hour: "3.0" min-vram-gb: "80" create-timeout: "1200" + test-jupyter: "true" + test-comfyui: "true" + test-comfyui-functional: ${{ inputs.run-functional-tests }} + save-comfyui-images: ${{ inputs.run-functional-tests }} + comfyui-images-artifact-name: comfyui-generated-images-${{ matrix.comfyui }} + test-ports: | + - 8080 diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 92b6214c..090c08d8 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -146,18 +146,20 @@ jobs: with: version: ${{ needs.version.outputs.version }} suffix: ${{ needs.version.outputs.suffix }} + run-functional-tests: true # Tag + GitHub Release ONLY on a releasable push to main, and ONLY after every # family built and smoke-tested successfully — so we never publish a release # that's missing an image family or built on a broken one. release: - needs: [version, base, nvidia, rocm] + needs: [version, base, nvidia, rocm, comfyui] if: >- github.event_name == 'push' && needs.version.outputs.should-release == 'true' && needs.base.result == 'success' && needs.nvidia.result == 'success' && needs.rocm.result == 'success' + && needs.comfyui.result == 'success' runs-on: ubuntu-latest permissions: contents: write diff --git a/tests/README.md b/tests/README.md index 4ca2bdb5..f2d5c683 100644 --- a/tests/README.md +++ b/tests/README.md @@ -1,9 +1,8 @@ # Smoke tests for RunPod container images Spins up each image on a real RunPod pod, waits for it to stay healthy -for `DWELL_SEC` seconds, runs an image-appropriate functional check -(CUDA / `nvidia-smi` / `torch.cuda` / optional JupyterLab), then -terminates the pod. Designed to catch the failure modes that **only +for `DWELL_SEC` seconds, runs image-appropriate CUDA, HTTP, and optional +ComfyUI functional checks, then terminates the pod. Designed to catch the failure modes that **only appear on a real GPU host** and that local `docker run` would miss: driver-version mismatches, broken NCCL/NVRTC, missing CUDA libs, `start.sh` regressions, JupyterLab proxy misconfiguration, etc. @@ -18,6 +17,7 @@ The code is split into a small package next to the entry point: tests/ ├── README.md ← you are here ├── test_images.py ← entry point: main() + summary + CLI +├── comfyui/ ← ComfyUI functional-test models and workflow └── runpod_smoke/ ├── config.py ← env vars, sentinels, shared mutable state ├── log.py ← thread-tagged logging @@ -25,7 +25,8 @@ tests/ ├── runpodctl.py ← subprocess wrappers around the `runpodctl` binary ├── instances.py ← GPU catalog, budget resolution, exclude filter, CUDA detection ├── pod.py ← pod create/lifecycle/signals, registry auth - ├── checks.py ← SSH probe, CUDA functional check, Jupyter probes, log dumper + ├── checks.py ← SSH/proxy checks, CUDA functional check, REST API v2 log diagnostics + ├── comfyui.py ← ComfyUI proxy, model, workflow, and PNG checks └── runner.py ← test_pair / test_image (per-image orchestration) ``` @@ -112,17 +113,23 @@ runs this sequence and reports the outcome as soon as one step fails. | # | Step | Failure → | |---|------|---| | 1 | `runpodctl pod create` (with `--gpu-id`, `--container-disk-in-gb`, `--ports`, registry auth, optional `--min-cuda-version`). Transient `5xx` / `Something went wrong` errors are retried silently up to `CREATE_RETRIES` with linear backoff. | `UNAVAILABLE` (no capacity for this instance type — try next) / `CREATE_FAIL` (bad image tag, registry auth, malformed request — any non-capacity, non-transient orchestrator error after retries are exhausted) | -| 2 | Poll `runpodctl pod get` until `ssh.ip` / `ssh.port` are assigned and one-shot `ssh root@ip -p port 'echo ready'` succeeds (the real readiness signal — `desiredStatus` is always `RUNNING` after create) | `STUCK` if no SSH endpoint within `CREATE_TIMEOUT` (almost always a bad host in the scheduler pool — try another instance type) | +| 2 | Poll `runpodctl pod get` and REST API v2 status until `ssh.ip` / `ssh.port` are assigned and one-shot `ssh root@ip -p port 'echo ready'` succeeds (SSH is the readiness signal; v2 surfaces terminal `ERROR` states the CLI misses) | `FAIL` on a terminal status; `STUCK` if no SSH endpoint within `CREATE_TIMEOUT` | | 3 | **CUDA functional check** over SSH — see [Functional check](#functional-check). Image-driven: pytorch ref → `torch.cuda` + matmul; cuda/rocm ref → `nvidia-smi` + `nvcc`; neither → skip | `FAIL` (image is broken — stop iterating; another GPU won't help) | -| 4 | **JupyterLab in-pod check** (only when `test_jupyter: true`) — see [Jupyter check](#jupyter-check-opt-in). SSH in, wait for `:8888` to bind, `jupyter server list`, `curl /api/status` with token | `FAIL` (`start.sh` didn't bring up Jupyter — usually wrong python interpreter) | -| 5 | **JupyterLab public-proxy check** (only when `test_jupyter: true`) — `GET https://-8888.proxy.runpod.net/api/status` from the test machine | `FAIL` (port not exposed as `8888/http`, or proxy never registered) | -| 6 | Sleep `DWELL_SEC`, re-probe SSH (catches "boots fine then crashes after 30s") | `FAIL` if SSH stops responding | -| 7 | `dump_pod_logs` — pull `uname`, `syslog`, `dmesg`, `/var/log/*.log`, `nvidia-smi` via SSH for the run log | _(diagnostic only)_ | -| 8 | `runpodctl pod delete` (always — even on Ctrl-C / exception via `atexit` + signal handlers) | _(diagnostic only)_ | +| 4 | **JupyterLab proxy-first check** (only when `test_jupyter: true`) — checks the public proxy; SSH probes `/api/status` only to diagnose a proxy failure | `FAIL` (Jupyter did not start, or is not exposed as `8888/http`) | +| 5 | **Generic proxy-first port checks** (optional `test_ports`) — each service must return HTTP 200 through `https://-.proxy.runpod.net/`; SSH diagnoses failures | `FAIL` (service unavailable or incorrectly exposed) | +| 6 | **ComfyUI proxy-first reachability** (only when `test_comfyui: true`) — public `:8188` first; SSH diagnoses a failed proxy check | `FAIL` (ComfyUI unavailable or incorrectly exposed) | +| 7 | **ComfyUI functional generation** (only when `test_comfyui_functional: true`) — provision models via RunpodDirect, execute the workflow, then validate a real PNG through the public proxy | `FAIL` (model, workflow, node, GPU, or output failure) | +| 8 | **REST API v2 container-log scan** — backfill stdout and scan it for configured error/crash markers | `FAIL` on a matching marker or an unverified empty API response | +| 9 | Sleep `DWELL_SEC`, re-probe SSH (catches "boots fine then crashes after 30s") | `FAIL` if SSH stops responding | +| 10 | Re-probe ComfyUI `/system_stats` after dwell, then re-scan REST API v2 logs | `FAIL` if ComfyUI died during dwell, or logs contain a new error marker | +| 11 | `dump_pod_logs` — API container logs, API system-log error markers, and GPU SMI over SSH | _(diagnostic only)_ | +| 12 | `runpodctl pod delete` (always — even on Ctrl-C / exception via `atexit` + signal handlers) | _(diagnostic only)_ | `test_image()` then iterates over the next instance candidate when the result was `UNAVAILABLE` or `STUCK`, and short-circuits on `PASS`, -`FAIL`, or `CREATE_FAIL`. +`FAIL`, or `CREATE_FAIL`. With `check_all_gpu: true`, each resolved GPU is +instead run as an independent job, so the summary shows compatibility across +the full selected GPU set. ## Outcomes @@ -195,6 +202,10 @@ ON_SKIP=warn ./test_images.py images.yaml # Fully lenient — script exits 0 on SKIP with no annotation. ON_SKIP=pass ./test_images.py images.yaml + +# Run the checked-in ComfyUI end-to-end example. This downloads the configured +# model and can take several minutes. +python3 ./tests/test_images.py ./tests/comfyui/images.example.yaml comfyui ``` If a pod gets stuck (rare), `Ctrl-C` cleans up — `SIGINT`/`SIGTERM` are @@ -229,7 +240,10 @@ groupname: exclude_instances: # subtract fnmatch patterns from candidates - "*Blackwell*" min_cuda_version: "13.0" # 'X.Y' string for --min-cuda-version (fallback only) + check_all_gpu: true # test every matching GPU independently test_jupyter: true # opt-in JupyterLab in-pod + proxy check + test_ports: # optional generic HTTP services + - 8080 ``` Field reference: @@ -242,8 +256,12 @@ Field reference: | `min_vram_gb` | Extra filter for budget mode (default 0). | | `manufacturer` | `Nvidia` or `AMD` filter for budget mode (default: any). | | `exclude_instances` | fnmatch-style patterns (case-insensitive) subtracted from the candidate list AFTER `instances:` or budget selection. Useful for blocking known-bad host pairings without rewriting the whole list — e.g. `"*Blackwell*"` skips every Blackwell GPU (sm\_100 / sm\_120 are not in the kernel set of PyTorch ≤ 2.6 wheels). | -| `min_cuda_version` | `X.Y` string passed to `runpodctl pod create --min-cuda-version`. Only used as a **fallback** when the image tag itself doesn't encode a CUDA version (e.g. NGC `nvidia-pytorch:25.11`). Image tags like `cu1281` / `cuda1281` always win. | +| `min_cuda_version` | `X.Y` string passed to `runpodctl pod create --min-cuda-version`. Only used as a **fallback** when the image tag itself doesn't encode a CUDA version (e.g. NGC `nvidia-pytorch:25.11`). Image tags like `cu1281` / `cuda1281` and `cuda13.0` always win. | +| `check_all_gpu` | `true` / `false` — use every catalog GPU matching `min_vram_gb` and `manufacturer`, with one independent result row per `(image, GPU)`. Mutually exclusive with budget selection in generated manifests and potentially expensive. Default: `false`. | | `test_jupyter` | `true` / `false` — when true, the pod is created with `JUPYTER_PASSWORD=admin` in env and HTTP port 8888 exposed, then the script SSHes in and verifies JupyterLab is actually listening. Use for groups whose images use `container-template/start.sh` (`runpod/base`, `runpod/pytorch`, `runpod/autoresearch`, `rocm`). Skip for NGC `nvidia-pytorch` (different entrypoint). Default: `false`. | +| `test_ports` | Optional list of HTTP ports. Each is exposed as `/http` and must return HTTP 200 through the RunPod public proxy. On a proxy failure, the test probes `127.0.0.1:` over SSH to distinguish a service startup failure from an exposure/configuration error. | +| `test_comfyui` | `true` / `false` — exposes `8188/http` and runs a labelled proxy-first ComfyUI reachability check. After dwell it verifies `/system_stats` again because the container can survive a ComfyUI crash. Default: `false`. | +| `test_comfyui_functional` | `true` / `false` — implies `test_comfyui`; downloads/verifies the configured model through ComfyUI-RunpodDirect, POSTs the workflow, waits for completion, then validates a non-empty PNG from `/view`. The ComfyUI workflow enables it for both PR and release runs. | The `base_cpu` group is special: `runpodctl` 2.3.0 does not let us pick a specific CPU flavor (`--gpu-id` is rejected for `--compute-type CPU`), @@ -251,6 +269,10 @@ so the manifest needs ONLY an `images:` list for that group — no `instances:` / `max_price_per_hour:` / `min_vram_gb:`. RunPod picks a CPU flavor for us. +The functional workflow and model manifest live under `tests/comfyui/`. +Set `COMFYUI_SAVE_DIR` to retain the validated PNG locally; the composite +action uploads it when `save-comfyui-images: "true"`. + ## Example manifest (the real one used in this repo) @@ -347,10 +369,24 @@ pytorch: | `CREATE_RETRIES` | `3` | Retry pod-create up to N times on transient RunPod 5xx errors (`Something went wrong`, 502/503). Capacity shortages are NOT retried. | | `CREATE_RETRY_BACKOFF` | `10` | Seconds between retries (linear backoff). | | `STALL_HINT_AFTER` | `180` | Seconds without an SSH endpoint before the script prints a hint about slow pulls / possible Docker Hub rate limit. | -| `SSH_LOG_FETCH` | `1` | `1`/`0` — fetch container logs via direct SSH at PASS/FAIL. | +| `LOG_ERROR_SCAN` | `1` | `1`/`0` — scan REST API v2 container logs for error markers after the functional checks. | +| `LOG_ERROR_PATTERN` | `\berr(or)?s?\b\|\bcrash(ed\|es\|ing)?\b` | Case-insensitive regex used by the container-log scan. | +| `LOG_API_TAIL` | `1000` | Number of historical lines to fetch from the REST API v2 log stream. | +| `SYS_LOG_ERROR_PATTERN` | error/failure/crash regex | Case-insensitive regex for host-side REST API system-log diagnostics during a failed boot. | +| `SSH_LOG_FETCH` | `1` | `1`/`0` — fetch only the GPU SMI diagnostic over SSH. Container logs use REST API v2. | | `RUNPOD_SSH_KEY` | _(empty)_ | Path to private key matching the `PUBLIC_KEY` `runpodctl` injects into pods. Auto-discovered from common locations if not set. | | `JUPYTER_WAIT_TIMEOUT` | `30` | Seconds the in-pod Jupyter probe waits for `:8888` to bind. | | `JUPYTER_PROXY_TIMEOUT` | `60` | Seconds the proxy probe retries while RunPod's ingress registers the new pod. | +| `PORT_WAIT_TIMEOUT` | `300` | Seconds the SSH diagnostic probe waits for a `test_ports` service to bind and return HTTP 200. | +| `PORT_PROXY_TIMEOUT` | `300` | Seconds the public-proxy check retries each `test_ports` service before failing. | +| `COMFYUI_PORT` | `8188` | HTTP port exposed for the ComfyUI public-proxy checks. | +| `COMFYUI_WORKFLOW` | `tests/comfyui/workflows/gsl_starter_1_1.api.json` | API workflow submitted by the functional test. | +| `COMFYUI_MODELS_MANIFEST` | `tests/comfyui/models.json` | Models provisioned through ComfyUI-RunpodDirect before the workflow runs. | +| `COMFYUI_WAIT_TIMEOUT` | `300` | Maximum wait for ComfyUI `/system_stats`. | +| `COMFYUI_ROUTES_TIMEOUT` | `120` | Maximum wait for ComfyUI-RunpodDirect routes. | +| `COMFYUI_DOWNLOAD_TIMEOUT` | `1800` | Maximum wait for each functional-test model download. | +| `COMFYUI_GEN_TIMEOUT` | `600` | Maximum wait for workflow execution and its image output. | +| `COMFYUI_SAVE_DIR` | _(empty)_ | Directory where validated ComfyUI PNGs are retained for artifact upload. | ## Functional check @@ -379,19 +415,10 @@ groups don't silently skip the check: ## Jupyter check (opt-in via manifest `test_jupyter: true`) -Two stages, both must pass: - -1. **In-pod.** SSH into the pod and `curl http://127.0.0.1:8888/api/status` - with our token. Catches silent `start.sh` failures (e.g. `python3 -m - jupyter` not finding the module on Ubuntu 22.04 — the kind of bug - that prints `Jupyter Lab started` in the container log while no - server is actually running). -2. **Public proxy.** From the test machine, `GET - https://-8888.proxy.runpod.net/api/status` with the token. - Catches port-type misconfigurations (`8888/tcp` instead of - `8888/http` — the proxy never wires up non-http ports) and DNS / - proxy registration issues that would prevent real users from - reaching Jupyter from the RunPod console. +The public proxy is checked first. If it returns HTTP 200, the service is +both running and exposed as `8888/http`, so SSH is skipped. On failure the +test SSHes in and probes `/api/status` to distinguish a Jupyter startup +problem from an HTTP exposure/proxy problem. ## Running in CI @@ -409,10 +436,12 @@ wraps everything in this script needs for a clean CI run: 4. Generates a manifest from the `image-refs` JSON array using `.github/scripts/generate_test_manifest.py`, applying the `profile`, `budget-usd-per-hour`, `min-vram-gb`, `manufacturer`, - `test-jupyter`, and `exclude-instances` inputs. + `test-jupyter`, `test-ports`, `test-comfyui`, + `test-comfyui-functional`, `check-all-gpu`, and + `exclude-instances` inputs. 5. Invokes `python3 tests/test_images.py ` with - `MAX_PARALLEL=` and `continue-on-error: true` so a - single broken image doesn't take the whole pipeline down. + `MAX_PARALLEL=`. A failed image makes the smoke-test + action fail, which prevents a release from being created. Typical caller (from a per-image-family build workflow): diff --git a/tests/comfyui/images.example.yaml b/tests/comfyui/images.example.yaml new file mode 100644 index 00000000..4cfe8b30 --- /dev/null +++ b/tests/comfyui/images.example.yaml @@ -0,0 +1,18 @@ +# Example manifest for local ComfyUI smoke tests. +# +# From the repository root (with runpodctl authenticated and an SSH key +# registered on the RunPod account): +# +# python3 tests/test_images.py tests/comfyui/images.example.yaml comfyui +# +# `test_comfyui` checks public :8188 reachability. The functional flag also +# provisions the models in models.json, runs the workflow, and validates the +# PNG returned through the public proxy. It implies `test_comfyui`. +comfyui: + images: + - runpod/comfyui:1.1.1-comfyui0.30.0-cuda13.0 + instances: + - "RTX A5000" + - "RTX 4090" + - "RTX A4000" + test_comfyui_functional: true diff --git a/tests/comfyui/models.json b/tests/comfyui/models.json new file mode 100644 index 00000000..c4100b1b --- /dev/null +++ b/tests/comfyui/models.json @@ -0,0 +1,8 @@ +[ + { + "filename": "DreamShaper_8_pruned.safetensors", + "directory": "checkpoints", + "url": "https://huggingface.co/Lykon/DreamShaper/resolve/main/DreamShaper_8_pruned.safetensors", + "sha256": "879db523c30d3b9017143d56705015e15a2cb5628762c11d086fed9538abd7fd" + } +] diff --git a/tests/comfyui/workflows/gsl_starter_1_1.api.json b/tests/comfyui/workflows/gsl_starter_1_1.api.json new file mode 100644 index 00000000..2a7452a8 --- /dev/null +++ b/tests/comfyui/workflows/gsl_starter_1_1.api.json @@ -0,0 +1,47 @@ +{ + "3": { + "class_type": "KSampler", + "inputs": { + "seed": 650101271515995, + "steps": 20, + "cfg": 8, + "sampler_name": "euler", + "scheduler": "normal", + "denoise": 1, + "model": ["4", 0], + "positive": ["6", 0], + "negative": ["7", 0], + "latent_image": ["5", 0] + } + }, + "4": { + "class_type": "CheckpointLoaderSimple", + "inputs": {"ckpt_name": "DreamShaper_8_pruned.safetensors"} + }, + "5": { + "class_type": "EmptyLatentImage", + "inputs": {"width": 512, "height": 512, "batch_size": 1} + }, + "6": { + "class_type": "CLIPTextEncode", + "inputs": { + "text": "cinematic oil painting of a glowing planet in space, detailed, high quality", + "clip": ["4", 1] + } + }, + "7": { + "class_type": "CLIPTextEncode", + "inputs": { + "text": "low quality, blurry, deformed, watermark, text", + "clip": ["4", 1] + } + }, + "8": { + "class_type": "VAEDecode", + "inputs": {"samples": ["3", 0], "vae": ["4", 2]} + }, + "9": { + "class_type": "SaveImage", + "inputs": {"filename_prefix": "smoke", "images": ["8", 0]} + } +} diff --git a/tests/runpod_smoke/checks.py b/tests/runpod_smoke/checks.py index 504e4a84..cf334aab 100644 --- a/tests/runpod_smoke/checks.py +++ b/tests/runpod_smoke/checks.py @@ -4,7 +4,7 @@ * cuda_check_command / run_cuda_check — torch.cuda or nvidia-smi assertion * jupyter_check_command / run_jupyter_check — in-pod Jupyter probe over SSH * run_jupyter_proxy_check — public proxy probe from the test machine - * fetch_logs_via_ssh / dump_pod_logs — pull diagnostic info before terminating + * REST API v2 log helpers / dump_pod_logs — diagnose failures before termination Selection of the CUDA check is driven by the IMAGE REF, not the manifest group name: new groups added in the future won't silently skip the check. @@ -12,13 +12,15 @@ from __future__ import annotations +import json import os import re import subprocess +import threading import time import urllib.error import urllib.request -from typing import Optional +from typing import Callable, Optional from . import config from .log import log @@ -322,6 +324,147 @@ def run_jupyter_check(host: str, port: int) -> tuple[bool, str]: return (r.returncode == 0), combined +# --------------------------------------------------------------------------- +# Generic per-port checks (test_ports manifest field) +# --------------------------------------------------------------------------- + +def port_check_command(test_port: int, wait_timeout: int) -> str: + """Wait for localhost HTTP readiness; 4xx is a valid app response.""" + return ( + "set -e; " + f"echo 'Probing 127.0.0.1:{test_port} (timeout {wait_timeout}s)...'; " + "CODE=pending; " + f"for i in $(seq 1 {wait_timeout}); do " + f" if (echo > /dev/tcp/127.0.0.1/{test_port}) 2>/dev/null; then " + f" CODE=$(curl -sS --max-time 5 -o /dev/null " + f" -w '%{{http_code}}' 'http://127.0.0.1:{test_port}/' " + " || echo 'curl_failed'); " + " case \"$CODE\" in " + f" [1234]*) echo \"port {test_port} responsive after $i" + "s: http=$CODE\"; break ;; " + " *) ;; " + " esac; " + " fi; " + " if [ $((i % 30)) -eq 0 ]; then " + f" echo \" ...still probing :{test_port} at ${{i}}s/{wait_timeout}s " + "(last code=$CODE)\"; " + " fi; " + " sleep 1; " + "done; " + "case \"$CODE\" in " + f" [1234]*) echo 'port {test_port} OK' ;; " + f" pending) echo 'FAIL: nothing ever listened on 127.0.0.1:{test_port} " + f"within {wait_timeout}s'; exit 1 ;; " + f" *) echo \"FAIL: port {test_port} never returned HTTP <500 " + f"within {wait_timeout}s (last code: $CODE)\"; exit 1 ;; " + "esac" + ) + + +def run_port_check( + host: str, + port: int, + test_port: int, + on_line: Optional[Callable[[str], None]] = None, +) -> tuple[bool, str]: + """Run the in-pod fallback probe with live progress and a hard timeout.""" + ssh_cmd = [ + *_ssh_command_prefix(host, port), + port_check_command(test_port, config.PORT_WAIT_TIMEOUT), + ] + outer_timeout = config.PORT_WAIT_TIMEOUT + 60 + try: + proc = subprocess.Popen( + ssh_cmd, + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, + text=True, + bufsize=1, + ) + except FileNotFoundError: + return False, _SSH_BINARY_NOT_FOUND + assert proc.stdout is not None + + timed_out = [False] + + def kill_on_timeout() -> None: + timed_out[0] = True + try: + proc.kill() + except ProcessLookupError: + pass + + watchdog = threading.Timer(outer_timeout, kill_on_timeout) + watchdog.daemon = True + watchdog.start() + last_line = "" + try: + for raw in iter(proc.stdout.readline, ""): + line = raw.rstrip("\n") + if line: + if on_line: + on_line(line) + last_line = line + proc.wait(timeout=10) + except subprocess.TimeoutExpired: + proc.kill() + except Exception as exc: # noqa: BLE001 + try: + proc.kill() + except ProcessLookupError: + pass + return False, f"port {test_port} check errored: {exc}" + finally: + watchdog.cancel() + + if timed_out[0]: + return False, ( + f"port {test_port} check wall-clock timeout after {outer_timeout}s" + ) + return proc.returncode == 0, last_line + + +def _proxy_status_ok(code: int) -> bool: + """A service is healthy through the public proxy only on HTTP 200.""" + return code == 200 + + +def run_port_proxy_check(pod_id: str, test_port: int) -> tuple[bool, str]: + """Probe a generic HTTP service through RunPod's public proxy.""" + url = f"https://{pod_id}-{test_port}.proxy.runpod.net/" + deadline = time.monotonic() + config.PORT_PROXY_TIMEOUT + lines = [f"GET {url}"] + last_error = "" + attempt = 0 + while time.monotonic() < deadline: + attempt += 1 + try: + req = urllib.request.Request( + url, headers={"User-Agent": "runpod-smoke-test/1.0"} + ) + with urllib.request.urlopen(req, timeout=10) as response: + code = response.status + body = response.read(256).decode("utf-8", errors="replace") + lines.append( + f"attempt #{attempt}: HTTP {code} body={body[:160]!r}" + ) + if _proxy_status_ok(code): + return True, "\n".join(lines) + last_error = f"HTTP {code}" + except urllib.error.HTTPError as exc: + last_error = f"HTTP {exc.code} {exc.reason}" + lines.append(f"attempt #{attempt}: {last_error}") + except OSError as exc: + last_error = f"{type(exc).__name__}: {exc}" + lines.append(f"attempt #{attempt}: {last_error}") + time.sleep(5) + lines.append( + f"FAIL: no HTTP 200 via proxy after {config.PORT_PROXY_TIMEOUT}s " + f"({attempt} attempts), last error: {last_error}" + ) + return False, "\n".join(lines) + + def run_jupyter_proxy_check(pod_id: str) -> tuple[bool, str]: """Hit `https://-8888.proxy.runpod.net/api/status?token=admin` from the test machine. Verifies that: @@ -385,6 +528,140 @@ def run_jupyter_proxy_check(pod_id: str) -> tuple[bool, str]: return False, "\n".join(lines) +# --------------------------------------------------------------------------- +# Container logs via REST API (v2) + error scan +# --------------------------------------------------------------------------- + +def fetch_pod_logs_api( + pod_id: str, + tail: int = 0, + source: str = "container", + deadline_sec: int = 15, +) -> Optional[list[str]]: + """Fetch the SSE backfill from `GET /v2/pods/{id}/logs`. + + The endpoint stays open for live logs. Stop after its historical + backfill is drained (socket idle) or the deadline expires. + """ + from .instances import _load_runpod_api_key + + api_key = _load_runpod_api_key() + if not api_key: + return None + tail = tail or config.LOG_API_TAIL + req = urllib.request.Request( + f"https://api.runpod.io/v2/pods/{pod_id}/logs?source={source}&tail={tail}", + headers={ + "Authorization": f"Bearer {api_key}", + "Accept": "text/event-stream", + "User-Agent": "test-images.py/1.0 (+runpod-smoketest)", + }, + ) + lines: list[str] = [] + deadline = time.monotonic() + deadline_sec + try: + with urllib.request.urlopen(req, timeout=3) as resp: + while time.monotonic() < deadline: + try: + raw = resp.readline() + except OSError: + break + if not raw: + break + text = raw.decode("utf-8", errors="replace").strip() + if not text.startswith("data:"): + continue + try: + payload = json.loads(text[len("data:"):].strip()) + except json.JSONDecodeError: + continue + line = payload.get("line") + if line is not None: + lines.append(line.rstrip()) + except (urllib.error.HTTPError, OSError) as exc: + log(f" (log API fetch failed: {exc})", indent=2) + return None + return lines + + +def pod_status_api(pod_id: str) -> Optional[str]: + """Return lifecycle status from `GET /v2/pods/{id}`, if available.""" + from .instances import _load_runpod_api_key + + api_key = _load_runpod_api_key() + if not api_key: + return None + req = urllib.request.Request( + f"https://api.runpod.io/v2/pods/{pod_id}", + headers={ + "Authorization": f"Bearer {api_key}", + "Accept": "application/json", + "User-Agent": "test-images.py/1.0 (+runpod-smoketest)", + }, + ) + try: + with urllib.request.urlopen(req, timeout=10) as resp: + payload = json.loads(resp.read()) + except (urllib.error.HTTPError, OSError, json.JSONDecodeError): + return None + status = payload.get("status") + return status if isinstance(status, str) else None + + +def system_log_errors(pod_id: str, max_lines: int = 20) -> Optional[list[str]]: + """Return error-marker lines from the host-side REST system-log stream.""" + lines = fetch_pod_logs_api(pod_id, source="system") + if lines is None: + return None + pattern = re.compile(config.SYS_LOG_ERROR_PATTERN, re.IGNORECASE) + return [line for line in lines if pattern.search(line)][:max_lines] + + +_LOG_SCAN_ATTEMPTS = 3 +_LOG_SCAN_RETRY_SLEEP_SEC = 10 + + +def scan_pod_logs_for_errors(pod_id: str) -> tuple[bool, str]: + """Scan container stdout for configured errors. + + Empty API responses are retried and then fail as unverified: every + supported image emits boot logs, so zero lines cannot prove a clean boot. + """ + from .instances import _load_runpod_api_key + + if not _load_runpod_api_key(): + return True, "(no API key — log scan skipped)" + failures: list[str] = [] + for attempt in range(1, _LOG_SCAN_ATTEMPTS + 1): + lines = fetch_pod_logs_api(pod_id) + if lines: + break + failures.append( + f"attempt #{attempt}: " + + ("fetch failed" if lines is None else "0 log lines") + ) + if attempt < _LOG_SCAN_ATTEMPTS: + time.sleep(_LOG_SCAN_RETRY_SLEEP_SEC) + else: + return False, ( + "log scan UNVERIFIED — the log API returned no container logs " + f"after {_LOG_SCAN_ATTEMPTS} attempts ({'; '.join(failures)})" + ) + + pattern = re.compile(config.LOG_ERROR_PATTERN, re.IGNORECASE) + matches = [line for line in lines if pattern.search(line)] + if not matches: + return True, f"scanned {len(lines)} log lines — no error markers" + report = [ + f"scanned {len(lines)} log lines — " + f"{len(matches)} matched /{config.LOG_ERROR_PATTERN}/i:" + ] + report.extend(f" {line}" for line in matches[:40]) + if len(matches) > 40: + report.append(f" ... (+{len(matches) - 40} more)") + return False, "\n".join(report) + + # --------------------------------------------------------------------------- # Diagnostic log fetch # --------------------------------------------------------------------------- @@ -419,26 +696,15 @@ def _gpu_smi_block(image: str) -> str: def fetch_logs_via_ssh( - host: str, port: int, image: str, tail: int = 20, + host: str, port: int, image: str, ) -> Optional[str]: - """SSH to the pod and grab the most useful diagnostic info from inside - the container. Returns stdout on success, None if SSH didn't work.""" + """Fetch the GPU SMI snapshot, the remaining SSH-only diagnostic.""" if not config.SSH_LOG_FETCH: return None - remote_cmd = ( - "set +e; " - "echo '=== uname / hostname ==='; uname -a; hostname; " - f"echo '=== last {tail} /var/log/syslog lines ==='; " - f"tail -n {tail} /var/log/syslog 2>/dev/null || echo '(no /var/log/syslog)'; " - f"echo '=== last {tail} dmesg lines ==='; " - f"dmesg --no-pager 2>/dev/null | tail -n {tail} || echo '(dmesg unavailable)'; " - "echo '=== /var/log/*.log tails ==='; " - "for f in /var/log/*.log; do " - " [ -f \"$f\" ] || continue; " - " echo \"--- $f ---\"; tail -n 5 \"$f\" 2>/dev/null; " - "done; " - + _gpu_smi_block(image) - ) + smi_block = _gpu_smi_block(image) + if not smi_block: + return None + remote_cmd = "set +e; " + smi_block cmd = [*_ssh_command_prefix(host, port), remote_cmd] try: r = subprocess.run(cmd, capture_output=True, text=True, timeout=45) @@ -451,8 +717,8 @@ def fetch_logs_via_ssh( return f"__SSH_FAILED__\nreturncode={r.returncode}\nstderr: {r.stderr.strip()[:400]}" -def dump_pod_logs(pod_id: str, image: str, tail: int = 20) -> None: - """Print pod metadata + container logs (via direct SSH) before terminating.""" +def dump_pod_logs(pod_id: str, image: str) -> None: + """Print metadata, API container logs, system errors, and GPU SMI.""" data = runpodctl_json("pod", "get", pod_id, timeout=30) if not isinstance(data, dict): log("(could not fetch pod state)", indent=2) @@ -472,17 +738,30 @@ def dump_pod_logs(pod_id: str, image: str, tail: int = 20) -> None: ]: log(f" {key:20s} = {val!r}", indent=2) + api_lines = fetch_pod_logs_api(pod_id) + if api_lines: + log(f"--- container logs via API ({len(api_lines)} lines) ---", indent=2) + for line in api_lines: + log(f" {line}", indent=2) + + sys_errors = system_log_errors(pod_id) + if sys_errors: + log( + f"--- system-log error markers via API ({len(sys_errors)}) ---", + indent=2, + ) + for line in sys_errors: + log(f" {line}", indent=2) + if not (host and port): - log(" (no SSH endpoint yet — skipping log fetch)", indent=2) + log(" (no SSH endpoint yet — skipping GPU SMI fetch)", indent=2) log(f" inspect via UI: https://www.runpod.io/console/pods/{pod_id}", indent=2) return - log(f"--- container/system logs via SSH (root@{host}:{port}) ---", indent=2) - logs = fetch_logs_via_ssh(host, int(port), image, tail=tail) + logs = fetch_logs_via_ssh(host, int(port), image) if logs is None: - log(" (SSH log fetch disabled or ssh binary not found)", indent=2) - log(f" inspect via UI: https://www.runpod.io/console/pods/{pod_id}", indent=2) return + log(f"--- GPU SMI via SSH (root@{host}:{port}) ---", indent=2) if logs.startswith("__SSH_FAILED__"): log(" SSH could not reach the pod:", indent=2) for line in logs.splitlines()[1:]: diff --git a/tests/runpod_smoke/comfyui.py b/tests/runpod_smoke/comfyui.py new file mode 100644 index 00000000..a07a995f --- /dev/null +++ b/tests/runpod_smoke/comfyui.py @@ -0,0 +1,295 @@ +"""ComfyUI public-proxy smoke and end-to-end generation checks.""" + +from __future__ import annotations + +import json +import os +import time +import urllib.error +import urllib.parse +import urllib.request +from typing import Callable, Optional + +from . import config + +_UA = "runpod-smoke-test/1.0" +_PNG_MAGIC = b"\x89PNG\r\n\x1a\n" + + +def _base_url(pod_id: str) -> str: + return f"https://{pod_id}-{config.COMFYUI_PORT}.proxy.runpod.net" + + +def _request( + url: str, data: Optional[bytes] = None, timeout: int = 30, +) -> tuple[int, bytes]: + headers = {"User-Agent": _UA} + if data is not None: + headers["Content-Type"] = "application/json" + request = urllib.request.Request(url, data=data, headers=headers) + try: + with urllib.request.urlopen(request, timeout=timeout) as response: + return response.status, response.read() + except urllib.error.HTTPError as exc: + return exc.code, exc.read() + + +def probe_comfyui_alive( + pod_id: str, retries: int = 3, retry_sleep: int = 5, +) -> tuple[bool, str]: + """Quick post-dwell probe; this is not the initial readiness wait.""" + last_error = "" + for attempt in range(1, retries + 1): + try: + code, _ = _request(_base_url(pod_id) + "/system_stats", timeout=10) + if code == 200: + return True, f"HTTP 200 (attempt #{attempt})" + last_error = f"HTTP {code}" + except OSError as exc: + last_error = f"{type(exc).__name__}: {exc}" + if attempt < retries: + time.sleep(retry_sleep) + return False, last_error + + +def _wait_for_system_stats(base: str, emit: Callable[[str], None]) -> bool: + deadline = time.monotonic() + config.COMFYUI_WAIT_TIMEOUT + attempt = 0 + last_error = "" + while time.monotonic() < deadline: + attempt += 1 + try: + code, _ = _request(base + "/system_stats", timeout=10) + if code == 200: + emit(f"ComfyUI /system_stats OK after {attempt} probe(s)") + return True + last_error = f"HTTP {code}" + except OSError as exc: + last_error = f"{type(exc).__name__}: {exc}" + if attempt % 10 == 0: + emit(f" ...waiting for ComfyUI (last={last_error})") + time.sleep(5) + emit(f"FAIL: /system_stats unavailable after {config.COMFYUI_WAIT_TIMEOUT}s ({last_error})") + return False + + +def _load_json(path: str) -> object: + with open(path, "rb") as file: + return json.load(file) + + +def _wait_for_routes(base: str, emit: Callable[[str], None]) -> bool: + deadline = time.monotonic() + config.COMFYUI_ROUTES_TIMEOUT + last_error = "" + while time.monotonic() < deadline: + try: + code, body = _request(base + "/server_download/folder_paths", timeout=20) + if code == 200: + json.loads(body) + return True + last_error = f"HTTP {code}" + except (OSError, json.JSONDecodeError) as exc: + last_error = str(exc) + time.sleep(3) + emit(f"FAIL: RunpodDirect routes unavailable ({last_error})") + return False + + +def _ensure_model(base: str, model: dict, emit: Callable[[str], None]) -> bool: + directory = model.get("directory", "checkpoints") + filename = model["filename"] + verify = {"directory": directory, "filename": filename} + if model.get("sha256"): + verify.update({"hash": model["sha256"], "hash_type": "sha256"}) + try: + code, body = _request( + base + "/server_download/verify_model_integrity", + json.dumps(verify).encode(), + timeout=180, + ) + verified = json.loads(body) if code == 200 else {} + except (OSError, json.JSONDecodeError): + verified = {} + if verified.get("exists") and verified.get("valid"): + emit(f"model already present + verified: {directory}/{filename}") + return True + + payload = {"url": model["url"], "save_path": directory, "filename": filename} + if model.get("sha256"): + payload.update({"hash": model["sha256"], "hash_type": "sha256"}) + try: + code, body = _request( + base + "/server_download/start", json.dumps(payload).encode(), timeout=60 + ) + except OSError as exc: + emit(f"FAIL: model download request errored: {exc}") + return False + if code == 400 and b"already exists" in body.lower(): + return True + if code != 200: + emit(f"FAIL: model download request returned HTTP {code}: {body[:300]!r}") + return False + + status_url = ( + f"{base}/server_download/status/{directory}/" + f"{urllib.parse.quote(filename)}" + ) + deadline = time.monotonic() + config.COMFYUI_DOWNLOAD_TIMEOUT + while time.monotonic() < deadline: + try: + code, body = _request(status_url, timeout=20) + status = json.loads(body) if code == 200 else {} + except (OSError, json.JSONDecodeError): + status = {} + if status.get("status") == "completed": + emit(f"download complete: {directory}/{filename}") + return True + if status.get("status") in {"error", "cancelled"}: + emit(f"FAIL: download {status['status']}: {status.get('error', '')}") + return False + time.sleep(3) + emit(f"FAIL: model download timed out after {config.COMFYUI_DOWNLOAD_TIMEOUT}s") + return False + + +def _wait_for_image( + base: str, prompt_id: str, emit: Callable[[str], None], +) -> Optional[dict]: + deadline = time.monotonic() + config.COMFYUI_GEN_TIMEOUT + while time.monotonic() < deadline: + try: + code, body = _request(base + f"/history/{prompt_id}", timeout=20) + history = json.loads(body).get(prompt_id) if code == 200 else None + except (OSError, json.JSONDecodeError): + history = None + if history: + status = history.get("status", {}) + if status.get("status_str") == "error": + emit(f"FAIL: execution error: {json.dumps(status)[:1000]}") + return None + for output in history.get("outputs", {}).values(): + images = output.get("images") or [] + if images: + return images[0] + if status.get("completed") is True: + emit("FAIL: prompt completed but produced no image outputs") + return None + time.sleep(3) + emit(f"FAIL: generation did not finish in {config.COMFYUI_GEN_TIMEOUT}s") + return None + + +def _wait_for_checkpoints(base: str, workflow: dict) -> bool: + """Wait briefly for ComfyUI to index models downloaded after startup.""" + names = { + node.get("inputs", {}).get("ckpt_name") + for node in workflow.values() + if isinstance(node, dict) + and node.get("class_type") == "CheckpointLoaderSimple" + } + names.discard(None) + if not names: + return True + for _ in range(15): + try: + code, body = _request( + base + "/object_info/CheckpointLoaderSimple", timeout=30 + ) + choices = json.loads(body)[ + "CheckpointLoaderSimple" + ]["input"]["required"]["ckpt_name"][0] + if names.issubset(choices): + return True + except (OSError, json.JSONDecodeError, KeyError, TypeError): + pass + time.sleep(2) + return False + + +def _validate_image( + base: str, image: dict, save_dir: str, tag: str, emit: Callable[[str], None], +) -> bool: + query = urllib.parse.urlencode( + { + "filename": image.get("filename", ""), + "subfolder": image.get("subfolder", ""), + "type": image.get("type", "output"), + } + ) + try: + code, data = _request(base + "/view?" + query, timeout=60) + except OSError as exc: + emit(f"FAIL: image fetch errored: {exc}") + return False + if code != 200 or len(data) < 1000 or data[:8] != _PNG_MAGIC: + emit(f"FAIL: invalid PNG response (HTTP {code}, {len(data)} bytes)") + return False + width = int.from_bytes(data[16:20], "big") + height = int.from_bytes(data[20:24], "big") + if not (width > 0 and height > 0): + emit(f"FAIL: PNG has invalid IHDR dimensions ({width}x{height})") + return False + if save_dir: + try: + os.makedirs(save_dir, exist_ok=True) + filename = os.path.basename(image.get("filename", "output.png")) + with open(os.path.join(save_dir, f"{tag}_{filename}"), "wb") as file: + file.write(data) + except OSError as exc: + emit(f"warn: could not save output PNG: {exc}") + emit(f"OK: validated PNG ({len(data)} bytes, {width}x{height})") + return True + + +def run_comfyui_check( + pod_id: str, + on_line: Optional[Callable[[str], None]] = None, + save_dir: str = "", + tag: str = "", +) -> tuple[bool, str]: + """Provision a model, generate an image, and validate the resulting PNG.""" + emit = on_line or (lambda _message: None) + base = _base_url(pod_id) + emit(f"ComfyUI functional check via proxy: {base}") + if not _wait_for_system_stats(base, emit): + return False, "ComfyUI /system_stats unavailable" + try: + workflow = _load_json(config.COMFYUI_WORKFLOW) + models = _load_json(config.COMFYUI_MODELS_MANIFEST) + except OSError as exc: + return False, f"could not read test assets: {exc}" + if not _wait_for_routes(base, emit): + return False, "ComfyUI-RunpodDirect routes unavailable" + for model in models: + if not _ensure_model(base, model, emit): + return False, f"model provisioning failed: {model.get('filename')}" + if not _wait_for_checkpoints(base, workflow): + return False, "ComfyUI did not index the downloaded checkpoint" + code, body = _request( + base + "/prompt", + json.dumps({"prompt": workflow, "client_id": "runpod-smoke"}).encode(), + timeout=60, + ) + body_text = body.decode("utf-8", "replace") + if code != 200: + emit(f"FAIL: POST /prompt returned HTTP {code}: {body_text[:1000]}") + return False, f"workflow rejected by /prompt (HTTP {code})" + try: + response = json.loads(body) + except json.JSONDecodeError: + emit(f"FAIL: /prompt returned non-JSON: {body_text[:300]}") + return False, "workflow returned invalid JSON" + node_errors = response.get("node_errors") or {} + if node_errors: + emit(f"FAIL: /prompt reported node_errors: {json.dumps(node_errors)[:1000]}") + return False, "workflow rejected with node errors" + prompt_id = response.get("prompt_id") + if not prompt_id: + return False, "workflow response did not include prompt_id" + image = _wait_for_image(base, prompt_id, emit) + if not image: + return False, "generation produced no image or errored" + if not _validate_image(base, image, save_dir, tag, emit): + return False, "output PNG failed validation" + emit("COMFYUI FUNCTIONAL CHECK OK") + return True, "generated + validated PNG" diff --git a/tests/runpod_smoke/config.py b/tests/runpod_smoke/config.py index 9b2491b8..dd791060 100644 --- a/tests/runpod_smoke/config.py +++ b/tests/runpod_smoke/config.py @@ -13,6 +13,8 @@ from dataclasses import dataclass from datetime import datetime, timedelta, timezone +_PKG_DIR = os.path.dirname(os.path.abspath(__file__)) +_TESTS_DIR = os.path.dirname(_PKG_DIR) # --------------------------------------------------------------------------- # Pod / scheduling @@ -115,10 +117,9 @@ def auto_terminate_deadline() -> str: # SSH # --------------------------------------------------------------------------- -# Container logs aren't exposed via runpodctl 2.3.0's JSON, so we SSH -# directly to the pod's exposed port 22 (mapped to a random high port on -# a public IP by RunPod) to grab them. The endpoint is discovered from -# `pod get`'s ssh.ip / ssh.port fields once the pod is scheduled. +# `runpodctl` does not expose container logs in JSON. REST API v2 is the +# primary log source; SSH is retained only for the GPU SMI diagnostic. +# The SSH endpoint is discovered from `pod get` once the pod is scheduled. # Override SSH_IDENTITY if your key lives in a non-standard location. # Set SSH_LOG_FETCH=0 to skip SSH-based log fetching entirely. SSH_IDENTITY = os.environ.get("RUNPOD_SSH_KEY", "") @@ -137,6 +138,29 @@ def auto_terminate_deadline() -> str: "-o", "HostKeyAlgorithms=+ssh-rsa", ] +# --------------------------------------------------------------------------- +# Container logs via REST API (v2) +# --------------------------------------------------------------------------- + +# `GET /v2/pods/{id}/logs` streams container stdout as SSE — the one source +# SSH cannot read (PID-1 stdout is not readable from a separate process). +# The same API is used for the always-on error scan and diagnostic dumps. +# LOG_ERROR_SCAN=0 disables the error-scan step +# LOG_ERROR_PATTERN=... overrides the case-insensitive regex +# LOG_API_TAIL=N historical lines to backfill (max 5000) +LOG_ERROR_SCAN = os.environ.get("LOG_ERROR_SCAN", "1") == "1" +LOG_ERROR_PATTERN = os.environ.get( + "LOG_ERROR_PATTERN", r"\berr(or)?s?\b|\bcrash(ed|es|ing)?\b" +) +LOG_API_TAIL = int(os.environ.get("LOG_API_TAIL", "1000")) + +# System logs contain host-side failures that never reach container stdout: +# image-pull errors and `runc` container-init aborts, for example. +SYS_LOG_ERROR_PATTERN = os.environ.get( + "SYS_LOG_ERROR_PATTERN", + r"\berr(or)?s?\b|\bfail(ed|ure)?\b|\bcrash(ed|es|ing)?\b", +) + # --------------------------------------------------------------------------- # Jupyter @@ -158,6 +182,32 @@ def auto_terminate_deadline() -> str: # seconds before giving up. JUPYTER_PROXY_TIMEOUT = int(os.environ.get("JUPYTER_PROXY_TIMEOUT", "60")) +# --------------------------------------------------------------------------- +# Generic per-port checks (test_ports manifest field) +# --------------------------------------------------------------------------- + +# Each requested port is exposed as `/http` and checked proxy-first. +# These independent limits include both app cold-start and proxy registration. +PORT_WAIT_TIMEOUT = int(os.environ.get("PORT_WAIT_TIMEOUT", "300")) +PORT_PROXY_TIMEOUT = int(os.environ.get("PORT_PROXY_TIMEOUT", "300")) + +# ComfyUI listens on this HTTP port. `test_comfyui` exposes it as +# `/http` and runs a labelled proxy-first reachability check. +COMFYUI_PORT = int(os.environ.get("COMFYUI_PORT", "8188")) +COMFYUI_WORKFLOW = os.environ.get( + "COMFYUI_WORKFLOW", + os.path.join(_TESTS_DIR, "comfyui", "workflows", "gsl_starter_1_1.api.json"), +) +COMFYUI_MODELS_MANIFEST = os.environ.get( + "COMFYUI_MODELS_MANIFEST", + os.path.join(_TESTS_DIR, "comfyui", "models.json"), +) +COMFYUI_WAIT_TIMEOUT = int(os.environ.get("COMFYUI_WAIT_TIMEOUT", "600")) +COMFYUI_ROUTES_TIMEOUT = int(os.environ.get("COMFYUI_ROUTES_TIMEOUT", "60")) +COMFYUI_DOWNLOAD_TIMEOUT = int(os.environ.get("COMFYUI_DOWNLOAD_TIMEOUT", "900")) +COMFYUI_GEN_TIMEOUT = int(os.environ.get("COMFYUI_GEN_TIMEOUT", "300")) +COMFYUI_SAVE_DIR = os.environ.get("COMFYUI_SAVE_DIR", "") + # --------------------------------------------------------------------------- # CPU groups + candidates @@ -312,3 +362,19 @@ def cpu_candidate_for(instance: str) -> CpuCandidate: # JUPYTER_PASSWORD env var and exposes :8888, and `runner.test_pair` runs # the Jupyter probes after the CUDA functional check. GROUP_TEST_JUPYTER: dict[str, bool] = {} + +# Per-group HTTP ports populated from the optional `test_ports:` manifest +# list. They are exposed as `/http` and checked through the public +# proxy first; SSH only diagnoses a proxy failure. +GROUP_TEST_PORTS: dict[str, list[int]] = {} + +# ComfyUI-specific public-proxy reachability smoke. +GROUP_TEST_COMFYUI: dict[str, bool] = {} + +# End-to-end ComfyUI generation test. It implies GROUP_TEST_COMFYUI so the +# public reachability check always runs before model provisioning. +GROUP_TEST_COMFYUI_FUNCTIONAL: dict[str, bool] = {} + +# Compatibility-matrix opt-in. Each selected GPU becomes an independent job, +# rather than stopping after the first passing candidate. +GROUP_CHECK_ALL_GPU: dict[str, bool] = {} diff --git a/tests/runpod_smoke/instances.py b/tests/runpod_smoke/instances.py index cf7ca83d..aff0dd2b 100644 --- a/tests/runpod_smoke/instances.py +++ b/tests/runpod_smoke/instances.py @@ -22,14 +22,15 @@ from . import config from .log import log +from .manifest import _normalize_bool from .runpodctl import runpodctl_json -# Extract CUDA version from image tag. Supports both `cuda1281` and -# `cu1281` (interpreted as 12.8.1). Returns "X.Y" suitable for -# --min-cuda-version, or None for images without an embedded CUDA version -# (CPU images, ROCm, NGC). Anchored with \b so we don't match e.g. 'cudnn'. -CUDA_TAG_RE = re.compile(r"\bcu(?:da)?(\d{2})(\d)(\d)\b", re.IGNORECASE) +# Supports `cuda1281` / `cu1300` and the ComfyUI `cuda13.0` convention. +CUDA_TAG_RE = re.compile( + r"\bcu(?:da)?(\d{2})(?:(\d)(\d)|\.(\d))\b", + re.IGNORECASE, +) def detect_cuda_version(image: str) -> Optional[str]: @@ -38,6 +39,7 @@ def detect_cuda_version(image: str) -> Optional[str]: Examples: runpod/base:...-cuda1281-ubuntu2204 -> '12.8' runpod/pytorch:...-cu1300-torch290-... -> '13.0' + runpod/comfyui:cuda13.0 -> '13.0' runpod/base:...-rocm644-... -> None runpod/base:...-ubuntu2404 -> None runpod/nvidia-pytorch:...-25.11 -> None (NGC tag, unknown CUDA) @@ -50,7 +52,8 @@ def detect_cuda_version(image: str) -> Optional[str]: m = CUDA_TAG_RE.search(image) if not m: return None - major, minor, _patch = m.groups() + major = m.group(1) + minor = m.group(2) or m.group(4) return f"{int(major)}.{int(minor)}" @@ -248,6 +251,30 @@ def _select_by_budget(group_name: str, group_config: dict) -> list[str]: return [name for _, name in candidates] +def _select_all_gpus(group_name: str, group_config: dict) -> list[str]: + """Return every catalog GPU matching optional vRAM/vendor filters.""" + if not config.GPU_CATALOG: + log( + f"warn: group '{group_name}' uses check_all_gpu but the " + "GPU catalog is empty — set RUNPOD_API_KEY or use explicit " + "instances" + ) + return [] + min_vram = int(group_config.get("min_vram_gb", 0)) + manufacturer = (group_config.get("manufacturer") or "").lower() + names = [ + gpu["displayName"] + for gpu in config.GPU_CATALOG + if gpu.get("displayName") + and gpu.get("memoryInGb", 0) >= min_vram + and ( + not manufacturer + or (gpu.get("manufacturer") or "").lower() == manufacturer + ) + ] + return sorted(set(names)) + + def resolve_instances(group_name: str, group_config: dict) -> list[str]: """Decide which GPU display names this group should try, in order. @@ -263,6 +290,8 @@ def resolve_instances(group_name: str, group_config: dict) -> list[str]: 1. Explicit `instances:` list in the manifest — wins, used as-is. 2. `max_price_per_hour: X` (+ optional `min_vram_gb`, `manufacturer`) — auto-pick from RunPod catalog, sorted cheapest first. + 3. `check_all_gpu: true` — every catalog GPU matching the optional + vRAM/vendor filters, for compatibility-matrix runs. After candidate selection, an optional `exclude_instances:` list of fnmatch-style patterns is subtracted. Use this to block known-bad @@ -285,6 +314,8 @@ def resolve_instances(group_name: str, group_config: dict) -> list[str]: names = list(explicit) elif group_config.get("max_price_per_hour") is not None: names = _select_by_budget(group_name, group_config) + elif _normalize_bool(group_config.get("check_all_gpu")): + names = _select_all_gpus(group_name, group_config) else: return [] diff --git a/tests/runpod_smoke/log.py b/tests/runpod_smoke/log.py index 18d04917..93e1d96e 100644 --- a/tests/runpod_smoke/log.py +++ b/tests/runpod_smoke/log.py @@ -10,6 +10,7 @@ import threading from datetime import datetime +from typing import Optional _log_lock = threading.Lock() @@ -32,9 +33,17 @@ def ensure_worker_tag() -> None: _thread_local.tag = f"W{_next_worker_id}" +def set_worker_context(ctx: Optional[str]) -> None: + """Attach/clear the instance currently being tested by this thread.""" + _thread_local.ctx = ctx + + def log(msg: str, indent: int = 0) -> None: ts = datetime.now().strftime("%H:%M:%S") tag = getattr(_thread_local, "tag", "") + ctx = getattr(_thread_local, "ctx", None) + if tag and ctx: + tag = f"{tag}-{ctx}" tag_part = f"[{tag}] " if tag else "" with _log_lock: print(f"[{ts}] {tag_part}{' ' * indent}{msg}", flush=True) diff --git a/tests/runpod_smoke/pod.py b/tests/runpod_smoke/pod.py index 8a6ea050..627c4c38 100644 --- a/tests/runpod_smoke/pod.py +++ b/tests/runpod_smoke/pod.py @@ -18,7 +18,7 @@ from typing import Optional from . import config -from .checks import ssh_probe +from .checks import pod_status_api, ssh_probe, system_log_errors from .instances import detect_cuda_version from .log import log from .runpodctl import runpodctl, runpodctl_json @@ -182,6 +182,7 @@ def create_pod( compute_type: str = "GPU", group: Optional[str] = None, test_jupyter: bool = False, + test_ports: Optional[list[int]] = None, cloud_type: Optional[str] = None, data_center_ids: str = "", ) -> tuple[Optional[str], str]: @@ -220,11 +221,17 @@ def create_pod( - `--ports` gains `8888/http` - `--env` sets `JUPYTER_PASSWORD` (the value start.sh checks before starting Jupyter) + + `test_ports` exposes each generic HTTP service as `/http`. """ disk_gb = config.CPU_DISK_GB if compute_type == "CPU" else config.DISK_GB ports = ["22/tcp"] if test_jupyter: ports.append("8888/http") + for test_port in test_ports or []: + spec = f"{test_port}/http" + if spec not in ports: + ports.append(spec) args = [ "pod", "create", "--image", image, @@ -402,6 +409,20 @@ def pod_runtime_error(pod_id: str) -> Optional[str]: # Pod-lifecycle states that mean "we will never become RUNNING — stop polling". _TERMINAL_DESIRED = {"EXITED", "FAILED", "DEAD", "TERMINATED"} +_TERMINAL_API_STATUSES = {"EXITED", "ERROR", "TERMINATED"} + + +def _log_system_errors(pod_id: str, context: str) -> None: + """Print host-side API log errors when the pod cannot become ready.""" + errors = system_log_errors(pod_id) + if errors is None: + log("system logs unavailable (no API key / request failed)", indent=2) + elif not errors: + log(f"system logs: no error markers ({context})", indent=2) + else: + log(f"system-log error markers ({context}):", indent=2) + for line in errors: + log(f" {line}", indent=2) def _print_stall_hint(pod_id: str, elapsed: int) -> None: @@ -482,6 +503,7 @@ def wait_for_running(pod_id: str) -> tuple[str, str]: start = time.time() deadline = start + config.CREATE_TIMEOUT last_summary: Optional[tuple] = None + last_api_status: Optional[str] = None ssh_attempts = 0 stall_hinted = False # one-time hint when pod has no ssh endpoint for a while @@ -496,8 +518,17 @@ def wait_for_running(pod_id: str) -> tuple[str, str]: port = st.get("ssh_port") or 0 elapsed = int(time.time() - start) - if desired in _TERMINAL_DESIRED: - return "TERMINAL", f"pod entered {desired} after {elapsed}s" + api_status = pod_status_api(pod_id) + if api_status and api_status != last_api_status: + log(f"t+{elapsed}s API status: {api_status}", indent=2) + last_api_status = api_status + + if desired in _TERMINAL_DESIRED or api_status in _TERMINAL_API_STATUSES: + terminal = ( + api_status if api_status in _TERMINAL_API_STATUSES else desired + ) + _log_system_errors(pod_id, f"pod entered {terminal}") + return "TERMINAL", f"pod entered {terminal} after {elapsed}s" if host and port: ssh_attempts += 1 @@ -507,10 +538,11 @@ def wait_for_running(pod_id: str) -> tuple[str, str]: if outcome is not None: return outcome else: - summary = (desired, host, port, False) + summary = (desired, api_status, host, port, False) if summary != last_summary: log( f"t+{elapsed}s desired={desired!r} " + f"api_status={api_status!r} " f"uptime={st.get('uptime') or 0}s " "ssh endpoint not assigned yet", indent=2, @@ -518,14 +550,17 @@ def wait_for_running(pod_id: str) -> tuple[str, str]: last_summary = summary if elapsed >= config.STALL_HINT_AFTER and not stall_hinted: _print_stall_hint(pod_id, elapsed) + _log_system_errors(pod_id, f"stalled {elapsed}s") stall_hinted = True time.sleep(config.POLL_INTERVAL) + _log_system_errors(pod_id, f"timeout after {config.CREATE_TIMEOUT}s") return "TIMEOUT", ( f"SSH endpoint never became reachable in {config.CREATE_TIMEOUT}s " f"({ssh_attempts} probes) — pod stuck initializing. Likely causes: " "(1) slow/throttled image pull (check UI for pull progress), " "(2) Docker Hub rate limit if many parallel pulls of the same image, " - "(3) host scheduling delay on a saturated DC" + "(3) host scheduling delay on a saturated DC — " + "see system-log error markers above (if any)" ) diff --git a/tests/runpod_smoke/runner.py b/tests/runpod_smoke/runner.py index 0ddfc4d9..13f79204 100644 --- a/tests/runpod_smoke/runner.py +++ b/tests/runpod_smoke/runner.py @@ -24,10 +24,14 @@ run_cuda_check, run_jupyter_check, run_jupyter_proxy_check, + run_port_check, + run_port_proxy_check, + scan_pod_logs_for_errors, ssh_probe, ) +from .comfyui import probe_comfyui_alive, run_comfyui_check from .instances import detect_cuda_version, resolve_gpu_id -from .log import log +from .log import log, set_worker_context from .pod import ( TRANSIENT_RE, UNAVAILABLE_RE, @@ -105,11 +109,15 @@ def _create_pod_with_retries( f"smoketest-{int(time.time())}-" f"{threading.get_ident() % 10000:04d}-{attempt}" ) + test_ports = list(config.GROUP_TEST_PORTS.get(group) or []) + if config.GROUP_TEST_COMFYUI.get(group, False): + test_ports.append(config.COMFYUI_PORT) pod_id, raw = create_pod( image, gpu_id, name, compute_type="CPU" if is_cpu else "GPU", group=group, test_jupyter=config.GROUP_TEST_JUPYTER.get(group, False), + test_ports=test_ports, cloud_type=cloud_override, data_center_ids=dc_ids, ) @@ -194,54 +202,142 @@ def _run_jupyter_steps( ) -> Optional[_Outcome]: """Jupyter checks: only when the group opted in via `test_jupyter`. - Two stages, both must pass: - 1. IN-POD: SSH into the pod and probe 127.0.0.1:8888. Catches - start.sh regressions (e.g. wrong python interpreter for - `-m jupyter`) that don't surface in container stdout. - 2. PROXY: from the test machine, hit - https://-8888.proxy.runpod.net/. Catches port-type - mistakes (`8888/tcp` instead of `8888/http`) — proxy never - registers a non-http port, so end users can't reach Jupyter - even though the in-pod check would happily pass.""" + Checks the public proxy first — the end-user path. SSH only diagnoses + a proxy failure, so a healthy public endpoint avoids redundant work.""" if not (host and port and config.GROUP_TEST_JUPYTER.get(group, False)): return None log( - f"running Jupyter Lab check (in-pod) for group '{group}'...", + f"running Jupyter Lab check (public proxy) for pod {pod_id}...", + indent=2, + ) + ok, output = run_jupyter_proxy_check(pod_id) + for line in (output or "").splitlines(): + log(f" {line}", indent=2) + if ok: + log("jupyter check (public proxy) passed — in-pod check skipped", indent=2) + return None + + log( + "jupyter check (public proxy) FAILED — running in-pod check " + "to diagnose...", indent=2, ) ok, output = run_jupyter_check(host, port) for line in (output or "").splitlines(): log(f" {line}", indent=2) - if not ok: + if ok: log( - "jupyter check (in-pod) FAILED -- start.sh did not " - "bring up JupyterLab", + "in-pod check passed -> Jupyter is up but unreachable via " + "proxy — port likely not exposed as 8888/http", indent=2, ) dump_pod_logs(pod_id, image) - return "FAIL", "Jupyter Lab check failed (in-pod)" - log("jupyter check (in-pod) passed", indent=2) + return "FAIL", "Jupyter reachable in-pod but not via proxy" + log("in-pod check FAILED too -> JupyterLab did not start", indent=2) + dump_pod_logs(pod_id, image) + return "FAIL", "Jupyter Lab not running (proxy + in-pod failed)" + +def _check_port_proxy_first( + host: str, port: int, pod_id: str, test_port: int, label: str, +) -> Optional[_Outcome]: + """Check public reachability first; use SSH only to diagnose failure.""" log( - f"running Jupyter Lab check (public proxy) for pod {pod_id}...", + f"running {label} check (public proxy) for pod {pod_id}...", indent=2, ) - ok, output = run_jupyter_proxy_check(pod_id) - for line in (output or "").splitlines(): + ok, output = run_port_proxy_check(pod_id, test_port) + for line in output.splitlines(): log(f" {line}", indent=2) - if not ok: - log( - "jupyter check (public proxy) FAILED -- port likely " - "not exposed as 8888/http", - indent=2, + if ok: + log(f"{label} check (public proxy) passed — in-pod check skipped", indent=2) + return None + + log( + f"{label} check (public proxy) FAILED — running in-pod check " + "to diagnose...", + indent=2, + ) + ok, detail = run_port_check( + host, port, test_port, on_line=lambda line: log(f" {line}", indent=2) + ) + if ok: + failure = f"{label}: reachable in-pod but not via proxy" + else: + failure = f"{label}: service not responding (proxy + in-pod)" + if detail: + failure += f" — {detail}" + log(f"{failure} -- FAIL", indent=2) + return "FAIL", failure + + +def _run_port_steps( + host: str, port: int, pod_id: str, image: str, group: str, +) -> Optional[_Outcome]: + """Run each generic `test_ports` check.""" + for test_port in config.GROUP_TEST_PORTS.get(group) or []: + outcome = _check_port_proxy_first( + host, port, pod_id, test_port, f"port {test_port}", ) - dump_pod_logs(pod_id, image) - return "FAIL", "Jupyter Lab check failed (public proxy)" - log("jupyter check (public proxy) passed", indent=2) + if outcome is not None: + dump_pod_logs(pod_id, image) + return outcome return None +def _run_comfyui_steps( + host: str, port: int, pod_id: str, image: str, group: str, +) -> Optional[_Outcome]: + """Run the labelled ComfyUI proxy-first reachability smoke.""" + if not (host and port and config.GROUP_TEST_COMFYUI.get(group, False)): + return None + outcome = _check_port_proxy_first( + host, port, pod_id, config.COMFYUI_PORT, "ComfyUI reachability", + ) + if outcome is not None: + dump_pod_logs(pod_id, image) + return outcome + if not config.GROUP_TEST_COMFYUI_FUNCTIONAL.get(group, False): + return None + + log("running ComfyUI functional check (via proxy)...", indent=2) + ok, detail = run_comfyui_check( + pod_id, + on_line=lambda line: log(f" {line}", indent=2), + save_dir=config.COMFYUI_SAVE_DIR, + tag=pod_id, + ) + if ok: + log("ComfyUI functional check passed", indent=2) + return None + log(f"ComfyUI functional check FAILED -- {detail}", indent=2) + dump_pod_logs(pod_id, image) + return "FAIL", f"ComfyUI functional check failed: {detail[:160]}" + + +def _run_log_scan_step(pod_id: str, image: str) -> Optional[_Outcome]: + """Scan the REST API container-log backfill for boot error markers.""" + if not config.LOG_ERROR_SCAN: + return None + log("scanning container logs for error markers (via API)...", indent=2) + ok, report = scan_pod_logs_for_errors(pod_id) + for line in report.splitlines(): + log(f" {line}", indent=2) + if ok: + log("log scan passed", indent=2) + return None + + detail = ( + "log scan unverified — log API returned no container logs" + if report.startswith("log scan UNVERIFIED") + else "error markers found in container logs" + ) + log(f"log scan FAILED -- {detail}", indent=2) + dump_pod_logs(pod_id, image) + return "FAIL", detail + + def _run_dwell_step(pod_id: str, image: str) -> Optional[_Outcome]: """Brief dwell to catch containers that boot, accept SSH, then crash. Most real images hit this in the first ~30s if they're going to crash. @@ -269,6 +365,33 @@ def _run_dwell_step(pod_id: str, image: str) -> Optional[_Outcome]: ) +def _run_post_dwell_steps( + pod_id: str, image: str, group: str, +) -> Optional[_Outcome]: + """Re-probe ComfyUI and scan logs after the dwell window.""" + if config.DWELL_SEC <= 0: + return None + if ( + config.GROUP_TEST_COMFYUI.get(group, False) + or config.GROUP_TEST_COMFYUI_FUNCTIONAL.get(group, False) + ): + log("re-probing ComfyUI after dwell...", indent=2) + ok, detail = probe_comfyui_alive(pod_id) + if not ok: + log( + f"ComfyUI re-probe FAILED after dwell ({detail})", + indent=2, + ) + dump_pod_logs(pod_id, image) + return "FAIL", ( + f"ComfyUI stopped answering during the {config.DWELL_SEC}s " + f"dwell (post-dwell probe: {detail})" + ) + log(f"ComfyUI re-probe passed ({detail})", indent=2) + log("re-scanning container logs after dwell...", indent=2) + return _run_log_scan_step(pod_id, image) + + def test_pair(image: str, instance: str, group: str) -> _Outcome: """Returns (status, detail). Statuses: 'PASS' — image booted, CUDA check OK, survived dwell @@ -326,9 +449,21 @@ def test_pair(image: str, instance: str, group: str) -> _Outcome: if outcome is not None: return outcome outcome = _run_jupyter_steps(host, port, pod_id, image, group) + if outcome is not None: + return outcome + outcome = _run_port_steps(host, port, pod_id, image, group) + if outcome is not None: + return outcome + outcome = _run_comfyui_steps(host, port, pod_id, image, group) + if outcome is not None: + return outcome + outcome = _run_log_scan_step(pod_id, image) if outcome is not None: return outcome outcome = _run_dwell_step(pod_id, image) + if outcome is not None: + return outcome + outcome = _run_post_dwell_steps(pod_id, image, group) if outcome is not None: return outcome @@ -357,10 +492,15 @@ def test_image( """ log(f"image: {image}") stuck_instances: list[str] = [] + unavailable_instances: list[str] = [] last_create_error = "" last_create_inst = "" for inst in instances: - result, detail = test_pair(image, inst, group) + set_worker_context(inst) + try: + result, detail = test_pair(image, inst, group) + finally: + set_worker_context(None) if result == "PASS": return "PASS", "", inst if result == "FAIL": @@ -379,7 +519,8 @@ def test_image( continue if result == "STUCK": stuck_instances.append(inst) - # UNAVAILABLE: silently try next + if result == "UNAVAILABLE": + unavailable_instances.append(inst) if last_create_error: # We never got past pod-create on any instance and the errors # weren't capacity-shortages. Surface the last orchestrator error @@ -401,11 +542,11 @@ def test_image( f"{len(stuck_instances)} instance type(s) — likely a " "scheduler issue, try again later" ), - "", + ", ".join(stuck_instances), ) log(f"all {len(instances)} instances unavailable (no capacity)", indent=1) return ( "SKIP", f"no capacity on any of {len(instances)} candidate instance type(s)", - "", + ", ".join(unavailable_instances), ) diff --git a/tests/test_images.py b/tests/test_images.py index 2dffffc4..4dfb7387 100755 --- a/tests/test_images.py +++ b/tests/test_images.py @@ -48,10 +48,9 @@ # entry; the runner iterates instances internally until something settles. Job = tuple[str, str, list[str]] -# Per-image outcome: (summary_status, note, instance_used). status is one -# of "PASS" / "FAIL" / "SKIP". instance_used is "" when the test never -# landed on any host (all UNAVAILABLE / STUCK). -Result = tuple[str, str, str] +# Per-attempt outcome: (image, status, note, instance_used). A list avoids +# overwriting rows when `check_all_gpu` creates one job per GPU. +Result = tuple[str, str, str, str] # --------------------------------------------------------------------------- @@ -125,11 +124,38 @@ def _init_registry_auth() -> None: # --------------------------------------------------------------------------- +def _coerce_ports(raw_ports: object, group: str) -> list[int]: + """Coerce optional `test_ports` list entries to valid TCP port numbers.""" + if not isinstance(raw_ports, list): + return [] + ports: list[int] = [] + for entry in raw_ports: + try: + port = int(str(entry).strip()) + except (TypeError, ValueError): + log( + f"warn: group '{group}': test_ports entry {entry!r} " + "is not a valid TCP port — skipping" + ) + continue + if 1 <= port <= 65535: + ports.append(port) + else: + log( + f"warn: group '{group}': test_ports entry {entry!r} " + "is outside 1–65535 — skipping" + ) + return ports + + def _apply_manifest_overrides(manifest: dict[str, dict]) -> None: """Populate the per-group dicts on `config` that `pod.create_pod` and `runner.test_pair` consult at run-time: `GROUP_MIN_CUDA` (fallback CUDA version for tag-less images like NGC `nvidia-pytorch:25.11`) - and `GROUP_TEST_JUPYTER` (opt-in for the Jupyter probes).""" + `GROUP_TEST_JUPYTER` (opt-in for the Jupyter probes), and + `GROUP_TEST_PORTS` (generic public HTTP service probes), + `GROUP_CHECK_ALL_GPU` (one independent job per matching GPU), and the + ComfyUI reachability / functional-generation opt-ins.""" for grp, contents in manifest.items(): normalized = _normalize_cuda_version(contents.get("min_cuda_version")) if normalized: @@ -149,6 +175,35 @@ def _apply_manifest_overrides(manifest: dict[str, dict]) -> None: f"group '{grp}': test_jupyter=true " "(JUPYTER_PASSWORD=, expose 8888/http)" ) + for grp, contents in manifest.items(): + ports = _coerce_ports(contents.get("test_ports"), grp) + if ports: + config.GROUP_TEST_PORTS[grp] = ports + log( + f"group '{grp}': test_ports={ports} " + "(expose as /http, probe public proxy first)" + ) + for grp, contents in manifest.items(): + if _normalize_bool(contents.get("check_all_gpu")): + config.GROUP_CHECK_ALL_GPU[grp] = True + log( + f"group '{grp}': check_all_gpu=true " + "(one independent smoke job per resolved GPU)" + ) + reach = _normalize_bool(contents.get("test_comfyui")) + functional = _normalize_bool(contents.get("test_comfyui_functional")) + if functional: + config.GROUP_TEST_COMFYUI_FUNCTIONAL[grp] = True + log( + f"group '{grp}': test_comfyui_functional=true " + "(provision model, run workflow, validate PNG)" + ) + if reach or functional: + config.GROUP_TEST_COMFYUI[grp] = True + log( + f"group '{grp}': test_comfyui=true " + f"(proxy-first reachability on :{config.COMFYUI_PORT})" + ) def _resolve_all_instances(manifest: dict[str, dict]) -> dict[str, list[str]]: @@ -208,7 +263,7 @@ def _build_jobs( manifest: dict[str, dict], resolved: dict[str, list[str]], group_filter: Optional[str], - results: dict[str, Result], + results: list[Result], ) -> list[Job]: """Flatten the manifest into a list of `(image, group, instances)` jobs that can run independently. Groups with no resolvable instances @@ -222,14 +277,18 @@ def _build_jobs( if not instances: log( f"skipping group '{group}': no instances resolved " - "(neither 'instances:' nor 'max_price_per_hour:' produced " - "any candidates)" + "(none of 'instances:', 'max_price_per_hour:' or " + "'check_all_gpu:' produced candidates)" ) for img in contents.get("images", []): - results[img] = ("SKIP", "no instances configured", "") + results.append((img, "SKIP", "no instances configured", "")) continue + check_all = config.GROUP_CHECK_ALL_GPU.get(group, False) for img in contents.get("images", []): - jobs.append((img, group, instances)) + if check_all: + jobs.extend((img, group, [inst]) for inst in instances) + else: + jobs.append((img, group, instances)) return jobs @@ -238,7 +297,7 @@ def _build_jobs( # --------------------------------------------------------------------------- -def _run_jobs_serial(jobs: list[Job], results: dict[str, Result]) -> None: +def _run_jobs_serial(jobs: list[Job], results: list[Result]) -> None: """Single-threaded run — no worker tags, simpler logs, group-header banner each time the group changes.""" current_group: Optional[str] = None @@ -247,32 +306,32 @@ def _run_jobs_serial(jobs: list[Job], results: dict[str, Result]) -> None: print() log(f"---------- group: {group} ----------") current_group = group - results[img] = test_image(img, instances, group) + status, note, instance = test_image(img, instances, group) + results.append((img, status, note, instance)) -def _run_one_tagged_job(job: Job) -> tuple[str, Result]: +def _run_one_tagged_job(job: Job) -> Result: """ThreadPool worker. The W tag is assigned to the THREAD (not the job), so e.g. with 5 jobs and 3 workers you still see only W1/W2/W3, each handling 1-2 jobs sequentially.""" img, grp, insts = job ensure_worker_tag() log(f"start [group={grp}] image={img}") - res = test_image(img, insts, grp) - log(f"done [group={grp}] image={img} -> {res[0]}") - return img, res + status, note, instance = test_image(img, insts, grp) + log(f"done [group={grp}] image={img} -> {status}") + return img, status, note, instance -def _run_jobs_parallel(jobs: list[Job], results: dict[str, Result]) -> None: +def _run_jobs_parallel(jobs: list[Job], results: list[Result]) -> None: """ThreadPool fan-out capped at MAX_PARALLEL. Each worker holds at most one pod at a time.""" with ThreadPoolExecutor(max_workers=config.MAX_PARALLEL) as pool: futures = [pool.submit(_run_one_tagged_job, job) for job in jobs] for fut in as_completed(futures): - img, res = fut.result() - results[img] = res + results.append(fut.result()) -def _run_jobs(jobs: list[Job], results: dict[str, Result]) -> None: +def _run_jobs(jobs: list[Job], results: list[Result]) -> None: if not jobs: log("no jobs to run after filtering") return @@ -304,7 +363,7 @@ def _format_result_line(want: str, img: str, status: str, note: str, return f" {want:6s} {img}{inst_str}{note_str}" -def _print_summary(results: dict[str, Result]) -> int: +def _print_summary(results: list[Result]) -> int: """Print the SUMMARY block and return the exit code. FAIL is ALWAYS fatal (exit 1) — a broken container is never something @@ -328,7 +387,7 @@ def _print_summary(results: dict[str, Result]) -> int: print(" SUMMARY ".center(84, "=")) print("=" * 84) counts: dict[str, int] = defaultdict(int) - for status, _, _ in results.values(): + for _img, status, _note, _instance in results: counts[status] += 1 print( f"totals: {counts['PASS']} PASS, " @@ -336,7 +395,7 @@ def _print_summary(results: dict[str, Result]) -> int: f"{counts['SKIP']} SKIP\n" ) for want in ("FAIL", "SKIP", "PASS"): - for img, (status, note, instance) in results.items(): + for img, status, note, instance in results: line = _format_result_line(want, img, status, note, instance) if line is not None: print(line) @@ -382,13 +441,17 @@ def main() -> int: _init_registry_auth() manifest = parse_manifest(manifest_path) - _apply_manifest_overrides(manifest) + try: + _apply_manifest_overrides(manifest) + except ValueError as exc: + log(f"error: {exc}") + return 1 resolved = _resolve_all_instances(manifest) _warn_unknown_instances(resolved) _log_budget_picks(manifest, resolved) - results: dict[str, Result] = {} + results: list[Result] = [] jobs = _build_jobs(manifest, resolved, group_filter, results) _run_jobs(jobs, results) From 3dfb7e72064b08c8d5acd115d5a76d2156fd29c3 Mon Sep 17 00:00:00 2001 From: chmokachka Date: Thu, 27 Aug 2026 13:25:45 +0300 Subject: [PATCH 12/33] fix: comfyui start --- official-templates/comfyui/Dockerfile | 2 +- official-templates/comfyui/scripts/start.sh | 88 ++++++++++++++++++++- 2 files changed, 86 insertions(+), 4 deletions(-) diff --git a/official-templates/comfyui/Dockerfile b/official-templates/comfyui/Dockerfile index fe1f3d3c..3e108e6d 100644 --- a/official-templates/comfyui/Dockerfile +++ b/official-templates/comfyui/Dockerfile @@ -217,7 +217,7 @@ ENV LD_LIBRARY_PATH=/usr/local/cuda/lib64 ENV NVIDIA_REQUIRE_CUDA="" ENV NVIDIA_DISABLE_REQUIRE=true ENV NVIDIA_VISIBLE_DEVICES=all -ENV NVIDIA_DRIVER_CAPABILITIES=compute,utility,video +ENV NVIDIA_DRIVER_CAPABILITIES=all # Jupyter is included in the lock file and installed in the builder stage diff --git a/official-templates/comfyui/scripts/start.sh b/official-templates/comfyui/scripts/start.sh index 5852ef2e..aab4f1f1 100755 --- a/official-templates/comfyui/scripts/start.sh +++ b/official-templates/comfyui/scripts/start.sh @@ -170,6 +170,88 @@ upgrade_comfyui_if_needed() { echo "ComfyUI workspace upgraded successfully" } +log_cuda_venv_diagnostics() { + local expected_build local_packages status + expected_build=$(sed -n 's/^torch==.*+\(cu[0-9][0-9]*\).*$/\1/p' \ + "$PIP_CONSTRAINT_FILE" | head -n 1) + + echo "=============================================" + echo " CUDA / venv diagnostics" + echo " active venv: $VENV_DIR" + echo " expected image torch build: ${expected_build:-unknown}" + + # Fail open: a crashed probe must not abort startup (set -e). + if ! status=$(VENV_DIR="$VENV_DIR" EXPECTED_TORCH_BUILD="$expected_build" \ + python - <<'PY' +import os +import sys + +try: + venv = os.path.realpath(os.environ["VENV_DIR"]) + expected = os.environ.get("EXPECTED_TORCH_BUILD", "") + print(f" python: {sys.executable}") + + try: + import torch + except Exception as exc: + print(f" torch import: FAILED ({type(exc).__name__}: {exc})") + print("TORCH_STATUS=IMPORT_FAILED") + else: + actual = f"cu{torch.version.cuda.replace('.', '')}" if torch.version.cuda else "cpu" + location = os.path.realpath(torch.__file__) + local_torch = location.startswith(venv + os.sep) + print(f" torch: {torch.__version__}") + print(f" torch CUDA build: {actual}") + print(f" torch location: {location}") + print(f" torch from active venv: {local_torch}") + print(f" CUDA available: {torch.cuda.is_available()}") + print(f" CUDA device count: {torch.cuda.device_count()}") + if expected and actual != expected: + print("TORCH_STATUS=CUDA_MISMATCH") + elif local_torch: + print("TORCH_STATUS=LOCAL_TORCH") + else: + print("TORCH_STATUS=OK") +except Exception as exc: + print(f" probe failed: {type(exc).__name__}: {exc}") + print("TORCH_STATUS=PROBE_FAILED") +PY + ); then + echo " WARNING: CUDA / venv probe exited unexpectedly; continuing startup." + fi + echo "$status" | grep -v '^TORCH_STATUS=' || true + + # Allow optional spaces before ==/@ so PEP 508 direct URL lines match + # (e.g. "xformers @ https://..."), and include onnxruntime-gpu. + local_packages=$(python -m pip freeze --local \ + | grep -Ei '^(torch|torchvision|torchaudio|xformers|triton|onnxruntime(-gpu)?|sageattention)[[:space:]]*(==|@)' \ + || true) + if [ -n "$local_packages" ]; then + echo " locally installed CUDA-sensitive packages:" + echo "$local_packages" | sed 's/^/ /' + else + echo " locally installed CUDA-sensitive packages: none" + fi + + case "$status" in + *"TORCH_STATUS=OK"*) + echo " CUDA / venv status: OK — using the image PyTorch stack." + ;; + *"TORCH_STATUS=LOCAL_TORCH"*) + echo " WARNING: PyTorch is installed inside the persistent venv." + echo " It overrides the image PyTorch stack; verify it matches this image's CUDA build." + ;; + *"TORCH_STATUS=CUDA_MISMATCH"*) + echo " WARNING: Persistent venv PyTorch does not match this image's CUDA build." + echo " This can prevent CUDA from initializing. No files were changed automatically." + ;; + *) + echo " WARNING: Could not verify the persistent venv PyTorch stack." + ;; + esac + echo "=============================================" +} + # ---------------------------------------------------------------------------- # # Main Program # # ---------------------------------------------------------------------------- # @@ -222,7 +304,6 @@ if [ -d "$OLD_VENV_DIR" ] && [ ! -d "$VENV_DIR" ]; then mv "$OLD_VENV_DIR" "${OLD_VENV_DIR}.bak" cd "$COMFYUI_DIR" python3.12 -m venv --system-site-packages "$VENV_DIR" - # shellcheck disable=SC1091 source "$VENV_DIR/bin/activate" python -m ensurepip # Skip nodes baked into the image — their deps are in system site-packages @@ -261,7 +342,6 @@ if [ ! -d "$COMFYUI_DIR" ] || [ ! -d "$VENV_DIR" ]; then if [ ! -d "$VENV_DIR" ]; then cd "$COMFYUI_DIR" python3.12 -m venv --system-site-packages "$VENV_DIR" - # shellcheck disable=SC1091 source "$VENV_DIR/bin/activate" # Ensure pip is available in the venv (needed for ComfyUI-Manager) @@ -282,6 +362,8 @@ fi echo "Warming up pip (Manager timeout is 5s)..." time python -m pip --version +log_cuda_venv_diagnostics + # Start ComfyUI — keep container alive if it crashes so SSH/Jupyter remain accessible cd $COMFYUI_DIR FIXED_ARGS="--listen 0.0.0.0 --port 8188 --enable-cors-header" @@ -293,7 +375,7 @@ if [ -s "$ARGS_FILE" ]; then fi echo "Starting ComfyUI with args: $FIXED_ARGS" -python main.py "$FIXED_ARGS" & +python main.py $FIXED_ARGS & COMFY_PID=$! # Distinguish a real ComfyUI crash from the pod being stopped/restarted/ From 62455b0bff3de6faba5ecab1d3adfc4e9ba50315 Mon Sep 17 00:00:00 2001 From: chmokachka Date: Fri, 28 Aug 2026 16:14:47 +0300 Subject: [PATCH 13/33] feat: compatibility tests --- .github/workflows/_tmp-gpu-compat-probe.yml | 60 ++++++ .github/workflows/base.yml | 1 - .github/workflows/gpu-compatibility.yml | 218 ++++++++++++++++++++ official-templates/comfyui/scripts/start.sh | 23 ++- tests/runpod_smoke/checks.py | 39 ++++ tests/runpod_smoke/pod.py | 32 ++- tests/runpod_smoke/runner.py | 43 +++- tests/test_images.py | 35 ++-- 8 files changed, 405 insertions(+), 46 deletions(-) create mode 100644 .github/workflows/_tmp-gpu-compat-probe.yml create mode 100644 .github/workflows/gpu-compatibility.yml diff --git a/.github/workflows/_tmp-gpu-compat-probe.yml b/.github/workflows/_tmp-gpu-compat-probe.yml new file mode 100644 index 00000000..efef79db --- /dev/null +++ b/.github/workflows/_tmp-gpu-compat-probe.yml @@ -0,0 +1,60 @@ +name: TEMP GPU compat probe + +# ============================================================================ +# TEMPORARY — DELETE BEFORE MERGING THIS PR. +# +# workflow_dispatch only appears in the UI once the file is on the default +# branch, so gpu-compatibility.yml can't be run by hand yet. This fires on the +# PR instead, purely to confirm the new `cudaVersion` column comes back +# populated and to see whether a cu1281 image actually lands on 12.8. +# +# Every push matching `paths` below spends real GPU money. +# ============================================================================ + +on: + pull_request: + paths: + - '.github/workflows/_tmp-gpu-compat-probe.yml' + - 'tests/**' + +permissions: + contents: read + +# Queue rather than cancel: a cancel SIGKILLs the runner before test_images.py +# can delete its pods. +concurrency: + group: tmp-gpu-compat-probe + cancel-in-progress: false + +jobs: + probe: + runs-on: blacksmith-4vcpu-ubuntu-2404 + timeout-minutes: 240 + steps: + - name: Checkout + uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 + with: + fetch-depth: 0 + + - name: GPU compat probe + uses: ./.github/actions/smoke-test + with: + image-refs: '["docker.io/runpod/pytorch:1.2.0-rc.162-cu1281-torch2121-ubuntu2404"]' + profile: gpu + runpod-api-key: ${{ secrets.TESTING_RUNPOD_API_KEY }} + ssh-private-key: ${{ secrets.TESTING_RUNPOD_SSH_PRIVATE_KEY }} + + # One pod per GPU, bounded to >=80GB cards so this stays ~7 pods + # instead of the full ~30-type catalog. + check-all-gpu: "true" + manufacturer: Nvidia + min-vram-gb: "80" + max-parallel: "3" + + # Deliberately NOT set: the floor is derived from the tag (12.8), so + # the summary shows whether the host was actually newer. + # min-cuda-version: + + test-jupyter: "true" + on-skip: pass + create-timeout: "1200" diff --git a/.github/workflows/base.yml b/.github/workflows/base.yml index deba0560..0035f4ca 100644 --- a/.github/workflows/base.yml +++ b/.github/workflows/base.yml @@ -619,4 +619,3 @@ jobs: profile: gpu runpod-api-key: ${{ secrets.TESTING_RUNPOD_API_KEY }} ssh-private-key: ${{ secrets.TESTING_RUNPOD_SSH_PRIVATE_KEY }} - min-cuda-version: "12.8" diff --git a/.github/workflows/gpu-compatibility.yml b/.github/workflows/gpu-compatibility.yml new file mode 100644 index 00000000..45b73dcd --- /dev/null +++ b/.github/workflows/gpu-compatibility.yml @@ -0,0 +1,218 @@ +name: GPU Compatibility Matrix + +# Manual sweep for any image: boots it on EVERY GPU in the RunPod catalog +# independently and reports a per-GPU pass/fail matrix. +# +# COST: ~30 NVIDIA types x 5-10 min per pod, far longer with the ComfyUI +# functional check. Narrow with min-vram-gb / exclude-instances. + +on: + workflow_dispatch: + inputs: + image: + description: "Image ref WITH tag, e.g. runpod/comfyui:1.0.7-comfyuiv0.30.0-cuda12.8" + type: string + required: true + test-ports: + description: "HTTP ports to probe, comma- or space-separated (e.g. 8080, 8888). Empty = none." + type: string + required: false + default: "" + cuda-version: + description: "CUDA the host driver must support, X.Y (e.g. 13.0). Empty = derive from the image tag." + type: string + required: false + default: "" + test-comfyui: + description: "ComfyUI smoke test — is ComfyUI up and reachable on :8188" + type: boolean + required: false + default: false + test-comfyui-functional: + description: "ComfyUI functional test — download model, run workflow, validate PNG (forces the smoke test first)" + type: boolean + required: false + default: false + exclude-instances: + description: "GPUs to skip: comma-separated fnmatch patterns (e.g. *Blackwell*, RTX A4000)" + type: string + required: false + default: "" + test-jupyter: + description: "JupyterLab probe on :8888 — only for images using container-template/start.sh" + type: boolean + required: false + default: false + manufacturer: + description: "GPU vendor to sweep (AMD for ROCm images)" + type: choice + required: false + default: "Nvidia" + options: + - Nvidia + - AMD + min-vram-gb: + description: "Skip GPUs below this vRAM. 0 = sweep the whole catalog." + type: string + required: false + default: "0" + max-parallel: + description: "Pods under test at once (each worker holds one pod — this is the cost throttle)" + type: string + required: false + default: "2" + +permissions: + contents: read + +# cancel-in-progress is false on purpose: a cancel SIGKILLs the runner before +# test_images.py can terminate its pods, so queue rather than leak paid pods. +concurrency: + group: gpu-compat-${{ inputs.image }} + cancel-in-progress: false + +jobs: + compatibility: + runs-on: blacksmith-4vcpu-ubuntu-2404 + # A full sweep outlives the 360-minute default. + timeout-minutes: 720 + steps: + - name: Checkout + uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 + with: + fetch-depth: 0 + + - name: Normalize inputs + id: prep + shell: bash + env: + IMAGE: ${{ inputs.image }} + PORTS: ${{ inputs.test-ports }} + EXCLUDE: ${{ inputs.exclude-instances }} + run: | + set -euo pipefail + + # Dispatch gives single-line strings; smoke-test wants one entry per + # line. Ports also accept spaces, exclude patterns must not — GPU + # display names contain them ("RTX A4000"). + to_lines() { + printf '%s' "$1" \ + | tr ',' '\n' \ + | sed 's/^[[:space:]]*//; s/[[:space:]]*$//' \ + | grep -v '^$' || true + } + + emit_multiline() { + { + echo "$1<<__GH_EOF__" + to_lines "$2" + echo "__GH_EOF__" + } >> "$GITHUB_OUTPUT" + } + + IMAGE_TRIMMED=$(printf '%s' "${IMAGE}" | tr -d '[:space:]') + if [[ -z "${IMAGE_TRIMMED}" ]]; then + echo "::error::image input is empty" + exit 1 + fi + if [[ "${IMAGE_TRIMMED}" != *:* ]]; then + echo "::error::image '${IMAGE_TRIMMED}' has no tag — pass an explicit tag, not a bare repository" + exit 1 + fi + printf 'image-refs=["%s"]\n' "${IMAGE_TRIMMED}" >> "$GITHUB_OUTPUT" + printf 'image=%s\n' "${IMAGE_TRIMMED}" >> "$GITHUB_OUTPUT" + + emit_multiline ports "$(printf '%s' "${PORTS}" | tr ' ' ',')" + emit_multiline exclude "${EXCLUDE}" + + - name: Validate numeric inputs + shell: bash + env: + MIN_VRAM: ${{ inputs.min-vram-gb }} + MAX_PARALLEL: ${{ inputs.max-parallel }} + CUDA: ${{ inputs.cuda-version }} + run: | + set -euo pipefail + # Fail before the runpodctl install and catalog fetch, not with an + # int()/float() traceback deep in the harness. + [[ "${MIN_VRAM}" =~ ^[0-9]+$ ]] \ + || { echo "::error::min-vram-gb must be a whole number, got '${MIN_VRAM}'"; exit 1; } + [[ "${MAX_PARALLEL}" =~ ^[1-9][0-9]*$ ]] \ + || { echo "::error::max-parallel must be a positive integer, got '${MAX_PARALLEL}'"; exit 1; } + [[ -z "${CUDA}" || "${CUDA}" =~ ^[0-9]+\.[0-9]+$ ]] \ + || { echo "::error::cuda-version must be X.Y (e.g. 13.0) or empty, got '${CUDA}'"; exit 1; } + + - name: Summarize run parameters + shell: bash + env: + # Free-text inputs go through env so shell metacharacters can't + # break out of the echo. + IMAGE: ${{ steps.prep.outputs.image }} + PORTS: ${{ steps.prep.outputs.ports }} + EXCLUDE: ${{ steps.prep.outputs.exclude }} + CUDA: ${{ inputs.cuda-version }} + VENDOR: ${{ inputs.manufacturer }} + MIN_VRAM: ${{ inputs.min-vram-gb }} + MAX_PARALLEL: ${{ inputs.max-parallel }} + JUPYTER: ${{ inputs.test-jupyter }} + COMFY_SMOKE: ${{ inputs.test-comfyui || inputs.test-comfyui-functional }} + COMFY_FUNC: ${{ inputs.test-comfyui-functional }} + run: | + set -euo pipefail + # Not `paste -sd', '` — -d takes a delimiter LIST and alternates + # through it, yielding '8080,8888 9000'. + ports_1l=$(printf '%s' "${PORTS}" | paste -sd, - | sed 's/,/, /g') + exclude_1l=$(printf '%s' "${EXCLUDE}" | paste -sd, - | sed 's/,/, /g') + { + echo "### GPU compatibility sweep" + echo + echo "| Setting | Value |" + echo "| --- | --- |" + echo "| Image | \`${IMAGE}\` |" + echo "| CUDA floor | ${CUDA:-derived from tag} |" + echo "| Vendor | ${VENDOR} |" + echo "| Min vRAM | ${MIN_VRAM} GB |" + echo "| Ports | ${ports_1l:-none} |" + echo "| Excluded GPUs | ${exclude_1l:-none} |" + echo "| Jupyter | ${JUPYTER} |" + echo "| ComfyUI smoke | ${COMFY_SMOKE} |" + echo "| ComfyUI functional | ${COMFY_FUNC} |" + echo "| Parallel pods | ${MAX_PARALLEL} |" + } >> "$GITHUB_STEP_SUMMARY" + + - name: Run compatibility matrix + uses: ./.github/actions/smoke-test + with: + image-refs: ${{ steps.prep.outputs.image-refs }} + profile: gpu + runpod-api-key: ${{ secrets.TESTING_RUNPOD_API_KEY }} + ssh-private-key: ${{ secrets.TESTING_RUNPOD_SSH_PRIVATE_KEY }} + + # One pod per GPU, no first-PASS short-circuit. This also drops the + # budget filter, so min-vram-gb and exclude-instances are the only + # ways to narrow the sweep. + check-all-gpu: "true" + manufacturer: ${{ inputs.manufacturer }} + min-vram-gb: ${{ inputs.min-vram-gb }} + exclude-instances: ${{ steps.prep.outputs.exclude }} + + # Empty = derived from the tag by instances.detect_cuda_version. + min-cuda-version: ${{ inputs.cuda-version }} + + test-ports: ${{ steps.prep.outputs.ports }} + test-jupyter: ${{ inputs.test-jupyter }} + + # test_images.py also forces this, but OR-ing keeps the generated + # manifest self-explanatory. + test-comfyui: ${{ inputs.test-comfyui || inputs.test-comfyui-functional }} + test-comfyui-functional: ${{ inputs.test-comfyui-functional }} + save-comfyui-images: ${{ inputs.test-comfyui-functional }} + comfyui-images-artifact-name: comfyui-images-gpu-matrix-${{ github.run_id }} + + # A catalog-wide sweep always hits GPUs with no free capacity. + # Those are SKIPs, not defects; real FAILs stay fatal. + on-skip: pass + + # Worst case across images: rocm/* bases are 30-50 GB. + create-timeout: "1200" + max-parallel: ${{ inputs.max-parallel }} diff --git a/official-templates/comfyui/scripts/start.sh b/official-templates/comfyui/scripts/start.sh index aab4f1f1..e181a8ae 100755 --- a/official-templates/comfyui/scripts/start.sh +++ b/official-templates/comfyui/scripts/start.sh @@ -365,17 +365,22 @@ time python -m pip --version log_cuda_venv_diagnostics # Start ComfyUI — keep container alive if it crashes so SSH/Jupyter remain accessible -cd $COMFYUI_DIR -FIXED_ARGS="--listen 0.0.0.0 --port 8188 --enable-cors-header" +cd "$COMFYUI_DIR" + +# Array, not a string: argparse needs each flag as its own argv entry. +COMFY_ARGS=(--listen 0.0.0.0 --port 8188 --enable-cors-header) + +# One array element per word, since a line may hold `--foo bar`. if [ -s "$ARGS_FILE" ]; then - CUSTOM_ARGS=$(grep -v '^#' "$ARGS_FILE" | tr '\n' ' ') - if [ ! -z "$CUSTOM_ARGS" ]; then - FIXED_ARGS="$FIXED_ARGS $CUSTOM_ARGS" - fi + while read -r word; do + if [ -n "$word" ]; then + COMFY_ARGS+=("$word") + fi + done < <(grep -v '^[[:space:]]*#' "$ARGS_FILE" | tr -s '[:space:]' '\n') fi -echo "Starting ComfyUI with args: $FIXED_ARGS" -python main.py $FIXED_ARGS & +echo "Starting ComfyUI with args: ${COMFY_ARGS[*]}" +python main.py "${COMFY_ARGS[@]}" & COMFY_PID=$! # Distinguish a real ComfyUI crash from the pod being stopped/restarted/ @@ -402,7 +407,7 @@ echo " Check the logs above for the error/traceback." echo " SSH and JupyterLab are still available." echo " To restart after fixing:" echo " cd $COMFYUI_DIR && source .venv-cu128/bin/activate" -echo " python main.py $FIXED_ARGS" +echo " python main.py ${COMFY_ARGS[*]}" echo "=============================================" sleep infinity diff --git a/tests/runpod_smoke/checks.py b/tests/runpod_smoke/checks.py index cf334aab..ee1a6aa1 100644 --- a/tests/runpod_smoke/checks.py +++ b/tests/runpod_smoke/checks.py @@ -141,6 +141,45 @@ def _image_expects_rocm(image: str) -> bool: return bool(_ROCM_TAG_RE.search(image)) +def fetch_pod_cuda_version(pod_id: str, attempts: int = 3) -> str: + """Return the CUDA version the host reported, e.g. '13.0'. + + `min_cuda_version` is only a floor, so the scheduler may place the pod on + any host at or above it — this reports what it actually got, which is the + point of a compatibility matrix. `runpodctl` drops the field, so it comes + from `GET /v2/pods/{id}` rather than `pod get`. + + Nullable per the API: CPU pods and hosts that never reported one give ''. + Retried a few times because the value only lands once the scheduler has + assigned a machine. Reporting only — never turns a PASS into a FAIL. + """ + from .instances import _load_runpod_api_key + + api_key = _load_runpod_api_key() + if not api_key: + return "" + req = urllib.request.Request( + f"https://api.runpod.io/v2/pods/{pod_id}", + headers={ + "Authorization": f"Bearer {api_key}", + "Accept": "application/json", + "User-Agent": "test-images.py/1.0 (+runpod-smoketest)", + }, + ) + for attempt in range(1, attempts + 1): + try: + with urllib.request.urlopen(req, timeout=15) as resp: + data = json.loads(resp.read()) + except (OSError, ValueError): + data = None + cuda = (data or {}).get("cudaVersion") + if cuda: + return str(cuda).strip() + if attempt < attempts: + time.sleep(2) + return "" + + def cuda_check_command(image: str) -> str: """Return a shell command that functionally validates the GPU/CUDA stack for a given image, or '' to skip the check (CPU images). diff --git a/tests/runpod_smoke/pod.py b/tests/runpod_smoke/pod.py index 627c4c38..474be22d 100644 --- a/tests/runpod_smoke/pod.py +++ b/tests/runpod_smoke/pod.py @@ -262,34 +262,32 @@ def create_pod( # request a specific tier with this subcommand. else: args.extend(["--gpu-id", gpu_id, "--gpu-count", "1"]) - # Constrain scheduling to hosts whose driver supports this image's - # CUDA. Without this, RunPod may land a cu13.0 image on an - # older-driver host and the container fails at startup with - # `nvidia-container-cli: cuda>=13.0`. Image tag wins; the manifest - # `min_cuda_version` is only consulted for opaque tags (NGC etc.). + # Pin scheduling to hosts whose driver supports the image's CUDA, + # else a cu130 image dies with `nvidia-container-cli: cuda>=13.0`. + # An explicit manifest value wins; the tag is the fallback. tag_cuda = detect_cuda_version(image) manifest_cuda = ( config.GROUP_MIN_CUDA.get(group) if group else None ) - cuda_version = tag_cuda or manifest_cuda + cuda_version = manifest_cuda or tag_cuda if cuda_version: args.extend(["--min-cuda-version", cuda_version]) - # Emit a one-line trace of which source won, so reading the logs - # later (or chasing why scheduling picked a particular host) you - # can see whether the tag or the manifest fallback was used — - # and notice when a manifest value got ignored because the tag - # already had one. - if tag_cuda and manifest_cuda and tag_cuda != manifest_cuda: + if manifest_cuda and tag_cuda and manifest_cuda != tag_cuda: log( - f"min-cuda-version: tag='{tag_cuda}' wins over " - f"manifest='{manifest_cuda}' (tag is the source of truth " - "for image-encoded CUDA; manifest is fallback-only)", + f"min-cuda-version: requested '{manifest_cuda}' overrides " + f"tag-derived '{tag_cuda}'", + indent=1, + ) + elif tag_cuda and not manifest_cuda: + log( + f"min-cuda-version: none requested, derived '{tag_cuda}' " + "from the image tag", indent=1, ) elif manifest_cuda and not tag_cuda: log( - f"min-cuda-version: tag has none, using manifest " - f"fallback '{manifest_cuda}'", + f"min-cuda-version: requested '{manifest_cuda}' (tag has " + "no CUDA marker to derive from)", indent=1, ) if config.REGISTRY_AUTH_ID: diff --git a/tests/runpod_smoke/runner.py b/tests/runpod_smoke/runner.py index 13f79204..e47bb716 100644 --- a/tests/runpod_smoke/runner.py +++ b/tests/runpod_smoke/runner.py @@ -21,6 +21,7 @@ from .checks import ( cuda_check_command, dump_pod_logs, + fetch_pod_cuda_version, run_cuda_check, run_jupyter_check, run_jupyter_proxy_check, @@ -45,6 +46,22 @@ _Outcome = tuple[str, str] +# test_pair records the host's CUDA/driver here instead of returning it, so +# the ~15 outcome returns in that function keep their 2-tuple shape. +# test_image reads it on the same thread right after test_pair returns. +_thread_local = threading.local() + + +def _set_host_gpu(label: str) -> None: + _thread_local.host_gpu = label + + +def _take_host_gpu() -> str: + """Read and clear the label left by the last test_pair on this thread.""" + label = getattr(_thread_local, "host_gpu", "") or "" + _thread_local.host_gpu = "" + return label + def _log_attempt_header(image: str, instance: str, group: str) -> tuple[bool, str]: """Log the per-attempt header line and resolve the gpu_id. @@ -67,7 +84,8 @@ def _log_attempt_header(image: str, instance: str, group: str) -> tuple[bool, st ) return True, "" gpu_id = resolve_gpu_id(instance) - cuda = detect_cuda_version(image) or config.GROUP_MIN_CUDA.get(group) + # Same precedence as create_pod: explicit request wins, tag is fallback. + cuda = config.GROUP_MIN_CUDA.get(group) or detect_cuda_version(image) cuda_note = f", min-cuda={cuda}" if cuda else "" log( f"attempt: instance='{instance}' (--gpu-id '{gpu_id}'){cuda_note}", @@ -413,6 +431,9 @@ def test_pair(image: str, instance: str, group: str) -> _Outcome: `group` is the manifest section name (e.g. 'pytorch', 'base_gpu') and is used to select the appropriate GPU/CUDA functional check.""" + # Clear first so a label from a previous instance can't leak into an + # attempt that never reaches the probe (UNAVAILABLE, STUCK). + _set_host_gpu("") is_cpu, gpu_id = _log_attempt_header(image, instance, group) pod_id, early, early_detail = _create_pod_with_retries( @@ -441,6 +462,11 @@ def test_pair(image: str, instance: str, group: str) -> _Outcome: host = st.get("ssh_ip") or "" port = int(st.get("ssh_port") or 0) + host_cuda = fetch_pod_cuda_version(pod_id) + if host_cuda: + _set_host_gpu(f"CUDA {host_cuda}") + log(f"host CUDA: {host_cuda}", indent=2) + # Sequence the checks. Each returns None on pass/skip, or a FAIL # outcome to surface to the caller. Kept as straight-line code # (no fancy abstraction) so the failure points stay easy to read @@ -476,14 +502,17 @@ def test_pair(image: str, instance: str, group: str) -> _Outcome: def test_image( image: str, instances: list[str], group: str -) -> tuple[str, str, str]: - """Returns (status, note, instance_used). +) -> tuple[str, str, str, str]: + """Returns (status, note, instance_used, host_gpu). `instance_used` is the GPU display name that produced the terminal status. For PASS / FAIL it's the actual instance the test landed on. For SKIP (no capacity / all stuck), it's an empty string — the test never settled on any one instance. + `host_gpu` is the CUDA/driver the pod actually landed on, or '' when no + pod ever booted (SKIP) or the host isn't NVIDIA. + Iterates instance types until one PASSes. Stops early on FAIL (real image bug — no point trying another GPU). UNAVAILABLE (capacity) and STUCK (RunPod gave us a dead host) just move on to the next instance. @@ -501,13 +530,15 @@ def test_image( result, detail = test_pair(image, inst, group) finally: set_worker_context(None) + host_gpu = _take_host_gpu() if result == "PASS": - return "PASS", "", inst + return "PASS", "", inst, host_gpu if result == "FAIL": return ( "FAIL", detail or "container did not stay healthy", inst, + host_gpu, ) if result == "CREATE_FAIL": # Last create error is most informative — capacity-shortage 5xx @@ -525,7 +556,7 @@ def test_image( # We never got past pod-create on any instance and the errors # weren't capacity-shortages. Surface the last orchestrator error # — this is usually an image / auth / registry problem. - return "FAIL", last_create_error, last_create_inst + return "FAIL", last_create_error, last_create_inst, "" if stuck_instances: # We tried every instance and RunPod never gave us a working host # on any of them — surface that distinctly from "no capacity at @@ -543,10 +574,12 @@ def test_image( "scheduler issue, try again later" ), ", ".join(stuck_instances), + "", ) log(f"all {len(instances)} instances unavailable (no capacity)", indent=1) return ( "SKIP", f"no capacity on any of {len(instances)} candidate instance type(s)", ", ".join(unavailable_instances), + "", ) diff --git a/tests/test_images.py b/tests/test_images.py index 4dfb7387..e8e0042c 100755 --- a/tests/test_images.py +++ b/tests/test_images.py @@ -48,9 +48,9 @@ # entry; the runner iterates instances internally until something settles. Job = tuple[str, str, list[str]] -# Per-attempt outcome: (image, status, note, instance_used). A list avoids -# overwriting rows when `check_all_gpu` creates one job per GPU. -Result = tuple[str, str, str, str] +# Per-attempt outcome: (image, status, note, instance_used, host_gpu). A list +# avoids overwriting rows when `check_all_gpu` creates one job per GPU. +Result = tuple[str, str, str, str, str] # --------------------------------------------------------------------------- @@ -281,7 +281,7 @@ def _build_jobs( "'check_all_gpu:' produced candidates)" ) for img in contents.get("images", []): - results.append((img, "SKIP", "no instances configured", "")) + results.append((img, "SKIP", "no instances configured", "", "")) continue check_all = config.GROUP_CHECK_ALL_GPU.get(group, False) for img in contents.get("images", []): @@ -306,8 +306,8 @@ def _run_jobs_serial(jobs: list[Job], results: list[Result]) -> None: print() log(f"---------- group: {group} ----------") current_group = group - status, note, instance = test_image(img, instances, group) - results.append((img, status, note, instance)) + status, note, instance, host_gpu = test_image(img, instances, group) + results.append((img, status, note, instance, host_gpu)) def _run_one_tagged_job(job: Job) -> Result: @@ -317,9 +317,9 @@ def _run_one_tagged_job(job: Job) -> Result: img, grp, insts = job ensure_worker_tag() log(f"start [group={grp}] image={img}") - status, note, instance = test_image(img, insts, grp) + status, note, instance, host_gpu = test_image(img, insts, grp) log(f"done [group={grp}] image={img} -> {status}") - return img, status, note, instance + return img, status, note, instance, host_gpu def _run_jobs_parallel(jobs: list[Job], results: list[Result]) -> None: @@ -352,13 +352,18 @@ def _run_jobs(jobs: list[Job], results: list[Result]) -> None: def _format_result_line(want: str, img: str, status: str, note: str, - instance: str) -> Optional[str]: + instance: str, host_gpu: str = "") -> Optional[str]: """Format one row of the summary, or None when this result doesn't belong in the `want` bucket. CPU labels ('cpu-secure', 'cpu-community', - …) are already human-readable, so they go to the summary verbatim.""" + …) are already human-readable, so they go to the summary verbatim. + + `host_gpu` is the CUDA/driver the pod actually ran on — reported because + the image tag only sets a floor, so the tag alone doesn't tell you what + the run proved.""" if status != want: return None - inst_str = f" [{instance}]" if instance else "" + label = f"{instance} - {host_gpu}" if instance and host_gpu else instance + inst_str = f" [{label}]" if label else "" note_str = f" -- {note}" if note else "" return f" {want:6s} {img}{inst_str}{note_str}" @@ -387,7 +392,7 @@ def _print_summary(results: list[Result]) -> int: print(" SUMMARY ".center(84, "=")) print("=" * 84) counts: dict[str, int] = defaultdict(int) - for _img, status, _note, _instance in results: + for _img, status, _note, _instance, _host_gpu in results: counts[status] += 1 print( f"totals: {counts['PASS']} PASS, " @@ -395,8 +400,10 @@ def _print_summary(results: list[Result]) -> int: f"{counts['SKIP']} SKIP\n" ) for want in ("FAIL", "SKIP", "PASS"): - for img, status, note, instance in results: - line = _format_result_line(want, img, status, note, instance) + for img, status, note, instance, host_gpu in results: + line = _format_result_line( + want, img, status, note, instance, host_gpu + ) if line is not None: print(line) From dd52370cd2735b6d1a79040b45fe9a603d8cddad Mon Sep 17 00:00:00 2001 From: chmokachka Date: Fri, 28 Aug 2026 17:33:31 +0300 Subject: [PATCH 14/33] min-vram-gb: "0" --- .github/workflows/_tmp-gpu-compat-probe.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/_tmp-gpu-compat-probe.yml b/.github/workflows/_tmp-gpu-compat-probe.yml index efef79db..94a9bd7c 100644 --- a/.github/workflows/_tmp-gpu-compat-probe.yml +++ b/.github/workflows/_tmp-gpu-compat-probe.yml @@ -48,7 +48,7 @@ jobs: # instead of the full ~30-type catalog. check-all-gpu: "true" manufacturer: Nvidia - min-vram-gb: "80" + min-vram-gb: "0" max-parallel: "3" # Deliberately NOT set: the floor is derived from the tag (12.8), so From ae24458a8c5d30d4fbe055697e294aea66b587b3 Mon Sep 17 00:00:00 2001 From: chmokachka Date: Fri, 28 Aug 2026 18:09:59 +0300 Subject: [PATCH 15/33] fix: shellcheck --- official-templates/comfyui/scripts/start.sh | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/official-templates/comfyui/scripts/start.sh b/official-templates/comfyui/scripts/start.sh index e181a8ae..fb87c0a3 100755 --- a/official-templates/comfyui/scripts/start.sh +++ b/official-templates/comfyui/scripts/start.sh @@ -228,7 +228,7 @@ PY || true) if [ -n "$local_packages" ]; then echo " locally installed CUDA-sensitive packages:" - echo "$local_packages" | sed 's/^/ /' + echo " ${local_packages//$'\n'/$'\n' }" else echo " locally installed CUDA-sensitive packages: none" fi @@ -304,6 +304,8 @@ if [ -d "$OLD_VENV_DIR" ] && [ ! -d "$VENV_DIR" ]; then mv "$OLD_VENV_DIR" "${OLD_VENV_DIR}.bak" cd "$COMFYUI_DIR" python3.12 -m venv --system-site-packages "$VENV_DIR" + # The venv is created at runtime, so there is nothing for shellcheck to follow. + # shellcheck source=/dev/null source "$VENV_DIR/bin/activate" python -m ensurepip # Skip nodes baked into the image — their deps are in system site-packages @@ -342,6 +344,7 @@ if [ ! -d "$COMFYUI_DIR" ] || [ ! -d "$VENV_DIR" ]; then if [ ! -d "$VENV_DIR" ]; then cd "$COMFYUI_DIR" python3.12 -m venv --system-site-packages "$VENV_DIR" + # shellcheck source=/dev/null source "$VENV_DIR/bin/activate" # Ensure pip is available in the venv (needed for ComfyUI-Manager) @@ -352,7 +355,7 @@ if [ ! -d "$COMFYUI_DIR" ] || [ ! -d "$VENV_DIR" ]; then fi else # Just activate the existing venv - # shellcheck disable=SC1091 + # shellcheck source=/dev/null source "$VENV_DIR/bin/activate" echo "Using existing ComfyUI installation" fi From bc1d00ba69fbac6e3780dcafdf158889f5c93401 Mon Sep 17 00:00:00 2001 From: chmokachka Date: Fri, 28 Aug 2026 18:29:54 +0300 Subject: [PATCH 16/33] feat: compatibility summary --- .github/actions/smoke-test/action.yml | 32 ++++++ .github/workflows/_tmp-gpu-compat-probe.yml | 8 +- .github/workflows/gpu-compatibility.yml | 6 ++ tests/runpod_smoke/runner.py | 32 +++--- tests/test_images.py | 111 ++++++++++++++++++-- 5 files changed, 160 insertions(+), 29 deletions(-) diff --git a/.github/actions/smoke-test/action.yml b/.github/actions/smoke-test/action.yml index a6f9aacb..3c0af0a3 100644 --- a/.github/actions/smoke-test/action.yml +++ b/.github/actions/smoke-test/action.yml @@ -86,6 +86,23 @@ inputs: description: "Retention period for the generated ComfyUI images artifact." required: false default: "14" + upload-results-json: + description: | + Upload the result matrix (status / instance / host CUDA / note per + attempt) as a JSON artifact. The markdown table in the job's step + summary is always written and needs no flag; this is for diffing runs + against each other. Give each call a distinct results-artifact-name + when one run invokes this action more than once. + required: false + default: "false" + results-artifact-name: + description: "Artifact name used with upload-results-json." + required: false + default: "smoke-results" + results-retention-days: + description: "Retention period for the results JSON artifact." + required: false + default: "90" check-all-gpu: description: | Test every GPU matching the vendor/vRAM filters independently instead @@ -354,6 +371,9 @@ runs: # deadline for slow pulls (mainly multi-GB ROCm base images). CREATE_TIMEOUT: ${{ inputs.create-timeout }} SAVE_COMFYUI_IMAGES: ${{ inputs.save-comfyui-images }} + # test_images.py appends the matrix to $GITHUB_STEP_SUMMARY (always) + # and writes this JSON, so results are readable without log access. + SMOKE_RESULTS_JSON: ${{ runner.temp }}/smoke-results/results.json run: | set -uo pipefail if [[ "${SAVE_COMFYUI_IMAGES,,}" == "true" ]]; then @@ -362,6 +382,18 @@ runs: fi python3 "${GITHUB_WORKSPACE}/tests/test_images.py" "${MANIFEST_PATH}" + # Opt-in: upload-artifact rejects duplicate names within a run, and the + # matrix workflows call this action from several jobs. Enable it where you + # want to diff runs, passing a distinct results-artifact-name per job. + - name: Upload results JSON + if: ${{ always() && inputs.upload-results-json == 'true' }} + uses: actions/upload-artifact@v4 + with: + name: ${{ inputs.results-artifact-name }} + path: ${{ runner.temp }}/smoke-results/results.json + retention-days: ${{ inputs.results-retention-days }} + if-no-files-found: warn + - name: Upload generated ComfyUI images if: ${{ always() && inputs.save-comfyui-images == 'true' }} uses: actions/upload-artifact@v4 diff --git a/.github/workflows/_tmp-gpu-compat-probe.yml b/.github/workflows/_tmp-gpu-compat-probe.yml index 94a9bd7c..aa7c0a41 100644 --- a/.github/workflows/_tmp-gpu-compat-probe.yml +++ b/.github/workflows/_tmp-gpu-compat-probe.yml @@ -39,18 +39,20 @@ jobs: - name: GPU compat probe uses: ./.github/actions/smoke-test with: - image-refs: '["docker.io/runpod/pytorch:1.2.0-rc.162-cu1281-torch2121-ubuntu2404"]' + image-refs: '["docker.io/runpod/pytorch:1.2.0-rc.162-cu1300-torch260-ubuntu2404"]' profile: gpu runpod-api-key: ${{ secrets.TESTING_RUNPOD_API_KEY }} ssh-private-key: ${{ secrets.TESTING_RUNPOD_SSH_PRIVATE_KEY }} - # One pod per GPU, bounded to >=80GB cards so this stays ~7 pods - # instead of the full ~30-type catalog. + # min-vram-gb 0 = the whole NVIDIA catalog (~46 attempts). check-all-gpu: "true" manufacturer: Nvidia min-vram-gb: "0" max-parallel: "3" + upload-results-json: "true" + results-artifact-name: gpu-matrix-${{ github.run_id }} + # Deliberately NOT set: the floor is derived from the tag (12.8), so # the summary shows whether the host was actually newer. # min-cuda-version: diff --git a/.github/workflows/gpu-compatibility.yml b/.github/workflows/gpu-compatibility.yml index 45b73dcd..c52107a3 100644 --- a/.github/workflows/gpu-compatibility.yml +++ b/.github/workflows/gpu-compatibility.yml @@ -209,6 +209,12 @@ jobs: save-comfyui-images: ${{ inputs.test-comfyui-functional }} comfyui-images-artifact-name: comfyui-images-gpu-matrix-${{ github.run_id }} + # The matrix goes to the job's step summary either way; the JSON + # artifact is what lets you diff sweeps and spot the fleet moving + # to a new CUDA. + upload-results-json: "true" + results-artifact-name: gpu-matrix-${{ github.run_id }} + # A catalog-wide sweep always hits GPUs with no free capacity. # Those are SKIPs, not defects; real FAILs stay fatal. on-skip: pass diff --git a/tests/runpod_smoke/runner.py b/tests/runpod_smoke/runner.py index e47bb716..43ee8ee6 100644 --- a/tests/runpod_smoke/runner.py +++ b/tests/runpod_smoke/runner.py @@ -46,21 +46,23 @@ _Outcome = tuple[str, str] -# test_pair records the host's CUDA/driver here instead of returning it, so +# test_pair records the host's CUDA version here instead of returning it, so # the ~15 outcome returns in that function keep their 2-tuple shape. # test_image reads it on the same thread right after test_pair returns. +# Stored bare ('13.0') — the 'CUDA ' prefix is added at render time so the +# JSON report can carry the raw value. _thread_local = threading.local() -def _set_host_gpu(label: str) -> None: - _thread_local.host_gpu = label +def _set_host_cuda(version: str) -> None: + _thread_local.host_cuda = version -def _take_host_gpu() -> str: - """Read and clear the label left by the last test_pair on this thread.""" - label = getattr(_thread_local, "host_gpu", "") or "" - _thread_local.host_gpu = "" - return label +def _take_host_cuda() -> str: + """Read and clear the version left by the last test_pair on this thread.""" + version = getattr(_thread_local, "host_cuda", "") or "" + _thread_local.host_cuda = "" + return version def _log_attempt_header(image: str, instance: str, group: str) -> tuple[bool, str]: @@ -433,7 +435,7 @@ def test_pair(image: str, instance: str, group: str) -> _Outcome: is used to select the appropriate GPU/CUDA functional check.""" # Clear first so a label from a previous instance can't leak into an # attempt that never reaches the probe (UNAVAILABLE, STUCK). - _set_host_gpu("") + _set_host_cuda("") is_cpu, gpu_id = _log_attempt_header(image, instance, group) pod_id, early, early_detail = _create_pod_with_retries( @@ -464,7 +466,7 @@ def test_pair(image: str, instance: str, group: str) -> _Outcome: host_cuda = fetch_pod_cuda_version(pod_id) if host_cuda: - _set_host_gpu(f"CUDA {host_cuda}") + _set_host_cuda(host_cuda) log(f"host CUDA: {host_cuda}", indent=2) # Sequence the checks. Each returns None on pass/skip, or a FAIL @@ -503,14 +505,14 @@ def test_pair(image: str, instance: str, group: str) -> _Outcome: def test_image( image: str, instances: list[str], group: str ) -> tuple[str, str, str, str]: - """Returns (status, note, instance_used, host_gpu). + """Returns (status, note, instance_used, host_cuda). `instance_used` is the GPU display name that produced the terminal status. For PASS / FAIL it's the actual instance the test landed on. For SKIP (no capacity / all stuck), it's an empty string — the test never settled on any one instance. - `host_gpu` is the CUDA/driver the pod actually landed on, or '' when no + `host_cuda` is the CUDA version the pod actually landed on, or '' when no pod ever booted (SKIP) or the host isn't NVIDIA. Iterates instance types until one PASSes. Stops early on FAIL (real @@ -530,15 +532,15 @@ def test_image( result, detail = test_pair(image, inst, group) finally: set_worker_context(None) - host_gpu = _take_host_gpu() + host_cuda = _take_host_cuda() if result == "PASS": - return "PASS", "", inst, host_gpu + return "PASS", "", inst, host_cuda if result == "FAIL": return ( "FAIL", detail or "container did not stay healthy", inst, - host_gpu, + host_cuda, ) if result == "CREATE_FAIL": # Last create error is most informative — capacity-shortage 5xx diff --git a/tests/test_images.py b/tests/test_images.py index e8e0042c..91a8b251 100755 --- a/tests/test_images.py +++ b/tests/test_images.py @@ -13,10 +13,12 @@ from __future__ import annotations +import json import os import sys from collections import defaultdict from concurrent.futures import ThreadPoolExecutor, as_completed +from datetime import datetime, timezone from pathlib import Path from typing import Optional @@ -48,7 +50,7 @@ # entry; the runner iterates instances internally until something settles. Job = tuple[str, str, list[str]] -# Per-attempt outcome: (image, status, note, instance_used, host_gpu). A list +# Per-attempt outcome: (image, status, note, instance_used, host_cuda). A list # avoids overwriting rows when `check_all_gpu` creates one job per GPU. Result = tuple[str, str, str, str, str] @@ -306,8 +308,8 @@ def _run_jobs_serial(jobs: list[Job], results: list[Result]) -> None: print() log(f"---------- group: {group} ----------") current_group = group - status, note, instance, host_gpu = test_image(img, instances, group) - results.append((img, status, note, instance, host_gpu)) + status, note, instance, host_cuda = test_image(img, instances, group) + results.append((img, status, note, instance, host_cuda)) def _run_one_tagged_job(job: Job) -> Result: @@ -317,9 +319,9 @@ def _run_one_tagged_job(job: Job) -> Result: img, grp, insts = job ensure_worker_tag() log(f"start [group={grp}] image={img}") - status, note, instance, host_gpu = test_image(img, insts, grp) + status, note, instance, host_cuda = test_image(img, insts, grp) log(f"done [group={grp}] image={img} -> {status}") - return img, status, note, instance, host_gpu + return img, status, note, instance, host_cuda def _run_jobs_parallel(jobs: list[Job], results: list[Result]) -> None: @@ -352,22 +354,106 @@ def _run_jobs(jobs: list[Job], results: list[Result]) -> None: def _format_result_line(want: str, img: str, status: str, note: str, - instance: str, host_gpu: str = "") -> Optional[str]: + instance: str, host_cuda: str = "") -> Optional[str]: """Format one row of the summary, or None when this result doesn't belong in the `want` bucket. CPU labels ('cpu-secure', 'cpu-community', …) are already human-readable, so they go to the summary verbatim. - `host_gpu` is the CUDA/driver the pod actually ran on — reported because + `host_cuda` is the CUDA version the pod actually ran on — reported because the image tag only sets a floor, so the tag alone doesn't tell you what the run proved.""" if status != want: return None - label = f"{instance} - {host_gpu}" if instance and host_gpu else instance + label = f"{instance} - CUDA {host_cuda}" if instance and host_cuda else instance inst_str = f" [{label}]" if label else "" note_str = f" -- {note}" if note else "" return f" {want:6s} {img}{inst_str}{note_str}" +_STATUS_ICON = {"PASS": "✅ PASS", "FAIL": "❌ FAIL", "SKIP": "⚠️ SKIP"} + + +def _md_cell(value: str) -> str: + """Escape a value for a markdown table cell.""" + return (value or "").replace("|", "\\|").replace("\n", " ") or "—" + + +def _emit_step_summary(results: list[Result], counts: dict[str, int]) -> None: + """Append the matrix to $GITHUB_STEP_SUMMARY as a markdown table. + + The stdout matrix is only reachable by downloading the job log, which + needs repo admin. The step summary renders on the run page for anyone. + No-op outside Actions; never fatal. + """ + path = os.environ.get("GITHUB_STEP_SUMMARY") + if not path: + return + images = {r[0] for r in results} + single = next(iter(images)) if len(images) == 1 else "" + head = ["Status", "Instance", "CUDA", "Note"] + if not single: + head.insert(1, "Image") + lines = [ + "## Smoke-test matrix", + "", + f"**{counts['PASS']} PASS · {counts['FAIL']} FAIL · {counts['SKIP']} SKIP**", + "", + ] + if single: + lines += [f"Image: `{single}`", ""] + lines.append("| " + " | ".join(head) + " |") + lines.append("|" + "|".join(["---"] * len(head)) + "|") + for want in ("FAIL", "SKIP", "PASS"): + for img, status, note, instance, host_cuda in results: + if status != want: + continue + row = [ + _STATUS_ICON.get(status, status), + _md_cell(instance), + _md_cell(host_cuda), + _md_cell(note), + ] + if not single: + row.insert(1, f"`{img}`") + lines.append("| " + " | ".join(row) + " |") + try: + with open(path, "a", encoding="utf-8") as fh: + fh.write("\n".join(lines) + "\n") + except OSError as exc: + log(f"warn: could not write step summary: {exc}") + + +def _write_results_json(results: list[Result], counts: dict[str, int]) -> None: + """Write the matrix to $SMOKE_RESULTS_JSON so CI can keep it as an + artifact and diff runs against each other. No-op when unset.""" + path = os.environ.get("SMOKE_RESULTS_JSON") + if not path: + return + payload = { + "generated_at": datetime.now(timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ"), + "totals": {k: counts[k] for k in ("PASS", "FAIL", "SKIP")}, + "results": [ + { + "image": img, + "status": status, + "instance": instance, + "cuda": host_cuda, + "note": note, + } + for img, status, note, instance, host_cuda in results + ], + } + try: + parent = os.path.dirname(path) + if parent: + os.makedirs(parent, exist_ok=True) + with open(path, "w", encoding="utf-8") as fh: + json.dump(payload, fh, indent=2) + log(f"wrote results JSON -> {path}") + except OSError as exc: + log(f"warn: could not write results JSON: {exc}") + + def _print_summary(results: list[Result]) -> int: """Print the SUMMARY block and return the exit code. @@ -392,7 +478,7 @@ def _print_summary(results: list[Result]) -> int: print(" SUMMARY ".center(84, "=")) print("=" * 84) counts: dict[str, int] = defaultdict(int) - for _img, status, _note, _instance, _host_gpu in results: + for _img, status, _note, _instance, _host_cuda in results: counts[status] += 1 print( f"totals: {counts['PASS']} PASS, " @@ -400,13 +486,16 @@ def _print_summary(results: list[Result]) -> int: f"{counts['SKIP']} SKIP\n" ) for want in ("FAIL", "SKIP", "PASS"): - for img, status, note, instance, host_gpu in results: + for img, status, note, instance, host_cuda in results: line = _format_result_line( - want, img, status, note, instance, host_gpu + want, img, status, note, instance, host_cuda ) if line is not None: print(line) + _emit_step_summary(results, counts) + _write_results_json(results, counts) + if counts["FAIL"] > 0: return 1 if counts["SKIP"] == 0 or config.ON_SKIP == "pass": From ccc4419bdba6dbfb76242b162eeb2d7769449459 Mon Sep 17 00:00:00 2001 From: chmokachka Date: Mon, 31 Aug 2026 13:56:40 +0300 Subject: [PATCH 17/33] feat: migration to apiv2 --- .github/actions/smoke-test/action.yml | 75 ++-- .github/scripts/generate_test_manifest.py | 47 ++- .github/workflows/_tmp-gpu-compat-probe.yml | 48 ++- .github/workflows/gpu-compatibility.yml | 28 +- tests/runpod_smoke/api.py | 234 +++++++++++ tests/runpod_smoke/checks.py | 85 ++-- tests/runpod_smoke/config.py | 95 +++-- tests/runpod_smoke/instances.py | 240 ++++++++---- tests/runpod_smoke/manifest.py | 2 +- tests/runpod_smoke/pod.py | 413 +++++++++----------- tests/runpod_smoke/runner.py | 37 +- tests/runpod_smoke/runpodctl.py | 41 -- tests/test_images.py | 233 +++++++++-- 13 files changed, 1017 insertions(+), 561 deletions(-) create mode 100644 tests/runpod_smoke/api.py delete mode 100644 tests/runpod_smoke/runpodctl.py diff --git a/.github/actions/smoke-test/action.yml b/.github/actions/smoke-test/action.yml index 3c0af0a3..44849232 100644 --- a/.github/actions/smoke-test/action.yml +++ b/.github/actions/smoke-test/action.yml @@ -103,6 +103,22 @@ inputs: description: "Retention period for the results JSON artifact." required: false default: "90" + cuda-versions: + description: | + Turn on the GPU x CUDA axis: test each candidate GPU once per CUDA + version instead of once overall, pinning the host with + `gpu.allowedCudaVersions`. Newline- or comma-separated X.Y versions, + or the literal `all` for every version each GPU reports capacity for. + + Only pairings the catalog reports capacity for are attempted — + matching is exact, so a version no machine reports yields a capacity + error rather than a fallback. Supersedes min-cuda-version, which the + API rejects alongside it. + + Empty (default) = no axis; one job per GPU and the floor comes from + the image tag, exactly as before. + required: false + default: "" check-all-gpu: description: | Test every GPU matching the vendor/vRAM filters independently instead @@ -128,7 +144,7 @@ inputs: Floor CUDA driver version that the pod host's driver must support (e.g. '13.0'). Emitted as `min_cuda_version` on every produced manifest group. test_images.py forwards this to - `runpodctl --min-cuda-version` for images whose tag has no embedded + `gpu.minCudaVersion` for images whose tag has no embedded CUDA marker — NGC nvidia-pytorch:25.11 and similar opaque tags. Use this for NGC PyTorch which ships torch built against CUDA 13.0 and refuses to run on hosts with a 12.x driver @@ -178,47 +194,24 @@ inputs: flips the safer default off. required: false default: "fail" - runpodctl-version: - description: "runpodctl release tag" - required: false - default: "v2.3.0" - runpodctl-sha256: - description: "SHA-256 of runpodctl-linux-amd64.tar.gz for the pinned version" - required: false - default: "908f2210571e8a26a1cba6fb45f09556b34dcad3e1b20dd502df2adf7a57c169" runs: using: composite steps: - - name: Install runpodctl - shell: bash - env: - VERSION: ${{ inputs.runpodctl-version }} - SHA256: ${{ inputs.runpodctl-sha256 }} - run: | - set -euo pipefail - curl -fsSL --proto '=https' --tlsv1.2 \ - "https://github.com/runpod/runpodctl/releases/download/${VERSION}/runpodctl-linux-amd64.tar.gz" \ - -o /tmp/runpodctl.tar.gz - echo "${SHA256} /tmp/runpodctl.tar.gz" | sha256sum -c - - sudo tar -xz -C /usr/local/bin -f /tmp/runpodctl.tar.gz runpodctl - rm /tmp/runpodctl.tar.gz - runpodctl version - - - name: Configure runpodctl + - name: Configure RunPod credentials shell: bash env: - # `runpodctl config --apiKey` is deprecated. The CLI (and our - # tests/test_images.py) both read RUNPOD_API_KEY out of the env, - # so we just propagate the secret through $GITHUB_ENV — every - # subsequent step in this composite action gets it automatically. - # GitHub keeps the secret masked in logs because the value still - # matches `secrets.RUNPOD_API_KEY`. + # tests/test_images.py reads RUNPOD_API_KEY out of the env, so the + # secret is propagated through $GITHUB_ENV and every subsequent step + # in this composite action gets it. GitHub keeps it masked in logs + # because the value still matches `secrets.RUNPOD_API_KEY`. RUNPOD_API_KEY: ${{ inputs.runpod-api-key }} run: | set -euo pipefail - # Validate auth before we attempt any pod operations — fail loud, fast. - runpodctl user >/dev/null + if [[ -z "${RUNPOD_API_KEY}" ]]; then + echo "::error::runpod-api-key input is empty" + exit 1 + fi echo "RUNPOD_API_KEY=${RUNPOD_API_KEY}" >> "${GITHUB_ENV}" - name: Write SSH private key @@ -301,6 +294,7 @@ runs: CHECK_ALL_GPU: ${{ inputs.check-all-gpu }} EXCLUDE_INSTANCES: ${{ inputs.exclude-instances }} MIN_CUDA_VERSION: ${{ inputs.min-cuda-version }} + CUDA_VERSIONS: ${{ inputs.cuda-versions }} run: | set -euo pipefail # Only the literal string "true" turns the flag on; anything else @@ -337,6 +331,19 @@ runs: if [[ -n "${MIN_CUDA_VERSION}" ]]; then EXTRA_ARGS+=(--min-cuda-version "${MIN_CUDA_VERSION}") fi + # Split CUDA_VERSIONS on commas AND newlines into one + # --cuda-version per entry. Accepts '12.8, 13.0', a newline list, or + # the literal 'all'. + while IFS= read -r line; do + line="${line#"${line%%[![:space:]]*}"}" + line="${line%"${line##*[![:space:]]}"}" + [[ -z "${line}" || "${line}" == \#* ]] && continue + line="${line#-}" + line="${line#"${line%%[![:space:]]*}"}" + line="${line%"${line##*[![:space:]]}"}" + [[ -z "${line}" ]] && continue + EXTRA_ARGS+=(--cuda-version "${line}") + done <<< "$(printf '%s' "${CUDA_VERSIONS}" | tr ',' '\n')" # Split EXCLUDE_INSTANCES on newlines into one --exclude-instance # per non-empty, non-whitespace, non-comment line. Each item is # passed as a single argv element so glob chars (*, ?) stay literal @@ -362,7 +369,7 @@ runs: shell: bash env: # RUNPOD_API_KEY and RUNPOD_SSH_KEY come in via $GITHUB_ENV from the - # 'Configure runpodctl' and 'Write SSH private key' steps above. + # 'Configure RunPod credentials' and 'Write SSH private key' steps. MAX_PARALLEL: ${{ inputs.max-parallel }} # ON_SKIP is validated by config._coerce_on_skip (unknown values # collapse to 'fail') so we just propagate the raw input. diff --git a/.github/scripts/generate_test_manifest.py b/.github/scripts/generate_test_manifest.py index 80ab7a03..5c66e436 100755 --- a/.github/scripts/generate_test_manifest.py +++ b/.github/scripts/generate_test_manifest.py @@ -91,6 +91,14 @@ def render_yaml(groups: dict) -> str: lines.append(" test_ports:") for port in body["test_ports"]: lines.append(f" - {port}") + # cuda_versions is either the literal `all` or a list of X.Y + # versions; the scalar form goes through the key loop above. + if isinstance(body.get("cuda_versions"), list): + lines.append(" cuda_versions:") + for version in body["cuda_versions"]: + lines.append(f" - {version}") + elif body.get("cuda_versions"): + lines.append(f" cuda_versions: {body['cuda_versions']}") # exclude_instances is a list, emitted at the bottom of the group so # it's visually grouped with other "filter" options. Patterns are # double-quoted to keep glob-leading characters ('*', '?') safe from @@ -116,6 +124,7 @@ def build_groups( check_all_gpu: bool = False, exclude_instances: list[str] | None = None, min_cuda_version: str | None = None, + cuda_versions: list[str] | None = None, ) -> dict: """Build the manifest dict for `profile`. @@ -134,6 +143,12 @@ def build_groups( landing on a Blackwell host fails with 'no kernel image is available for execution on the device'. + `cuda_versions` turns on the GPU x CUDA axis: `['all']` tests every + version each GPU reports capacity for, an explicit list tests just + those. Each version becomes its own job and is pinned with + `gpu.allowedCudaVersions`, so it supersedes `min_cuda_version` — the + API rejects both fields on one request. + `min_cuda_version` is the floor CUDA driver version (X.Y) the pod's host driver must support. test_images.py only consults this for images whose tag has no embedded CUDA marker (NGC nvidia-pytorch:25.11 @@ -143,6 +158,8 @@ def build_groups( """ exclude_instances = list(exclude_instances or []) test_ports = list(test_ports or []) + cuda_versions = list(cuda_versions or []) + wants_all_cuda = any(v.strip().lower() == "all" for v in cuda_versions) def _decorate(body: dict, *, gpu_group: bool) -> dict: if gpu_group: @@ -162,16 +179,21 @@ def _decorate(body: dict, *, gpu_group: bool) -> dict: body["test_comfyui_functional"] = True if exclude_instances: body["exclude_instances"] = list(exclude_instances) - if min_cuda_version: + if gpu_group and wants_all_cuda: + body["cuda_versions"] = "all" + elif gpu_group and cuda_versions: + body["cuda_versions"] = list(cuda_versions) + # The floor is meaningless once exact versions are pinned, and + # sending both makes the API reject the create outright. + if min_cuda_version and not (gpu_group and (wants_all_cuda or cuda_versions)): body["min_cuda_version"] = min_cuda_version return body if profile == "base": # Split refs into CPU- vs GPU-targeted images by tag content. - # CPU images: tested via runpodctl --compute-type CPU. RunPod selects - # the CPU flavor for us — runpodctl 2.3.0 doesn't expose --gpu-id - # for CPU, so we can't (and don't) constrain the manifest with an - # `instances:` or `max_price_per_hour:` field for CPU groups. + # CPU images: the harness picks a CPU flavor from + # GET /v2/catalog/cpus, so CPU groups carry no `instances:` or + # `max_price_per_hour:` field. # GPU images: tested with the normal --gpu-id flow and budget filter. cpu = [r for r in refs if not is_gpu_ref(r)] gpu = [r for r in refs if is_gpu_ref(r)] @@ -271,6 +293,20 @@ def main() -> int: "Empty (default) = no floor." ), ) + ap.add_argument( + "--cuda-version", + action="append", + default=[], + dest="cuda_versions", + metavar="X.Y|all", + help=( + "Turn on the GPU x CUDA axis: test each candidate GPU once per " + "CUDA version instead of once overall. Repeat for several " + "versions, or pass 'all' to use every version each GPU reports " + "capacity for. Versions are pinned exactly via " + "gpu.allowedCudaVersions, so this supersedes --min-cuda-version." + ), + ) ap.add_argument("--output", required=True, type=Path) args = ap.parse_args() @@ -297,6 +333,7 @@ def main() -> int: check_all_gpu=args.check_all_gpu, exclude_instances=args.exclude_instance, min_cuda_version=(args.min_cuda_version or None), + cuda_versions=args.cuda_versions, ) if not groups: diff --git a/.github/workflows/_tmp-gpu-compat-probe.yml b/.github/workflows/_tmp-gpu-compat-probe.yml index aa7c0a41..6d0f0e47 100644 --- a/.github/workflows/_tmp-gpu-compat-probe.yml +++ b/.github/workflows/_tmp-gpu-compat-probe.yml @@ -3,25 +3,34 @@ name: TEMP GPU compat probe # ============================================================================ # TEMPORARY — DELETE BEFORE MERGING THIS PR. # -# workflow_dispatch only appears in the UI once the file is on the default -# branch, so gpu-compatibility.yml can't be run by hand yet. This fires on the -# PR instead, purely to confirm the new `cudaVersion` column comes back -# populated and to see whether a cu1281 image actually lands on 12.8. +# Verifies the new pieces end-to-end in CI: +# * the CUDA axis (`cuda-versions`) expanding into per-version jobs +# * `gpu.allowedCudaVersions` pinning the host to an exact version +# * the GPU x CUDA pivot in the job's step summary +# * the results JSON artifact carrying `requested_cuda` +# * the whole pod lifecycle on REST API v2 (create / get / delete) # -# Every push matching `paths` below spends real GPU money. +# Deliberately the CHEAP mode: no check-all-gpu, so the axis produces one +# job per CUDA version rather than a full GPU x CUDA matrix, and each job +# short-circuits on the first card that passes. With the budget filter the +# candidate list is cheapest-first, so this lands on ~$0.24/hr cards — +# roughly $0.08 for the run instead of the ~32 pods a matrix would spend. # ============================================================================ on: pull_request: paths: - '.github/workflows/_tmp-gpu-compat-probe.yml' + - '.github/actions/smoke-test/**' + - '.github/scripts/generate_test_manifest.py' - 'tests/**' permissions: contents: read -# Queue rather than cancel: a cancel SIGKILLs the runner before test_images.py -# can delete its pods. +# Queue rather than cancel: a cancel SIGKILLs the runner before +# test_images.py can delete its pods, and v2 has no server-side +# terminate-after to fall back on. concurrency: group: tmp-gpu-compat-probe cancel-in-progress: false @@ -29,7 +38,7 @@ concurrency: jobs: probe: runs-on: blacksmith-4vcpu-ubuntu-2404 - timeout-minutes: 240 + timeout-minutes: 120 steps: - name: Checkout uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 @@ -39,24 +48,27 @@ jobs: - name: GPU compat probe uses: ./.github/actions/smoke-test with: - image-refs: '["docker.io/runpod/pytorch:1.2.0-rc.162-cu1300-torch260-ubuntu2404"]' + # torch 2.12 supports every current architecture, so a FAIL here + # means our plumbing, not the image. + image-refs: '["docker.io/runpod/pytorch:1.2.0-rc.162-cu1281-torch2121-ubuntu2404"]' profile: gpu runpod-api-key: ${{ secrets.TESTING_RUNPOD_API_KEY }} ssh-private-key: ${{ secrets.TESTING_RUNPOD_SSH_PRIVATE_KEY }} - # min-vram-gb 0 = the whole NVIDIA catalog (~46 attempts). - check-all-gpu: "true" + # THE THING UNDER TEST. Two versions -> two jobs. The tag says + # cu1281, so pinning 13.0 also proves the pin beats the + # tag-derived floor. + cuda-versions: "12.8, 13.0" + + # No check-all-gpu on purpose — see the header. + budget-usd-per-hour: "1.0" manufacturer: Nvidia - min-vram-gb: "0" - max-parallel: "3" + min-vram-gb: "16" + max-parallel: "2" upload-results-json: "true" results-artifact-name: gpu-matrix-${{ github.run_id }} - # Deliberately NOT set: the floor is derived from the tag (12.8), so - # the summary shows whether the host was actually newer. - # min-cuda-version: - test-jupyter: "true" on-skip: pass - create-timeout: "1200" + create-timeout: "600" diff --git a/.github/workflows/gpu-compatibility.yml b/.github/workflows/gpu-compatibility.yml index c52107a3..db87a862 100644 --- a/.github/workflows/gpu-compatibility.yml +++ b/.github/workflows/gpu-compatibility.yml @@ -18,8 +18,8 @@ on: type: string required: false default: "" - cuda-version: - description: "CUDA the host driver must support, X.Y (e.g. 13.0). Empty = derive from the image tag." + cuda-versions: + description: "CUDA axis: 'all', or exact versions like '12.8, 13.0'. Empty = one job per GPU, floor from the tag." type: string required: false default: "" @@ -130,17 +130,22 @@ jobs: env: MIN_VRAM: ${{ inputs.min-vram-gb }} MAX_PARALLEL: ${{ inputs.max-parallel }} - CUDA: ${{ inputs.cuda-version }} + CUDA: ${{ inputs.cuda-versions }} run: | set -euo pipefail - # Fail before the runpodctl install and catalog fetch, not with an - # int()/float() traceback deep in the harness. + # Fail before the catalog fetch, not with an int()/float() + # traceback deep in the harness. [[ "${MIN_VRAM}" =~ ^[0-9]+$ ]] \ || { echo "::error::min-vram-gb must be a whole number, got '${MIN_VRAM}'"; exit 1; } [[ "${MAX_PARALLEL}" =~ ^[1-9][0-9]*$ ]] \ || { echo "::error::max-parallel must be a positive integer, got '${MAX_PARALLEL}'"; exit 1; } - [[ -z "${CUDA}" || "${CUDA}" =~ ^[0-9]+\.[0-9]+$ ]] \ - || { echo "::error::cuda-version must be X.Y (e.g. 13.0) or empty, got '${CUDA}'"; exit 1; } + # 'all', or a comma list of X.Y. Rejecting a bare major here saves + # a whole sweep that would pin a version nothing reports. + if [[ -n "${CUDA}" && "${CUDA,,}" != "all" ]]; then + normalized=$(printf '%s' "${CUDA}" | tr -d '[:space:]') + [[ "${normalized}" =~ ^[0-9]+\.[0-9]+(,[0-9]+\.[0-9]+)*$ ]] \ + || { echo "::error::cuda-versions must be 'all' or X.Y[,X.Y...] (e.g. '12.8, 13.0'), got '${CUDA}'"; exit 1; } + fi - name: Summarize run parameters shell: bash @@ -150,7 +155,7 @@ jobs: IMAGE: ${{ steps.prep.outputs.image }} PORTS: ${{ steps.prep.outputs.ports }} EXCLUDE: ${{ steps.prep.outputs.exclude }} - CUDA: ${{ inputs.cuda-version }} + CUDA: ${{ inputs.cuda-versions }} VENDOR: ${{ inputs.manufacturer }} MIN_VRAM: ${{ inputs.min-vram-gb }} MAX_PARALLEL: ${{ inputs.max-parallel }} @@ -169,7 +174,7 @@ jobs: echo "| Setting | Value |" echo "| --- | --- |" echo "| Image | \`${IMAGE}\` |" - echo "| CUDA floor | ${CUDA:-derived from tag} |" + echo "| CUDA axis | ${CUDA:-none (floor from tag)} |" echo "| Vendor | ${VENDOR} |" echo "| Min vRAM | ${MIN_VRAM} GB |" echo "| Ports | ${ports_1l:-none} |" @@ -196,8 +201,9 @@ jobs: min-vram-gb: ${{ inputs.min-vram-gb }} exclude-instances: ${{ steps.prep.outputs.exclude }} - # Empty = derived from the tag by instances.detect_cuda_version. - min-cuda-version: ${{ inputs.cuda-version }} + # Empty = no axis; one job per GPU with the floor derived from the + # image tag by instances.detect_cuda_version. + cuda-versions: ${{ inputs.cuda-versions }} test-ports: ${{ steps.prep.outputs.ports }} test-jupyter: ${{ inputs.test-jupyter }} diff --git a/tests/runpod_smoke/api.py b/tests/runpod_smoke/api.py new file mode 100644 index 00000000..0aea5000 --- /dev/null +++ b/tests/runpod_smoke/api.py @@ -0,0 +1,234 @@ +"""RunPod REST API v2 client. + +Replaces the `runpodctl` subprocess calls. One place for the API key, +request plumbing and the mapping from HTTP status + `ErrorResponse` onto +the outcome vocabulary the runner speaks. + +Every helper returns `(status, data)` instead of raising, matching the +style in comfyui.py: transport failures come back as status 0 so callers +classify them like any other error. +""" + +from __future__ import annotations + +import json +import os +import re +import time +import urllib.error +import urllib.parse +import urllib.request +from pathlib import Path +from typing import Optional + +from .log import log + + +BASE_URL = "https://api.runpod.io/v2" + +# api.runpod.io sits behind Cloudflare, which rejects the default +# Python-urllib User-Agent with error 1010. +_UA = "test-images.py/1.0 (+runpod-smoketest)" + +# Status 0 is our own marker for "the request never got an HTTP reply" +# (DNS, connection reset, socket timeout). +_TRANSPORT_ERROR = 0 + + +def load_api_key() -> Optional[str]: + """Read the API key from RUNPOD_API_KEY, else ~/.runpod/config.toml. + + The file is regex'd rather than parsed so we don't need tomli — the + CLI always writes the key on a single line as `apikey = '...'`. + """ + env = os.environ.get("RUNPOD_API_KEY", "").strip() + if env: + return env + cfg = Path.home() / ".runpod" / "config.toml" + if not cfg.is_file(): + return None + try: + text = cfg.read_text() + except OSError: + return None + m = re.search(r"apikey\s*=\s*['\"]([^'\"]+)['\"]", text) + return m.group(1) if m else None + + +def request( + method: str, + path: str, + *, + body: Optional[dict] = None, + params: Optional[dict] = None, + timeout: int = 60, +) -> tuple[int, object]: + """Call the v2 API. Returns `(status, parsed_body)`. + + status 0 means the request never reached the API. `parsed_body` is the + decoded JSON, or None for 204 / non-JSON / transport failures. + """ + api_key = load_api_key() + if not api_key: + return _TRANSPORT_ERROR, {"detail": "no RunPod API key available"} + + url = f"{BASE_URL}{path}" + if params: + # Repeated keys would be wrong here: the v2 API takes multi-valued + # query params (include, product, cudaVersions) comma-separated. + flat = { + k: ",".join(str(x) for x in v) if isinstance(v, (list, tuple)) else v + for k, v in params.items() + if v not in (None, "", [], ()) + } + if flat: + url += "?" + urllib.parse.urlencode(flat) + + data = json.dumps(body).encode() if body is not None else None + headers = { + "Authorization": f"Bearer {api_key}", + "Accept": "application/json", + "User-Agent": _UA, + } + if data is not None: + headers["Content-Type"] = "application/json" + + req = urllib.request.Request(url, data=data, headers=headers, method=method) + try: + with urllib.request.urlopen(req, timeout=timeout) as resp: + raw = resp.read() + return resp.status, (json.loads(raw) if raw else None) + except urllib.error.HTTPError as exc: + raw = b"" + try: + raw = exc.read() + except Exception: # noqa: BLE001 — body is best-effort + pass + try: + return exc.code, json.loads(raw) if raw else None + except ValueError: + return exc.code, {"detail": raw.decode("utf-8", "replace")[:500]} + except (OSError, ValueError) as exc: + # urllib.error.URLError and TimeoutError both derive from OSError. + return _TRANSPORT_ERROR, {"detail": f"{type(exc).__name__}: {exc}"} + + +def error_detail(status: int, data: object) -> str: + """Flatten an ErrorResponse into one line for logs and FAIL notes.""" + if not isinstance(data, dict): + return f"HTTP {status}" + parts = [str(data.get("detail") or data.get("title") or "").strip()] + errors = data.get("errors") + if isinstance(errors, list) and errors: + parts.append("; ".join(str(e) for e in errors[:3])) + text = " — ".join(p for p in parts if p) + return f"HTTP {status}: {text}" if text else f"HTTP {status}" + + +# --------------------------------------------------------------------------- +# Error classification +# --------------------------------------------------------------------------- + +# Capacity shortage has no machine-readable code of its own — the v2 docs +# say so explicitly — so it has to be recognised from `detail`. Same +# phrasings the runpodctl-era regex covered, since the wording comes from +# the same orchestrator. +_UNAVAILABLE_RE = re.compile( + r"no\s+longer\s+any\s+instances\s+available" + r"|please\s+refresh\s+and\s+try\s+again" + r"|does\s+not\s+have\s+the\s+resources" + r"|try\s+a\s+different\s+machine" + r"|no\s+(?:machines|capacity|hosts|gpus|instances)\s+available" + r"|insufficient\s+capacity" + r"|unavailable" + r"|out\s+of\s+stock" + r"|sold\s+out" + r"|no\s+capacity", + re.IGNORECASE, +) + +_TRANSIENT_RE = re.compile( + r"something\s+went\s+wrong" + r"|please\s+try\s+again\s+later" + r"|contact\s+support" + r"|internal\s+server\s+error" + r"|timeout|timed\s+out" + r"|connection\s+(?:reset|refused)", + re.IGNORECASE, +) + +# Retry these regardless of body: 429 is rate limiting, 5xx is the API +# itself, 0 is a transport failure that may not have reached the API. +_TRANSIENT_STATUSES = {_TRANSPORT_ERROR, 429, 500, 502, 503, 504} + + +def classify_error(status: int, data: object) -> str: + """Map a failed response to 'UNAVAILABLE', 'TRANSIENT' or 'FATAL'. + + UNAVAILABLE — no capacity for this instance; caller tries the next one. + TRANSIENT — worth retrying the same request. + FATAL — bad request, auth, missing image; retrying won't help. + + Status is consulted first: rate limiting and 5xx are infrastructure, so + a 503 is never "this GPU type is full". Capacity is reported as 400 with + only a human-readable `detail`, so it can only be recognised by wording. + + Only `detail` is matched, never `title` — the title is a generic HTTP + reason phrase, and "Service Unavailable" would otherwise read as a + capacity shortage and make us abandon a perfectly good GPU type. + """ + if status in _TRANSIENT_STATUSES: + return "TRANSIENT" + detail = str(data.get("detail") or "") if isinstance(data, dict) else "" + if detail and _UNAVAILABLE_RE.search(detail): + return "UNAVAILABLE" + if detail and _TRANSIENT_RE.search(detail): + return "TRANSIENT" + return "FATAL" + + +def request_with_retries( + method: str, + path: str, + *, + body: Optional[dict] = None, + params: Optional[dict] = None, + timeout: int = 60, + attempts: int = 3, + backoff: int = 3, +) -> tuple[int, object]: + """`request` plus retries on TRANSIENT. For idempotent reads and for + deletes; pod creation drives its own retry loop so it can log per + attempt and count against CREATE_RETRIES.""" + status: int = _TRANSPORT_ERROR + data: object = None + for attempt in range(1, attempts + 1): + status, data = request( + method, path, body=body, params=params, timeout=timeout + ) + if 200 <= status < 300: + return status, data + if classify_error(status, data) != "TRANSIENT" or attempt == attempts: + return status, data + time.sleep(backoff * attempt) + return status, data + + +def api_available() -> tuple[bool, str]: + """Cheap credential check used at startup. Any authenticated 2xx will + do; the SSH-keys endpoint is the smallest one that needs no arguments.""" + if not load_api_key(): + return False, ( + "no RunPod API key — set RUNPOD_API_KEY or log in so " + "~/.runpod/config.toml has one" + ) + status, data = request("GET", "/account/ssh-keys", timeout=20) + if 200 <= status < 300: + return True, "" + if status in (401, 403): + return False, f"RunPod API rejected the key ({error_detail(status, data)})" + return False, f"RunPod API unreachable ({error_detail(status, data)})" + + +def log_error(context: str, status: int, data: object, indent: int = 1) -> None: + log(f"{context}: {error_detail(status, data)}", indent=indent) diff --git a/tests/runpod_smoke/checks.py b/tests/runpod_smoke/checks.py index ee1a6aa1..d76140ff 100644 --- a/tests/runpod_smoke/checks.py +++ b/tests/runpod_smoke/checks.py @@ -22,9 +22,8 @@ import urllib.request from typing import Callable, Optional -from . import config +from . import api, config from .log import log -from .runpodctl import runpodctl_json # Error string returned by every helper that shells out to `ssh` and fails @@ -146,34 +145,16 @@ def fetch_pod_cuda_version(pod_id: str, attempts: int = 3) -> str: `min_cuda_version` is only a floor, so the scheduler may place the pod on any host at or above it — this reports what it actually got, which is the - point of a compatibility matrix. `runpodctl` drops the field, so it comes - from `GET /v2/pods/{id}` rather than `pod get`. + point of a compatibility matrix. Nullable per the API: CPU pods and hosts that never reported one give ''. Retried a few times because the value only lands once the scheduler has assigned a machine. Reporting only — never turns a PASS into a FAIL. """ - from .instances import _load_runpod_api_key - - api_key = _load_runpod_api_key() - if not api_key: - return "" - req = urllib.request.Request( - f"https://api.runpod.io/v2/pods/{pod_id}", - headers={ - "Authorization": f"Bearer {api_key}", - "Accept": "application/json", - "User-Agent": "test-images.py/1.0 (+runpod-smoketest)", - }, - ) for attempt in range(1, attempts + 1): - try: - with urllib.request.urlopen(req, timeout=15) as resp: - data = json.loads(resp.read()) - except (OSError, ValueError): - data = None - cuda = (data or {}).get("cudaVersion") - if cuda: + status, data = api.request("GET", f"/pods/{pod_id}", timeout=15) + cuda = data.get("cudaVersion") if isinstance(data, dict) else None + if 200 <= status < 300 and cuda: return str(cuda).strip() if attempt < attempts: time.sleep(2) @@ -582,9 +563,7 @@ def fetch_pod_logs_api( The endpoint stays open for live logs. Stop after its historical backfill is drained (socket idle) or the deadline expires. """ - from .instances import _load_runpod_api_key - - api_key = _load_runpod_api_key() + api_key = api.load_api_key() if not api_key: return None tail = tail or config.LOG_API_TAIL @@ -625,26 +604,11 @@ def fetch_pod_logs_api( def pod_status_api(pod_id: str) -> Optional[str]: """Return lifecycle status from `GET /v2/pods/{id}`, if available.""" - from .instances import _load_runpod_api_key - - api_key = _load_runpod_api_key() - if not api_key: - return None - req = urllib.request.Request( - f"https://api.runpod.io/v2/pods/{pod_id}", - headers={ - "Authorization": f"Bearer {api_key}", - "Accept": "application/json", - "User-Agent": "test-images.py/1.0 (+runpod-smoketest)", - }, - ) - try: - with urllib.request.urlopen(req, timeout=10) as resp: - payload = json.loads(resp.read()) - except (urllib.error.HTTPError, OSError, json.JSONDecodeError): + status, data = api.request("GET", f"/pods/{pod_id}", timeout=10) + if not (200 <= status < 300) or not isinstance(data, dict): return None - status = payload.get("status") - return status if isinstance(status, str) else None + value = data.get("status") + return value if isinstance(value, str) else None def system_log_errors(pod_id: str, max_lines: int = 20) -> Optional[list[str]]: @@ -666,9 +630,7 @@ def scan_pod_logs_for_errors(pod_id: str) -> tuple[bool, str]: Empty API responses are retried and then fail as unverified: every supported image emits boot logs, so zero lines cannot prove a clean boot. """ - from .instances import _load_runpod_api_key - - if not _load_runpod_api_key(): + if not api.load_api_key(): return True, "(no API key — log scan skipped)" failures: list[str] = [] for attempt in range(1, _LOG_SCAN_ATTEMPTS + 1): @@ -758,22 +720,25 @@ def fetch_logs_via_ssh( def dump_pod_logs(pod_id: str, image: str) -> None: """Print metadata, API container logs, system errors, and GPU SMI.""" - data = runpodctl_json("pod", "get", pod_id, timeout=30) - if not isinstance(data, dict): - log("(could not fetch pod state)", indent=2) + status, data = api.request("GET", f"/pods/{pod_id}", timeout=30) + if not (200 <= status < 300) or not isinstance(data, dict): + api.log_error("(could not fetch pod state)", status, data, indent=2) return ssh = data.get("ssh") or {} - host, port = ssh.get("ip"), ssh.get("port") + direct = ssh.get("direct") or {} + proxy = ssh.get("proxy") or {} + host, port = direct.get("host"), direct.get("port") log(f"--- pod metadata for {pod_id} ---", indent=2) for key, val in [ - ("desiredStatus", data.get("desiredStatus")), - ("uptimeSeconds", data.get("uptimeSeconds")), - ("ssh.ip:port", f"{host}:{port}" if host and port else None), - ("ssh.error", ssh.get("error")), - ("ssh.key_in_account", (ssh.get("ssh_key") or {}).get("in_account")), - ("imageName", data.get("imageName")), - ("lastStatusChange", data.get("lastStatusChange")), + ("status", data.get("status")), + ("cudaVersion", data.get("cudaVersion")), + ("dataCenterId", data.get("dataCenterId")), + ("cost", data.get("cost")), + ("ssh.direct", f"{host}:{port}" if host and port else None), + ("ssh.proxy", proxy.get("host") or None), + ("image", data.get("image")), + ("startedAt", data.get("startedAt")), ]: log(f" {key:20s} = {val!r}", indent=2) diff --git a/tests/runpod_smoke/config.py b/tests/runpod_smoke/config.py index dd791060..3cc0cae0 100644 --- a/tests/runpod_smoke/config.py +++ b/tests/runpod_smoke/config.py @@ -11,7 +11,6 @@ import os from dataclasses import dataclass -from datetime import datetime, timedelta, timezone _PKG_DIR = os.path.dirname(os.path.abspath(__file__)) _TESTS_DIR = os.path.dirname(_PKG_DIR) @@ -83,7 +82,7 @@ def _coerce_on_skip(raw: str) -> str: # an anonymous IP pool that hits Docker Hub's `toomanyrequests` rate limit # fast. Either set REGISTRY_AUTH_ID explicitly, or REGISTRY_AUTH_NAME to # pick by display name, or the script auto-picks the first entry from -# `runpodctl registry list`. +# `GET /v2/registries`. # # REGISTRY_AUTH_ID is reassigned by main() after auto-discovery — access # it via `config.REGISTRY_AUTH_ID` (not a bare `from config import`) to @@ -91,35 +90,21 @@ def _coerce_on_skip(raw: str) -> str: REGISTRY_AUTH_ID = os.environ.get("REGISTRY_AUTH_ID", "") REGISTRY_AUTH_NAME = os.environ.get("REGISTRY_AUTH_NAME", "") -# Server-side auto-terminate window, passed to `--terminate-after` so -# RunPod self-destructs anything we leak (crash before cleanup, hung SSH, -# or — the common one — a GitHub Actions `cancel-in-progress` cancel that -# SIGKILLs us before cleanup_all() finishes). -# -# FORMAT: `runpodctl pod create --terminate-after` wants an RFC3339 -# DATETIME (`--help`: "auto-terminate datetime (e.g., 2026-04-15T00:00:00Z)"), -# NOT a duration. So we pass an absolute `now + AUTO_TERMINATE_HOURS` -# timestamp, recomputed per pod. -AUTO_TERMINATE_HOURS = int(os.environ.get("AUTO_TERMINATE_HOURS", "2")) - - -def auto_terminate_deadline() -> str: - """Return an RFC3339 'Z' timestamp AUTO_TERMINATE_HOURS hours from - NOW — the datetime format `--terminate-after` expects. Always call - this at pod-create time; never cache the result. - """ - return ( - datetime.now(timezone.utc) + timedelta(hours=AUTO_TERMINATE_HOURS) - ).strftime("%Y-%m-%dT%H:%M:%SZ") +# NO server-side auto-terminate any more. The old CLI took +# `--terminate-after `, which self-destructed anything we leaked +# (crash before cleanup, or a GitHub Actions `cancel-in-progress` that +# SIGKILLs us mid-cleanup_all). REST API v2 has no equivalent field, so +# .github/workflows/reap-pods.yml — hourly, deletes `smoketest-*` older +# than 60 min — is now the ONLY backstop. Keep it healthy. # --------------------------------------------------------------------------- # SSH # --------------------------------------------------------------------------- -# `runpodctl` does not expose container logs in JSON. REST API v2 is the -# primary log source; SSH is retained only for the GPU SMI diagnostic. -# The SSH endpoint is discovered from `pod get` once the pod is scheduled. +# REST API v2 is the primary log source; SSH is retained only for the GPU +# SMI diagnostic. The endpoint comes from `ssh.direct` on GET /v2/pods/{id} +# once the pod is scheduled. # Override SSH_IDENTITY if your key lives in a non-standard location. # Set SSH_LOG_FETCH=0 to skip SSH-based log fetching entirely. SSH_IDENTITY = os.environ.get("RUNPOD_SSH_KEY", "") @@ -167,7 +152,7 @@ def auto_terminate_deadline() -> str: # --------------------------------------------------------------------------- # Password we hand to start.sh via env. Not a secret — every pod we spin -# up is auto-terminated within AUTO_TERMINATE and is only reachable through +# up is short-lived (reaped within the hour) and is only reachable through # RunPod's authenticated proxy. We just need ANY non-empty value so start.sh # decides to launch Jupyter (see start.sh: `if [[ $JUPYTER_PASSWORD ]]`). JUPYTER_TEST_PASSWORD = "admin" @@ -227,15 +212,10 @@ def auto_terminate_deadline() -> str: # CPU "instance" candidates. # -# `runpodctl pod create` (the new subcommand we use) does NOT expose -# `--cpu-flavor`, `--vcpu`, or `--mem`. The legacy `runpodctl create pod` -# command does have them, but it's a different code path with different -# flags — the new command takes (`--compute-type CPU`, `--cloud-type`, -# optional `--data-center-ids`) and lets RunPod pick the cheapest CPU -# flavor that fits the container disk size for the chosen cloud + DC. -# -# To get MULTIPLE CPU "candidates" we vary the axes runpodctl actually -# accepts: +# The flavor itself (`cpu.id` + `vcpuCount`) is chosen by +# `instances.pick_cpu_flavor` from GET /v2/catalog/cpus. What varies between +# candidates is placement, so that a full pool in one cloud doesn't end the +# attempt: # - cloud_type: SECURE vs COMMUNITY. Totally different capacity # pools — when SECURE is full, COMMUNITY almost # always has free CPU hosts (and is cheaper). @@ -259,12 +239,25 @@ def auto_terminate_deadline() -> str: # * Empty / all-malformed input falls back to DEFAULT_CPU_CANDIDATES. +# CPU flavors from GET /v2/catalog/cpus, populated at startup. v2 requires +# an explicit `cpu.id` + `vcpuCount` on pod create, unlike the old CLI which +# let RunPod pick — so we choose the cheapest fitting flavor ourselves. +CPU_CATALOG: list[dict] = [] + +# vCPU count requested for CPU pods. Must be a power of two and inside the +# chosen flavor's `vcpu.min..max`. 4 is the smallest that every flavor +# offers and is plenty for a boot-and-dwell smoke test. +CPU_VCPU_COUNT = int(os.environ.get("CPU_VCPU_COUNT", "4")) + +# Pin a specific CPU flavor id (e.g. 'cpu5c') instead of auto-picking the +# cheapest one. Empty = auto. +CPU_FLAVOR_ID = os.environ.get("CPU_FLAVOR_ID", "") + + @dataclass(frozen=True) class CpuCandidate: - """One CPU pod-create attempt — varied along the axes that - `runpodctl pod create` exposes for CPU pods. RunPod's scheduler - still picks the actual CPU flavor (vCPU/RAM tier) inside the chosen - cloud + DC, based on container disk size.""" + """One CPU pod-create attempt — varied along cloud and data centre. + The flavor itself is chosen by `instances.pick_cpu_flavor`.""" cloud_type: str # 'SECURE' or 'COMMUNITY' data_center_ids: str = "" # comma-separated; "" = any DC in the cloud @@ -330,13 +323,13 @@ def cpu_candidate_for(instance: str) -> CpuCandidate: # Shared mutable state (populated at startup / from the manifest) # --------------------------------------------------------------------------- -# Display-name -> runpodctl --gpu-id mapping. Populated at startup from -# `runpodctl gpu list --include-unavailable`. Keeps the YAML manifest free +# Display-name -> gpu id mapping, populated at startup from +# `GET /v2/catalog/gpus`. Keeps the YAML manifest free # of RunPod-internal gpuId strings — users only put display names there. GPU_ID_MAP: dict[str, str] = {} # GPU catalog with pricing. Populated at startup via GraphQL since -# `runpodctl gpu list` does NOT include price fields. Used for budget-based +# Populated from GET /v2/catalog/gpus. Used for budget-based # instance selection (manifest's `max_price_per_hour`). Empty list if API # unreachable; in that case budget filters silently no-op and the script # falls back to whatever's in the manifest's explicit `instances:` list. @@ -348,7 +341,7 @@ def cpu_candidate_for(instance: str) -> CpuCandidate: # tag has no embedded CUDA — NGC `nvidia-pytorch:25.11` and similar # opaque tags). # -# Despite the "min" naming (which matches runpodctl's underlying +# Despite the "min" naming (which matches the API's underlying # `--min-cuda-version` flag), this field is a FALLBACK, not an override: # if the tag contains `cu1281` / `cuda1300` / etc., the manifest value # is silently ignored. By design — tag is the most accurate source for @@ -357,6 +350,22 @@ def cpu_candidate_for(instance: str) -> CpuCandidate: # notices. GROUP_MIN_CUDA: dict[str, str] = {} +# Per-group CUDA axis, populated from the `cuda_versions:` manifest field. +# When active, each candidate GPU is expanded into one job per CUDA version +# it actually offers, and the version is pinned via +# `gpu.allowedCudaVersions`. That overrides GROUP_MIN_CUDA for the group, +# because the API rejects allowedCudaVersions and minCudaVersion together. +# +# GROUP_CUDA_VERSIONS — explicit versions asked for, e.g. ['12.8', '13.0'] +# GROUP_CUDA_ALL — 'all': every version the GPU reports capacity for +GROUP_CUDA_VERSIONS: dict[str, list[str]] = {} +GROUP_CUDA_ALL: dict[str, bool] = {} + +# Safety cap on the GPU x CUDA fan-out. A catalog-wide `all` sweep is ~34 +# jobs today, but the fleet grows and a stray run shouldn't turn into a +# day of GPU time. Jobs past the cap are dropped with a warning. +MAX_CUDA_COMBOS = int(os.environ.get("MAX_CUDA_COMBOS", "120")) + # Per-group Jupyter-check opt-in, populated in main() from the # `test_jupyter:` manifest field. When True, `pod.create_pod` adds the # JUPYTER_PASSWORD env var and exposes :8888, and `runner.test_pair` runs diff --git a/tests/runpod_smoke/instances.py b/tests/runpod_smoke/instances.py index aff0dd2b..e93ed48a 100644 --- a/tests/runpod_smoke/instances.py +++ b/tests/runpod_smoke/instances.py @@ -12,18 +12,12 @@ from __future__ import annotations import fnmatch -import json -import os import re -import urllib.error -import urllib.request -from pathlib import Path from typing import Optional -from . import config +from . import api, config from .log import log from .manifest import _normalize_bool -from .runpodctl import runpodctl_json # Supports `cuda1281` / `cu1300` and the ComfyUI `cuda13.0` convention. @@ -58,7 +52,7 @@ def detect_cuda_version(image: str) -> Optional[str]: def resolve_gpu_id(display_name: str) -> str: - """Map a user-supplied GPU display name to its runpodctl gpuId. + """Map a user-supplied GPU display name to its RunPod gpu id. Tries exact match first, then case-insensitive match (so 'RTX 4070 TI' in the manifest still finds 'RTX 4070 Ti' in the RunPod catalog). @@ -82,70 +76,182 @@ def is_known_gpu(display_name: str) -> bool: return any(name.lower() == lowered for name in config.GPU_ID_MAP) -def discover_gpu_id_map() -> dict[str, str]: - """Build {displayName: gpuId} from `runpodctl gpu list`.""" - data = runpodctl_json( - "gpu", "list", "--include-unavailable", timeout=30 +def discover_gpu_catalog() -> list[dict]: + """Fetch GPU types from `GET /v2/catalog/gpus`. + + Entries keep the field names the rest of the module already uses + (`displayName`, `memoryInGb`, `securePrice`, `communityPrice`) so the + budget/vRAM filters didn't have to change; `cudaVersions` is new and + carries per-version capacity, which is what makes a GPU x CUDA sweep + possible without blind pod-create attempts. + + `include=AVAILABILITY` needs `product`, and scopes availability and + lowest-price to that context. Returns [] on any failure — the script + still works if the manifest uses explicit `instances:` lists. + """ + status, data = api.request_with_retries( + "GET", + "/catalog/gpus", + params={ + "include": ["AVAILABILITY"], + "product": ["POD"], + "cloud": config.CLOUD_TYPE.upper(), + "count": 1, + }, + timeout=30, ) - if not isinstance(data, list): - return {} + if not (200 <= status < 300) or not isinstance(data, dict): + api.log_error("warn: could not fetch GPU catalog", status, data, indent=0) + return [] + out: list[dict] = [] + for gpu in data.get("gpus") or []: + if not isinstance(gpu, dict): + continue + price = gpu.get("price") or {} + cuda = [ + cv.get("version") + for cv in (gpu.get("cudaVersions") or []) + if isinstance(cv, dict) and cv.get("version") + ] + cuda_available = [ + cv.get("version") + for cv in (gpu.get("cudaVersions") or []) + if isinstance(cv, dict) and cv.get("version") and cv.get("available") + ] + out.append({ + "id": gpu.get("id") or "", + "displayName": gpu.get("name") or "", + "memoryInGb": gpu.get("memory") or 0, + "manufacturer": gpu.get("manufacturer") or "", + "securePrice": price.get("secure") or 0, + "communityPrice": price.get("community") or 0, + "availability": gpu.get("availability") or "", + "cudaVersions": cuda, + "cudaVersionsAvailable": cuda_available, + }) + return out + + +def discover_gpu_id_map() -> dict[str, str]: + """Build {displayName: gpuId} from the already-fetched catalog. + + Falls back to its own request when called before the catalog is loaded, + so callers don't have to care about ordering. + """ + catalog = config.GPU_CATALOG or discover_gpu_catalog() return { - item["displayName"]: item["gpuId"] - for item in data - if item.get("displayName") and item.get("gpuId") + gpu["displayName"]: gpu["id"] + for gpu in catalog + if gpu.get("displayName") and gpu.get("id") } -def _load_runpod_api_key() -> Optional[str]: - """Read the API key out of ~/.runpod/config.toml. We avoid a tomli - dependency by regex'ing the file — the CLI always writes the key on a - single line like `apikey = '...'`. Also honours RUNPOD_API_KEY env var - so CI / containerized runs can inject it without touching the file.""" - env = os.environ.get("RUNPOD_API_KEY", "").strip() - if env: - return env - cfg = Path.home() / ".runpod" / "config.toml" - if not cfg.is_file(): - return None - try: - text = cfg.read_text() - except OSError: - return None - m = re.search(r"apikey\s*=\s*['\"]([^'\"]+)['\"]", text) - return m.group(1) if m else None +def cuda_versions_offered(display_name: str, *, only_available: bool = True) -> list[str]: + """CUDA versions the catalog reports for one GPU display name. + `only_available` keeps just the versions with free capacity right now — + the field the API describes as "at least one machine on this CUDA version + has free capacity". Pinning a version nobody reports yields a capacity + error rather than a fallback, so filtering here is what keeps a GPU x + CUDA sweep from being mostly wasted attempts. + """ + key = "cudaVersionsAvailable" if only_available else "cudaVersions" + lowered = display_name.lower() + for gpu in config.GPU_CATALOG: + if (gpu.get("displayName") or "").lower() == lowered: + return list(gpu.get(key) or []) + return [] -def discover_gpu_catalog() -> list[dict]: - """Fetch GPU types + per-hour prices from RunPod GraphQL. - Each entry has: id, displayName, memoryInGb, securePrice, - communityPrice, manufacturer. Returns [] on any failure (script will - still work if the manifest uses explicit `instances:` lists).""" - api_key = _load_runpod_api_key() - if not api_key: +def cuda_axis_for(group: str, display_name: str) -> list[str]: + """Versions to test `display_name` on for `group`, newest first. + + [] means "no CUDA axis for this group" — the caller then emits a single + unpinned job and the floor is derived from the image tag as before. + """ + all_mode = config.GROUP_CUDA_ALL.get(group, False) + requested = config.GROUP_CUDA_VERSIONS.get(group) or [] + if not (all_mode or requested): return [] - query = ("{ gpuTypes { id displayName memoryInGb " - "securePrice communityPrice manufacturer } }") - req = urllib.request.Request( - "https://api.runpod.io/graphql", - data=json.dumps({"query": query}).encode(), - headers={ - "Authorization": f"Bearer {api_key}", - "Content-Type": "application/json", - # api.runpod.io is fronted by Cloudflare, which rejects the - # default Python-urllib User-Agent with error code 1010. - # Identify as a generic client to get through. - "User-Agent": "test-images.py/1.0 (+runpod-smoketest)", + offered = cuda_versions_offered(display_name) + if all_mode: + picked = offered + else: + picked = [v for v in requested if v in offered] + return sorted(picked, key=_version_key, reverse=True) + + +def _version_key(version: str) -> tuple: + try: + return tuple(int(p) for p in version.split(".")) + except (TypeError, ValueError): + return (0,) + + +def discover_cpu_catalog() -> list[dict]: + """Fetch CPU flavors from `GET /v2/catalog/cpus`. + + Needed because v2 requires an explicit `cpu.id` + `vcpuCount` on pod + create, where the old CLI let RunPod pick the flavor itself. + """ + status, data = api.request_with_retries( + "GET", + "/catalog/cpus", + params={ + "include": ["AVAILABILITY"], + "product": ["POD"], }, - method="POST", + timeout=30, ) - try: - with urllib.request.urlopen(req, timeout=15) as resp: - payload = json.loads(resp.read()) - except (urllib.error.URLError, json.JSONDecodeError, TimeoutError) as exc: - log(f"warn: could not fetch GPU prices: {exc}") + if not (200 <= status < 300) or not isinstance(data, dict): + api.log_error("warn: could not fetch CPU catalog", status, data, indent=0) return [] - return ((payload.get("data") or {}).get("gpuTypes") or []) + return [c for c in (data.get("cpus") or []) if isinstance(c, dict)] + + +# Availability is a point-in-time snapshot, so it ORDERS candidates rather +# than excluding them: at the time of writing every CPU flavor reports NONE, +# and refusing to try would mean never creating a CPU pod at all. A create +# against a full pool comes back UNAVAILABLE, which the runner already +# handles by moving to the next candidate. +_AVAILABILITY_RANK = {"HIGH": 0, "MEDIUM": 1, "LOW": 2, "NONE": 3} + + +def pick_cpu_flavor() -> tuple[str, int]: + """Choose (flavor_id, vcpuCount) for a CPU pod. + + `CPU_FLAVOR_ID` wins when set. Otherwise the flavor whose `vcpu` range + admits `config.CPU_VCPU_COUNT`, preferring better-reported availability + and then the lower `price.securePerVcpu`. Returns ("", 0) only when the + catalog is unreachable or no flavor supports the requested vCPU count. + """ + catalog = config.CPU_CATALOG or discover_cpu_catalog() + if catalog and not config.CPU_CATALOG: + config.CPU_CATALOG.extend(catalog) + want = config.CPU_VCPU_COUNT + + if config.CPU_FLAVOR_ID: + # Trust the override even if it isn't in the catalog — the API is + # the authority and a stale catalog shouldn't block an explicit ask. + return config.CPU_FLAVOR_ID, want + + def fits(flavor: dict) -> bool: + vcpu = flavor.get("vcpu") or {} + lo = int(vcpu.get("min") or 0) + hi = int(vcpu.get("max") or 0) + return bool(flavor.get("id")) and lo <= want <= (hi or want) + + candidates = [f for f in catalog if fits(f)] + if not candidates: + return "", 0 + best = min( + candidates, + key=lambda f: ( + _AVAILABILITY_RANK.get(f.get("availability") or "", 1), + float((f.get("price") or {}).get("securePerVcpu") or 1e9), + ), + ) + return best.get("id") or "", want def _apply_exclude_filter( @@ -279,14 +385,12 @@ def resolve_instances(group_name: str, group_config: dict) -> list[str]: """Decide which GPU display names this group should try, in order. Priority: - 0. CPU groups (name in `config.CPU_GROUP_NAMES`) — `runpodctl pod - create` doesn't accept a CPU-flavor flag, so we expand to one - entry per `config.CPU_CANDIDATES` label. Each label varies the - axes that runpodctl DOES accept for CPU (`--cloud-type` SECURE - vs COMMUNITY, optional `--data-center-ids`) — see config.py - for rationale. The caller's per-instance loop walks them in - order on UNAVAILABLE / STUCK, identical to how it cycles - through GPU types. + 0. CPU groups (name in `config.CPU_GROUP_NAMES`) — expand to one + entry per `config.CPU_CANDIDATES` label. The flavor comes from + `pick_cpu_flavor`; the labels vary only placement (SECURE vs + COMMUNITY, optional data centres). The caller's per-instance loop + walks them in order on UNAVAILABLE / STUCK, identical to how it + cycles through GPU types. 1. Explicit `instances:` list in the manifest — wins, used as-is. 2. `max_price_per_hour: X` (+ optional `min_vram_gb`, `manufacturer`) — auto-pick from RunPod catalog, sorted cheapest first. diff --git a/tests/runpod_smoke/manifest.py b/tests/runpod_smoke/manifest.py index b5490571..bb620b70 100644 --- a/tests/runpod_smoke/manifest.py +++ b/tests/runpod_smoke/manifest.py @@ -34,7 +34,7 @@ def _normalize_bool(value: object) -> Optional[bool]: def _normalize_cuda_version(value: object) -> Optional[str]: """Coerce a manifest `min_cuda_version` value to the 'X.Y' string format - that `runpodctl --min-cuda-version` expects. + that `gpu.minCudaVersion` expects. Accepts ints (`13` → '13.0'), floats (`12.8` → '12.8', `13.0` → '13.0'), and strings (with or without surrounding quotes). Returns None for diff --git a/tests/runpod_smoke/pod.py b/tests/runpod_smoke/pod.py index 474be22d..3a702cf6 100644 --- a/tests/runpod_smoke/pod.py +++ b/tests/runpod_smoke/pod.py @@ -1,5 +1,7 @@ """Pod creation, lifecycle tracking, signal-safe cleanup, registry auth. +Everything here talks to REST API v2 (see api.py) — no runpodctl. + Owns the `ACTIVE_POD_IDS` set + lock — the source of truth for "what's still alive on RunPod" across all workers. atexit + SIGINT/SIGTERM handlers are installed at import time so anything we leak on crash @@ -9,7 +11,6 @@ from __future__ import annotations import atexit -import json import re import signal import sys @@ -17,51 +18,19 @@ import time from typing import Optional -from . import config -from .checks import pod_status_api, ssh_probe, system_log_errors -from .instances import detect_cuda_version +from . import api, config +from .checks import ssh_probe, system_log_errors +from .instances import detect_cuda_version, pick_cpu_flavor from .log import log -from .runpodctl import runpodctl, runpodctl_json # --------------------------------------------------------------------------- # Error-classification regexes # --------------------------------------------------------------------------- -UNAVAILABLE_RE = re.compile( - # RunPod-specific phrasings observed in pod-create errors: - r"no\s+longer\s+any\s+instances\s+available" - r"|please\s+refresh\s+and\s+try\s+again" - # "This machine does not have the resources to deploy your pod. Please - # try a different machine" — RunPod returns this when a candidate host - # was picked but couldn't actually fit the pod (vRAM, disk, CPU). Same - # remediation as "no capacity": move on to the next instance type. - r"|does\s+not\s+have\s+the\s+resources" - r"|try\s+a\s+different\s+machine" - # Generic capacity-shortage phrasings: - r"|no\s+(?:machines|capacity|hosts|gpus|instances)\s+available" - r"|insufficient\s+capacity" - r"|unavailable" - r"|out\s+of\s+stock" - r"|sold\s+out", - re.IGNORECASE, -) - -# Generic "RunPod orchestrator hiccupped" errors that we should retry rather -# than treat as a real failure. These appear when several workers race for -# the same scarce GPU at the same instant, or the API is just transiently -# flaky. -TRANSIENT_RE = re.compile( - r"something\s+went\s+wrong" - r"|please\s+try\s+again\s+later" - r"|contact\s+support" - r"|internal\s+server\s+error" - r"|timeout|timed\s+out" - r"|502|503|504" - r"|connection\s+(?:reset|refused)", - re.IGNORECASE, -) - +# Capacity / transient classification of API failures lives in +# api.classify_error. This one is different: it scans pod-get FIELDS for a +# container-runtime failure that happened before the pod reached RUNNING. RUNTIME_ERROR_RE = re.compile( r"toomanyrequests" r"|rate\s+limit" @@ -95,12 +64,26 @@ def unregister_pod(pod_id: str) -> None: ACTIVE_POD_IDS.discard(pod_id) +def _terminate_pod(pod_id: str) -> tuple[bool, str]: + """DELETE the pod. 404 counts as success — it's already gone.""" + status, data = api.request_with_retries( + "DELETE", f"/pods/{pod_id}", timeout=30 + ) + if 200 <= status < 300 or status == 404: + return True, "" + return False, api.error_detail(status, data) + + def cleanup_pod(pod_id: str) -> None: """Delete a single pod and unregister it from the tracking set.""" if not pod_id: return log(f"Cleaning up pod {pod_id}...") - runpodctl("pod", "delete", pod_id, timeout=30) + ok, detail = _terminate_pod(pod_id) + if not ok: + # Loud, because a pod we failed to delete keeps billing until + # reap-pods.yml catches it. + log(f" WARNING: could not delete {pod_id}: {detail}") unregister_pod(pod_id) @@ -113,7 +96,9 @@ def cleanup_all() -> None: log(f"Cleaning up {len(leftover)} leftover pod(s)...") for pid in leftover: try: - runpodctl("pod", "delete", pid, timeout=30) + ok, detail = _terminate_pod(pid) + if not ok: + log(f" WARNING: could not delete {pid}: {detail}") except Exception as exc: # noqa: BLE001 log(f" failed to delete {pid}: {exc}") unregister_pod(pid) @@ -138,16 +123,18 @@ def _signal_handler(signum: int, _frame) -> None: def discover_registry_auth(prefer_name: str = "") -> Optional[str]: - """Find a registry auth id from `runpodctl registry list`.""" - data = runpodctl_json("registry", "list", timeout=30) - if not isinstance(data, list) or not data: + """Find a registry credential id from `GET /v2/registries`.""" + status, data = api.request_with_retries("GET", "/registries", timeout=30) + if not (200 <= status < 300) or not isinstance(data, dict): + return None + registries = data.get("registries") + if not isinstance(registries, list) or not registries: return None if prefer_name: - for item in data: + for item in registries: if (item.get("name") or "").lower() == prefer_name.lower(): - return item.get("id") or item.get("registryAuthId") - first = data[0] - return first.get("id") or first.get("registryAuthId") + return item.get("id") + return registries[0].get("id") # --------------------------------------------------------------------------- @@ -155,23 +142,34 @@ def discover_registry_auth(prefer_name: str = "") -> Optional[str]: # --------------------------------------------------------------------------- -def _extract_error(raw: str) -> str: - """Pull a concise error string out of runpodctl's noisy create-failure - output (which usually dumps the JSON error followed by a full `--help` - listing).""" - for line in raw.splitlines(): - line = line.strip() - if line.startswith("{") and '"error"' in line: - try: - return json.loads(line).get("error", line) - except json.JSONDecodeError: - return line - if line and line.split()[0] in {"Usage:", "Flags:", "Aliases:", - "Examples:", "Global"}: - break - if line: - return line - return raw[:200] +def _resolve_min_cuda(image: str, group: Optional[str]) -> Optional[str]: + """Pick the CUDA floor and log which source won. + + An explicit manifest value wins; the image tag is the fallback. Without + a floor a cu130 image can land on a 12.x driver and die with + `nvidia-container-cli: cuda>=13.0`. + """ + tag_cuda = detect_cuda_version(image) + manifest_cuda = config.GROUP_MIN_CUDA.get(group) if group else None + if manifest_cuda and tag_cuda and manifest_cuda != tag_cuda: + log( + f"min-cuda-version: requested '{manifest_cuda}' overrides " + f"tag-derived '{tag_cuda}'", + indent=1, + ) + elif tag_cuda and not manifest_cuda: + log( + f"min-cuda-version: none requested, derived '{tag_cuda}' " + "from the image tag", + indent=1, + ) + elif manifest_cuda and not tag_cuda: + log( + f"min-cuda-version: requested '{manifest_cuda}' (tag has " + "no CUDA marker to derive from)", + indent=1, + ) + return manifest_cuda or tag_cuda def create_pod( @@ -185,44 +183,30 @@ def create_pod( test_ports: Optional[list[int]] = None, cloud_type: Optional[str] = None, data_center_ids: str = "", -) -> tuple[Optional[str], str]: - """Create a pod via `runpodctl pod create`. Returns (pod_id, raw_output). - - compute_type='GPU' uses --gpu-id to target a specific GPU type (caller - must pass a non-empty gpu_id). - compute_type='CPU' creates a CPU pod. `runpodctl pod create` doesn't - accept a CPU-flavor flag, so RunPod picks the flavor based on - container disk size + selected cloud/DC. The caller varies CPU - candidates by overriding `cloud_type` (SECURE vs COMMUNITY) and - optionally `data_center_ids`. gpu_id is ignored in CPU mode. - - `cloud_type` overrides the global `config.CLOUD_TYPE` for this call - only — used by the CPU-candidate retry loop to try SECURE first - and then COMMUNITY. When None, `config.CLOUD_TYPE` is used. - - `data_center_ids` is a csv that, when non-empty, becomes - `--data-center-ids ` and pins the pod to those DCs. - - `group` is used to look up `min_cuda_version` from the manifest when - the image tag doesn't encode a CUDA version (e.g. NGC - `nvidia-pytorch:25.11`). Two-step resolution: - 1. Try to parse the CUDA version out of the image tag itself - (`cu1281` / `cuda1300` / ...). Tag wins because it's the most - accurate source — it's literally the CUDA the image was built - against. - 2. Fall back to the manifest's `min_cuda_version` when the tag has - no CUDA marker. This is the ONLY case where the manifest field - takes effect — for explicit-CUDA tags, the manifest value is - ignored even when set. - The merged result is forwarded to `runpodctl --min-cuda-version` so - RunPod's scheduler only picks hosts whose driver supports it. - - `test_jupyter=True` expands the pod config so JupyterLab can be tested: - - `--ports` gains `8888/http` - - `--env` sets `JUPYTER_PASSWORD` (the value start.sh checks before - starting Jupyter) - - `test_ports` exposes each generic HTTP service as `/http`. + allowed_cuda_versions: Optional[list[str]] = None, +) -> tuple[Optional[str], str, str]: + """Create a pod via `POST /v2/pods`. + + Returns `(pod_id, kind, detail)`: + * success -> (id, "", "") + * no capacity -> (None, "UNAVAILABLE", detail) + * retry me -> (None, "TRANSIENT", detail) + * give up -> (None, "FATAL", detail) + + compute_type='GPU' targets `gpu_id`; 'CPU' picks a flavor from the CPU + catalog (v2 requires an explicit `cpu.id` + `vcpuCount`, unlike the old + CLI which let RunPod choose). + + `cloud_type` overrides `config.CLOUD_TYPE` for this call only — used by + the CPU-candidate loop to try SECURE then COMMUNITY. `data_center_ids` + is a csv that, when non-empty, pins placement. + + `allowed_cuda_versions` pins the host to those exact CUDA versions. + Mutually exclusive with the derived floor: the API rejects + `allowedCudaVersions` and `minCudaVersion` together with a 400. + + NOTE: v2 has no `terminateAfter`, so there is no server-side deadline + to fall back on any more — reap-pods.yml is the only backstop. """ disk_gb = config.CPU_DISK_GB if compute_type == "CPU" else config.DISK_GB ports = ["22/tcp"] @@ -232,120 +216,99 @@ def create_pod( spec = f"{test_port}/http" if spec not in ports: ports.append(spec) - args = [ - "pod", "create", - "--image", image, - "--cloud-type", cloud_type or config.CLOUD_TYPE, - "--container-disk-in-gb", str(disk_gb), - "--ports", ",".join(ports), - "--name", name, - # Server-side backstop: RunPod auto-terminates the pod at this - # RFC3339 datetime (the format --terminate-after wants), so a leaked - # pod can't run forever. MUST be recomputed per pod — see - # config.auto_terminate_deadline. The reap-pods.yml cron is the - # primary sweep; this is defense-in-depth. - "--terminate-after", config.auto_terminate_deadline(), - "-o", "json", - ] + + body: dict = { + "name": name, + "image": image, + "cloud": (cloud_type or config.CLOUD_TYPE).upper(), + "disk": disk_gb, + "ports": ports, + # Injects PUBLIC_KEY from the account's registered keys, which is + # what makes the SSH readiness probe work. + "startSsh": True, + } if data_center_ids: - args.extend(["--data-center-ids", data_center_ids]) + body["dataCenterIds"] = [ + d.strip() for d in data_center_ids.split(",") if d.strip() + ] if test_jupyter: - # runpodctl wants --env as a single JSON-object string. - env_obj = {"JUPYTER_PASSWORD": config.JUPYTER_TEST_PASSWORD} - args.extend(["--env", json.dumps(env_obj)]) + body["env"] = {"JUPYTER_PASSWORD": config.JUPYTER_TEST_PASSWORD} + if config.REGISTRY_AUTH_ID: + body["registry"] = config.REGISTRY_AUTH_ID + if compute_type == "CPU": - args.extend(["--compute-type", "CPU"]) - # CPU images have no CUDA, no GPU — `--min-cuda-version` would be - # nonsensical and `--gpu-id` is rejected by runpodctl for CPU pods. - # CPU flavor (vCPU/RAM tier) is chosen by RunPod from the - # selected cloud+DC's pool based on container disk size; we can't - # request a specific tier with this subcommand. - else: - args.extend(["--gpu-id", gpu_id, "--gpu-count", "1"]) - # Pin scheduling to hosts whose driver supports the image's CUDA, - # else a cu130 image dies with `nvidia-container-cli: cuda>=13.0`. - # An explicit manifest value wins; the tag is the fallback. - tag_cuda = detect_cuda_version(image) - manifest_cuda = ( - config.GROUP_MIN_CUDA.get(group) if group else None - ) - cuda_version = manifest_cuda or tag_cuda - if cuda_version: - args.extend(["--min-cuda-version", cuda_version]) - if manifest_cuda and tag_cuda and manifest_cuda != tag_cuda: - log( - f"min-cuda-version: requested '{manifest_cuda}' overrides " - f"tag-derived '{tag_cuda}'", - indent=1, + flavor_id, vcpu = pick_cpu_flavor() + if not flavor_id: + return None, "FATAL", ( + "no CPU flavor available from GET /v2/catalog/cpus — set " + "CPU_FLAVOR_ID to choose one explicitly" ) - elif tag_cuda and not manifest_cuda: - log( - f"min-cuda-version: none requested, derived '{tag_cuda}' " - "from the image tag", - indent=1, - ) - elif manifest_cuda and not tag_cuda: + body["cpu"] = {"id": flavor_id, "vcpuCount": vcpu} + else: + gpu: dict = {"id": gpu_id, "count": 1} + if allowed_cuda_versions: + gpu["allowedCudaVersions"] = list(allowed_cuda_versions) log( - f"min-cuda-version: requested '{manifest_cuda}' (tag has " - "no CUDA marker to derive from)", + "allowedCudaVersions: pinned to " + f"{', '.join(allowed_cuda_versions)}", indent=1, ) - if config.REGISTRY_AUTH_ID: - args.extend(["--registry-auth-id", config.REGISTRY_AUTH_ID]) - proc = runpodctl(*args, timeout=120) - raw = (proc.stderr + proc.stdout).strip() - if proc.returncode != 0: - return None, _extract_error(raw) - try: - data = json.loads(proc.stdout) - except json.JSONDecodeError: - return None, _extract_error(raw) - pod_id = data.get("id") or (data.get("pod") or {}).get("id") - return pod_id, raw + else: + cuda_version = _resolve_min_cuda(image, group) + if cuda_version: + gpu["minCudaVersion"] = cuda_version + body["gpu"] = gpu + + status, data = api.request("POST", "/pods", body=body, timeout=120) + if 200 <= status < 300 and isinstance(data, dict): + pod_id = data.get("id") + if pod_id: + return pod_id, "", "" + return None, "FATAL", "pod create returned no id" + return None, api.classify_error(status, data), api.error_detail(status, data) def pod_state(pod_id: str) -> dict: """Return the relevant subset of pod state for decision-making. - When the pod is created with `--ports 22/tcp` (which we do), `pod get` - populates a useful `ssh` block with `ip`, `port`, and - `ssh_key.in_account` once the pod is scheduled. We use these as the - real readiness signal. + `status` is the real observed PodStatus enum (PROVISIONING / STARTING / + RUNNING / EXITED / ERROR / TERMINATED) — unlike the CLI's + `desiredStatus`, which was always RUNNING and could only ever detect + terminal states. + + `ssh.direct` is null until the pod has a machine assignment and a public + port for `22/tcp`; that transition is the readiness signal we poll for. """ - data = runpodctl_json("pod", "get", pod_id, timeout=30) - if not isinstance(data, dict): + status, data = api.request_with_retries("GET", f"/pods/{pod_id}", timeout=30) + if not (200 <= status < 300) or not isinstance(data, dict): return {} ssh = data.get("ssh") or {} + direct = ssh.get("direct") or {} return { - "desired": data.get("desiredStatus"), - "uptime": data.get("uptimeSeconds") or 0, - "ssh_ip": ssh.get("ip") or "", - "ssh_port": ssh.get("port") or 0, - "ssh_error": ssh.get("error") or "", - "ssh_key_in_account": (ssh.get("ssh_key") or {}).get("in_account"), - "last_status_change": data.get("lastStatusChange"), + "status": data.get("status"), + "ssh_ip": direct.get("host") or "", + "ssh_port": int(direct.get("port") or 0), + "cuda_version": data.get("cudaVersion") or "", + "cost": data.get("cost"), + "data_center": data.get("dataCenterId") or "", + "started_at": data.get("startedAt"), "raw": data, } def pod_status(pod_id: str) -> Optional[str]: - """Returns `desiredStatus` — note this is always RUNNING after creation - so it can ONLY be used to detect terminal states (EXITED/FAILED/DEAD).""" - return pod_state(pod_id).get("desired") + """Observed PodStatus, or None when the pod could not be read.""" + return pod_state(pod_id).get("status") -# Top-level fields on `pod get` that may carry a runtime error message -# directly. Checked verbatim with `isinstance(value, str)`. -_DIRECT_ERROR_FIELDS = ("lastError", "errorMessage", "statusMessage", - "lastStatusChange") +# Fields on the pod object that may carry a runtime error message directly. +_DIRECT_ERROR_FIELDS = ("lastError", "errorMessage", "statusMessage") -# Same as above but expected on the nested `runtime` dict that RunPod -# returns alongside top-level fields. +# Same, on the nested `runtime` dict. _RUNTIME_ERROR_FIELDS = ("lastError", "errorMessage", "statusMessage") # Fields whose value is a list of event objects (or strings); each item's -# `message` is harvested. `events` is the standard one; the other two -# show up on older `pod get` responses. +# `message` is harvested. _EVENT_LIST_FIELDS = ("events", "statusEvents", "containerEvents") # Fields whose value is a single block of log lines that may contain @@ -373,6 +336,8 @@ def _gather_runtime_error_candidates(data: dict) -> list[str]: return a flat list of candidate lines. Doesn't filter — that's `pod_runtime_error`'s job.""" runtime = data.get("runtime") or {} + if not isinstance(runtime, dict): + runtime = {} candidates: list[str] = [] for key in _DIRECT_ERROR_FIELDS: _collect_string_field(candidates, data, key) @@ -388,10 +353,11 @@ def _gather_runtime_error_candidates(data: dict) -> list[str]: def pod_runtime_error(pod_id: str) -> Optional[str]: - """Inspect pod-get response for container-runtime errors (pull failures, + """Inspect the pod object for container-runtime errors (pull failures, bad images, etc.) that appear *before* the pod ever reaches RUNNING. Returns a short error string or None.""" - data = runpodctl_json("pod", "get", pod_id, timeout=30) + state = pod_state(pod_id) + data = state.get("raw") if not isinstance(data, dict): return None for line in _gather_runtime_error_candidates(data): @@ -405,9 +371,8 @@ def pod_runtime_error(pod_id: str) -> Optional[str]: # --------------------------------------------------------------------------- -# Pod-lifecycle states that mean "we will never become RUNNING — stop polling". -_TERMINAL_DESIRED = {"EXITED", "FAILED", "DEAD", "TERMINATED"} -_TERMINAL_API_STATUSES = {"EXITED", "ERROR", "TERMINATED"} +# PodStatus values that mean "we will never become RUNNING — stop polling". +_TERMINAL_STATUSES = {"EXITED", "ERROR", "TERMINATED"} def _log_system_errors(pod_id: str, context: str) -> None: @@ -426,7 +391,7 @@ def _log_system_errors(pod_id: str, context: str) -> None: def _print_stall_hint(pod_id: str, elapsed: int) -> None: """One-time hint for pods that sit with no SSH endpoint for too long. - RunPod doesn't surface pull progress via API/CLI, so this points the + RunPod doesn't surface pull progress via the API, so this points the user at the UI plus the single most common root cause — Docker Hub rate-limiting an anonymous pull. """ @@ -448,7 +413,7 @@ def _print_stall_hint(pod_id: str, elapsed: int) -> None: def _probe_ssh_endpoint( host: str, port: int, - desired: object, + pod_status_value: object, elapsed: int, ssh_attempts: int, last_summary: Optional[tuple], @@ -461,7 +426,7 @@ def _probe_ssh_endpoint( caller compares against `last_summary` to dedup the log line. """ ok, err = ssh_probe(host, port, timeout=8) - summary = (desired, host, port, ok) + summary = (pod_status_value, host, port, ok) if summary != last_summary: log( f"t+{elapsed}s endpoint=root@{host}:{port} " @@ -480,28 +445,23 @@ def _probe_ssh_endpoint( def wait_for_running(pod_id: str) -> tuple[str, str]: """Returns (outcome, detail). Outcome is one of: - 'RUNNING' SSH probe to root@: succeeded — the - container's sshd is up, which means the container has - fully booted and we can trust it as healthy. - 'TERMINAL' desiredStatus flipped to EXITED/FAILED/DEAD/TERMINATED. + 'RUNNING' SSH probe to root@: succeeded — + the container's sshd is up, which means it has fully + booted and we can trust it as healthy. + 'TERMINAL' status reached EXITED / ERROR / TERMINATED. 'TIMEOUT' SSH never reachable within CREATE_TIMEOUT — pod stuck initializing (capacity issue or image broken). - SSH probing is the real health-check now. We poll `pod get` to discover - ssh.ip / ssh.port (assigned by RunPod once a machine is allocated), then - try `ssh root@ip -p port 'echo ready'` until it succeeds. This works - because: - * `--ports 22/tcp` in pod create makes RunPod NAT a public port to the - container's 22, so the container's sshd is reachable from anywhere. - * The PUBLIC_KEY env we inject lands in /root/.ssh/authorized_keys. - * A successful SSH means the container booted + sshd started — the - canonical signal of readiness, much stronger than `desiredStatus` - (always RUNNING) or `uptimeSeconds` (stale in this CLI version). + SSH probing is the real health-check. We poll `GET /v2/pods/{id}` for + `ssh.direct` (populated once a machine is allocated and `22/tcp` gets a + public port), then try `ssh root@host -p port 'echo ready'` until it + succeeds. A successful SSH means the container booted and sshd started — + a much stronger signal than any status field. """ start = time.time() deadline = start + config.CREATE_TIMEOUT last_summary: Optional[tuple] = None - last_api_status: Optional[str] = None + last_status: Optional[str] = None ssh_attempts = 0 stall_hinted = False # one-time hint when pod has no ssh endpoint for a while @@ -511,37 +471,34 @@ def wait_for_running(pod_id: str) -> tuple[str, str]: time.sleep(config.POLL_INTERVAL) continue - desired = st.get("desired") + pod_status_value = st.get("status") host = st.get("ssh_ip") or "" port = st.get("ssh_port") or 0 elapsed = int(time.time() - start) - api_status = pod_status_api(pod_id) - if api_status and api_status != last_api_status: - log(f"t+{elapsed}s API status: {api_status}", indent=2) - last_api_status = api_status + if pod_status_value != last_status: + log(f"t+{elapsed}s status: {pod_status_value}", indent=2) + last_status = pod_status_value - if desired in _TERMINAL_DESIRED or api_status in _TERMINAL_API_STATUSES: - terminal = ( - api_status if api_status in _TERMINAL_API_STATUSES else desired + if pod_status_value in _TERMINAL_STATUSES: + _log_system_errors(pod_id, f"pod entered {pod_status_value}") + return "TERMINAL", ( + f"pod entered {pod_status_value} after {elapsed}s" ) - _log_system_errors(pod_id, f"pod entered {terminal}") - return "TERMINAL", f"pod entered {terminal} after {elapsed}s" if host and port: ssh_attempts += 1 outcome, last_summary = _probe_ssh_endpoint( - host, int(port), desired, elapsed, ssh_attempts, last_summary, + host, int(port), pod_status_value, elapsed, ssh_attempts, + last_summary, ) if outcome is not None: return outcome else: - summary = (desired, api_status, host, port, False) + summary = (pod_status_value, host, port, False) if summary != last_summary: log( - f"t+{elapsed}s desired={desired!r} " - f"api_status={api_status!r} " - f"uptime={st.get('uptime') or 0}s " + f"t+{elapsed}s status={pod_status_value!r} " "ssh endpoint not assigned yet", indent=2, ) diff --git a/tests/runpod_smoke/runner.py b/tests/runpod_smoke/runner.py index 43ee8ee6..8e68d7dc 100644 --- a/tests/runpod_smoke/runner.py +++ b/tests/runpod_smoke/runner.py @@ -34,8 +34,6 @@ from .instances import detect_cuda_version, resolve_gpu_id from .log import log, set_worker_context from .pod import ( - TRANSIENT_RE, - UNAVAILABLE_RE, cleanup_pod, create_pod, pod_state, @@ -65,11 +63,13 @@ def _take_host_cuda() -> str: return version -def _log_attempt_header(image: str, instance: str, group: str) -> tuple[bool, str]: +def _log_attempt_header( + image: str, instance: str, group: str, cuda_pin: str = "", +) -> tuple[bool, str]: """Log the per-attempt header line and resolve the gpu_id. Returns (is_cpu, gpu_id). CPU attempts get an empty gpu_id since - runpodctl doesn't accept --gpu-id together with --compute-type CPU. + CPU pods carry a `cpu` block instead of a `gpu` one, so no gpu_id. Per-candidate (cloud_type, data_center_ids) is looked up separately by the caller via `config.cpu_candidate_for(instance)`.""" if config.is_cpu_instance(instance): @@ -86,9 +86,12 @@ def _log_attempt_header(image: str, instance: str, group: str) -> tuple[bool, st ) return True, "" gpu_id = resolve_gpu_id(instance) - # Same precedence as create_pod: explicit request wins, tag is fallback. - cuda = config.GROUP_MIN_CUDA.get(group) or detect_cuda_version(image) - cuda_note = f", min-cuda={cuda}" if cuda else "" + if cuda_pin: + cuda_note = f", cuda-pinned={cuda_pin}" + else: + # Same precedence as create_pod: explicit request wins, tag is fallback. + cuda = config.GROUP_MIN_CUDA.get(group) or detect_cuda_version(image) + cuda_note = f", min-cuda={cuda}" if cuda else "" log( f"attempt: instance='{instance}' (--gpu-id '{gpu_id}'){cuda_note}", indent=1, @@ -98,6 +101,7 @@ def _log_attempt_header(image: str, instance: str, group: str) -> tuple[bool, st def _create_pod_with_retries( image: str, instance: str, gpu_id: str, is_cpu: bool, group: str, + cuda_pin: str = "", ) -> tuple[Optional[str], str, str]: """Drive `create_pod` through the transient-error retry budget. @@ -132,7 +136,7 @@ def _create_pod_with_retries( test_ports = list(config.GROUP_TEST_PORTS.get(group) or []) if config.GROUP_TEST_COMFYUI.get(group, False): test_ports.append(config.COMFYUI_PORT) - pod_id, raw = create_pod( + pod_id, kind, raw = create_pod( image, gpu_id, name, compute_type="CPU" if is_cpu else "GPU", group=group, @@ -140,13 +144,14 @@ def _create_pod_with_retries( test_ports=test_ports, cloud_type=cloud_override, data_center_ids=dc_ids, + allowed_cuda_versions=[cuda_pin] if cuda_pin else None, ) if pod_id: return pod_id, "", "" - if UNAVAILABLE_RE.search(raw): + if kind == "UNAVAILABLE": log(f"instance unavailable, will try next ({raw[:120]})", indent=2) return None, "UNAVAILABLE", "" - if TRANSIENT_RE.search(raw) and attempt < config.CREATE_RETRIES: + if kind == "TRANSIENT" and attempt < config.CREATE_RETRIES: backoff = config.CREATE_RETRY_BACKOFF * attempt log( f"transient pod-create error ({raw[:120]}), " @@ -412,7 +417,9 @@ def _run_post_dwell_steps( return _run_log_scan_step(pod_id, image) -def test_pair(image: str, instance: str, group: str) -> _Outcome: +def test_pair( + image: str, instance: str, group: str, cuda_pin: str = "", +) -> _Outcome: """Returns (status, detail). Statuses: 'PASS' — image booted, CUDA check OK, survived dwell 'FAIL' — pod was created and the CONTAINER itself proved @@ -436,10 +443,10 @@ def test_pair(image: str, instance: str, group: str) -> _Outcome: # Clear first so a label from a previous instance can't leak into an # attempt that never reaches the probe (UNAVAILABLE, STUCK). _set_host_cuda("") - is_cpu, gpu_id = _log_attempt_header(image, instance, group) + is_cpu, gpu_id = _log_attempt_header(image, instance, group, cuda_pin) pod_id, early, early_detail = _create_pod_with_retries( - image, instance, gpu_id, is_cpu, group, + image, instance, gpu_id, is_cpu, group, cuda_pin, ) if early: return early, early_detail @@ -503,7 +510,7 @@ def test_pair(image: str, instance: str, group: str) -> _Outcome: def test_image( - image: str, instances: list[str], group: str + image: str, instances: list[str], group: str, cuda_pin: str = "", ) -> tuple[str, str, str, str]: """Returns (status, note, instance_used, host_cuda). @@ -529,7 +536,7 @@ def test_image( for inst in instances: set_worker_context(inst) try: - result, detail = test_pair(image, inst, group) + result, detail = test_pair(image, inst, group, cuda_pin) finally: set_worker_context(None) host_cuda = _take_host_cuda() diff --git a/tests/runpod_smoke/runpodctl.py b/tests/runpod_smoke/runpodctl.py deleted file mode 100644 index 07e9dcd3..00000000 --- a/tests/runpod_smoke/runpodctl.py +++ /dev/null @@ -1,41 +0,0 @@ -"""Thin subprocess wrappers around the `runpodctl` binary. - -Other modules go through these so we have one place to handle the -"binary not on PATH" / timeout cases consistently. JSON-mode parsing is -also centralized here. -""" - -from __future__ import annotations - -import json -import subprocess -import sys - -from .log import log - - -def runpodctl(*args: str, timeout: int = 60) -> subprocess.CompletedProcess: - cmd = ["runpodctl", *args] - try: - return subprocess.run( - cmd, capture_output=True, text=True, timeout=timeout, check=False - ) - except FileNotFoundError: - log("runpodctl not found in PATH. Install it first.") - sys.exit(1) - except subprocess.TimeoutExpired: - return subprocess.CompletedProcess( - cmd, returncode=124, stdout="", stderr="timeout" - ) - - -def runpodctl_json(*args: str, timeout: int = 60): - """Invoke `runpodctl ... -o json` and parse stdout. Returns None on - non-zero exit or malformed JSON (callers must handle the None case).""" - proc = runpodctl(*args, "-o", "json", timeout=timeout) - if proc.returncode != 0: - return None - try: - return json.loads(proc.stdout) - except json.JSONDecodeError: - return None diff --git a/tests/test_images.py b/tests/test_images.py index 91a8b251..1ef06587 100755 --- a/tests/test_images.py +++ b/tests/test_images.py @@ -8,7 +8,8 @@ Usage: ./test_images.py [path/to/images.yaml] [group_filter] -Requirements: runpodctl (logged in), python3 >= 3.9 +Requirements: a RunPod API key (RUNPOD_API_KEY or ~/.runpod/config.toml), +python3 >= 3.9 """ from __future__ import annotations @@ -28,8 +29,9 @@ # our package imports work regardless of how the script was launched. sys.path.insert(0, os.path.dirname(os.path.realpath(__file__))) -from runpod_smoke import config +from runpod_smoke import api, config from runpod_smoke.instances import ( + cuda_axis_for, discover_gpu_catalog, discover_gpu_id_map, is_known_gpu, @@ -42,17 +44,20 @@ parse_manifest, ) from runpod_smoke.pod import discover_registry_auth -from runpod_smoke.runpodctl import runpodctl from runpod_smoke.runner import test_image -# Each entry: (image, group, instances-to-try). One pod is created per -# entry; the runner iterates instances internally until something settles. -Job = tuple[str, str, list[str]] +# Each entry: (image, group, instances-to-try, cuda_pin). One pod is created +# per entry; the runner iterates instances internally until something +# settles. `cuda_pin` is "" unless the group has a CUDA axis. +Job = tuple[str, str, list[str], str] -# Per-attempt outcome: (image, status, note, instance_used, host_cuda). A list -# avoids overwriting rows when `check_all_gpu` creates one job per GPU. -Result = tuple[str, str, str, str, str] +# Per-attempt outcome: +# (image, status, note, instance_used, host_cuda, requested_cuda) +# `requested_cuda` is the pinned version, kept separately so a SKIPped +# attempt still says WHICH cell of the matrix it belongs to. A list avoids +# overwriting rows when one GPU produces several jobs. +Result = tuple[str, str, str, str, str, str] # --------------------------------------------------------------------------- @@ -71,15 +76,15 @@ def _parse_args() -> tuple[Path, Optional[str]]: def _check_prereqs(manifest_path: Path) -> Optional[int]: """Return None on success, or an exit-code int on failure. Verifies: 1. the manifest file actually exists - 2. `runpodctl user` succeeds (the CLI has a valid API key) + 2. the REST API v2 accepts our key Anything else (GPU catalog, registry auth) is best-effort — the script degrades gracefully if those are missing.""" if not manifest_path.is_file(): log(f"Images manifest not found: {manifest_path}") return 1 - auth = runpodctl("user", timeout=15) - if auth.returncode != 0: - log("runpodctl is not authenticated. Run 'runpodctl doctor'.") + ok, detail = api.api_available() + if not ok: + log(detail) return 1 return None @@ -90,18 +95,18 @@ def _check_prereqs(manifest_path: Path) -> Optional[int]: def _init_gpu_catalog() -> None: - config.GPU_ID_MAP.update(discover_gpu_id_map()) - log(f"discovered {len(config.GPU_ID_MAP)} GPU types from runpodctl") - + # One request now serves both the id map and the price/vRAM filters. config.GPU_CATALOG.extend(discover_gpu_catalog()) + config.GPU_ID_MAP.update(discover_gpu_id_map()) if config.GPU_CATALOG: + with_cuda = sum(1 for g in config.GPU_CATALOG if g.get("cudaVersions")) log( - f"loaded GPU pricing for {len(config.GPU_CATALOG)} types " - "(GraphQL: gpuTypes)" + f"loaded {len(config.GPU_CATALOG)} GPU types from " + f"GET /v2/catalog/gpus ({with_cuda} reporting CUDA availability)" ) else: log( - "warn: no GPU pricing data — budget-based instance selection " + "warn: no GPU catalog — budget-based instance selection " "(max_price_per_hour) will be disabled. Set RUNPOD_API_KEY or " "ensure ~/.runpod/config.toml has 'apikey'." ) @@ -150,6 +155,53 @@ def _coerce_ports(raw_ports: object, group: str) -> list[int]: return ports +def _apply_cuda_axis(group: str, raw: object) -> None: + """Read the `cuda_versions:` manifest field for one group. + + Accepts the literal `all` (every version the GPU reports capacity for) + or a list of exact X.Y versions. Enabling it turns each candidate GPU + into one job per version and pins the host with + `gpu.allowedCudaVersions`, which the API matches exactly — so a + `min_cuda_version` floor on the same group would be both redundant and + rejected, and is dropped here. + """ + if raw is None or raw == "": + return + entries = raw if isinstance(raw, list) else [raw] + wants_all = any(str(e).strip().strip('"\'').lower() == "all" for e in entries) + versions: list[str] = [] + if not wants_all: + for entry in entries: + normalized = _normalize_cuda_version(entry) + if normalized: + versions.append(normalized) + else: + log( + f"warn: group '{group}': cuda_versions entry {entry!r} " + "is not an X.Y version — skipping" + ) + if not (wants_all or versions): + return + if wants_all: + config.GROUP_CUDA_ALL[group] = True + log( + f"group '{group}': cuda_versions=all (one job per GPU x every " + "CUDA version that GPU reports capacity for)" + ) + else: + config.GROUP_CUDA_VERSIONS[group] = versions + log( + f"group '{group}': cuda_versions={versions} " + "(pinned via gpu.allowedCudaVersions, exact match)" + ) + if config.GROUP_MIN_CUDA.pop(group, None): + log( + f"group '{group}': dropped min_cuda_version — a CUDA axis pins " + "exact versions and the API rejects both fields together", + indent=1, + ) + + def _apply_manifest_overrides(manifest: dict[str, dict]) -> None: """Populate the per-group dicts on `config` that `pod.create_pod` and `runner.test_pair` consult at run-time: `GROUP_MIN_CUDA` (fallback @@ -185,6 +237,8 @@ def _apply_manifest_overrides(manifest: dict[str, dict]) -> None: f"group '{grp}': test_ports={ports} " "(expose as /http, probe public proxy first)" ) + for grp, contents in manifest.items(): + _apply_cuda_axis(grp, contents.get("cuda_versions")) for grp, contents in manifest.items(): if _normalize_bool(contents.get("check_all_gpu")): config.GROUP_CHECK_ALL_GPU[grp] = True @@ -283,17 +337,85 @@ def _build_jobs( "'check_all_gpu:' produced candidates)" ) for img in contents.get("images", []): - results.append((img, "SKIP", "no instances configured", "", "")) + results.append((img, "SKIP", "no instances configured", "", "", "")) continue check_all = config.GROUP_CHECK_ALL_GPU.get(group, False) + cuda_axis = bool( + config.GROUP_CUDA_ALL.get(group) + or config.GROUP_CUDA_VERSIONS.get(group) + ) for img in contents.get("images", []): - if check_all: - jobs.extend((img, group, [inst]) for inst in instances) + if cuda_axis and check_all: + jobs.extend(_cuda_matrix_jobs(img, group, instances, results)) + elif cuda_axis: + jobs.extend(_cuda_per_version_jobs(img, group, instances)) + elif check_all: + jobs.extend((img, group, [inst], "") for inst in instances) else: - jobs.append((img, group, instances)) + jobs.append((img, group, instances, "")) + return _cap_jobs(jobs) + + +def _cuda_matrix_jobs( + image: str, group: str, instances: list[str], results: list[Result], +) -> list[Job]: + """One job per (GPU, CUDA version) — the full matrix, for check_all_gpu. + + Versions with no free capacity are dropped rather than attempted: the + API matches `allowedCudaVersions` exactly and answers a full pool with a + capacity error, so trying them would just buy SKIP rows. A GPU that + reports nothing usable is recorded as a SKIP so it still shows up in the + matrix instead of vanishing. + """ + jobs: list[Job] = [] + for inst in instances: + versions = cuda_axis_for(group, inst) + if not versions: + results.append(( + image, "SKIP", + "no requested CUDA version has capacity on this GPU", + inst, "", "", + )) + continue + jobs.extend((image, group, [inst], v) for v in versions) return jobs +def _cuda_per_version_jobs( + image: str, group: str, instances: list[str], +) -> list[Job]: + """One job per CUDA version, each keeping the full candidate list. + + Without `check_all_gpu` the contract is "try candidates until one + passes", and adding a CUDA axis shouldn't silently turn that into a + full product — that would multiply a budget-filtered pool of ~20 cards + into ~40 pods. So each version gets one job whose candidates are the + GPUs that actually offer it, and the runner short-circuits on the first + PASS exactly as it does without the axis. + """ + per_version: dict[str, list[str]] = {} + for inst in instances: + for version in cuda_axis_for(group, inst): + per_version.setdefault(version, []).append(inst) + return [ + (image, group, candidates, version) + for version, candidates in sorted(per_version.items(), reverse=True) + ] + + +def _cap_jobs(jobs: list[Job]) -> list[Job]: + """Enforce MAX_CUDA_COMBOS so a stray sweep can't run for a day.""" + if len(jobs) <= config.MAX_CUDA_COMBOS: + return jobs + log( + f"warn: {len(jobs)} jobs exceeds MAX_CUDA_COMBOS=" + f"{config.MAX_CUDA_COMBOS} — dropping the last " + f"{len(jobs) - config.MAX_CUDA_COMBOS}. Narrow the sweep with " + "min-vram-gb / exclude-instances, or raise the cap deliberately." + ) + return jobs[:config.MAX_CUDA_COMBOS] + + # --------------------------------------------------------------------------- # Job execution # --------------------------------------------------------------------------- @@ -303,25 +425,28 @@ def _run_jobs_serial(jobs: list[Job], results: list[Result]) -> None: """Single-threaded run — no worker tags, simpler logs, group-header banner each time the group changes.""" current_group: Optional[str] = None - for img, group, instances in jobs: + for img, group, instances, cuda_pin in jobs: if group != current_group: print() log(f"---------- group: {group} ----------") current_group = group - status, note, instance, host_cuda = test_image(img, instances, group) - results.append((img, status, note, instance, host_cuda)) + status, note, instance, host_cuda = test_image( + img, instances, group, cuda_pin + ) + results.append((img, status, note, instance, host_cuda, cuda_pin)) def _run_one_tagged_job(job: Job) -> Result: """ThreadPool worker. The W tag is assigned to the THREAD (not the job), so e.g. with 5 jobs and 3 workers you still see only W1/W2/W3, each handling 1-2 jobs sequentially.""" - img, grp, insts = job + img, grp, insts, cuda_pin = job ensure_worker_tag() - log(f"start [group={grp}] image={img}") - status, note, instance, host_cuda = test_image(img, insts, grp) - log(f"done [group={grp}] image={img} -> {status}") - return img, status, note, instance, host_cuda + pin_note = f" cuda={cuda_pin}" if cuda_pin else "" + log(f"start [group={grp}] image={img}{pin_note}") + status, note, instance, host_cuda = test_image(img, insts, grp, cuda_pin) + log(f"done [group={grp}] image={img}{pin_note} -> {status}") + return img, status, note, instance, host_cuda, cuda_pin def _run_jobs_parallel(jobs: list[Job], results: list[Result]) -> None: @@ -378,6 +503,38 @@ def _md_cell(value: str) -> str: return (value or "").replace("|", "\\|").replace("\n", " ") or "—" +_CELL_ICON = {"PASS": "✅", "FAIL": "❌", "SKIP": "⚠️"} + + +def _emit_cuda_pivot(results: list[Result]) -> list[str]: + """GPU-by-CUDA pivot table, or [] when no CUDA axis was requested. + + A flat list is unreadable at 34+ rows, and the whole point of the axis + is comparing one GPU across versions — so rows are GPUs, columns are + CUDA versions, and a blank cell means that pairing was never attempted + (the catalog reported no capacity for it). + """ + rows = [r for r in results if r[5]] + if not rows: + return [] + versions = sorted( + {r[5] for r in rows}, + key=lambda v: tuple(int(p) for p in v.split(".")) if "." in v else (0,), + ) + gpus = sorted({r[3] for r in rows}) + cell: dict[tuple[str, str], str] = {} + for _img, status, _note, inst, _host, req in rows: + cell[(inst, req)] = _CELL_ICON.get(status, status) + out = ["", "### GPU x CUDA", ""] + out.append("| GPU | " + " | ".join(f"CUDA {v}" for v in versions) + " |") + out.append("|" + "|".join(["---"] * (len(versions) + 1)) + "|") + for gpu in gpus: + cells = [cell.get((gpu, v), "·") for v in versions] + out.append(f"| {_md_cell(gpu)} | " + " | ".join(cells) + " |") + out += ["", "✅ pass · ❌ fail · ⚠️ skip · · not offered / no capacity", ""] + return out + + def _emit_step_summary(results: list[Result], counts: dict[str, int]) -> None: """Append the matrix to $GITHUB_STEP_SUMMARY as a markdown table. @@ -401,16 +558,17 @@ def _emit_step_summary(results: list[Result], counts: dict[str, int]) -> None: ] if single: lines += [f"Image: `{single}`", ""] + lines += _emit_cuda_pivot(results) lines.append("| " + " | ".join(head) + " |") lines.append("|" + "|".join(["---"] * len(head)) + "|") for want in ("FAIL", "SKIP", "PASS"): - for img, status, note, instance, host_cuda in results: + for img, status, note, instance, host_cuda, req_cuda in results: if status != want: continue row = [ _STATUS_ICON.get(status, status), _md_cell(instance), - _md_cell(host_cuda), + _md_cell(host_cuda or req_cuda), _md_cell(note), ] if not single: @@ -438,9 +596,10 @@ def _write_results_json(results: list[Result], counts: dict[str, int]) -> None: "status": status, "instance": instance, "cuda": host_cuda, + "requested_cuda": req_cuda, "note": note, } - for img, status, note, instance, host_cuda in results + for img, status, note, instance, host_cuda, req_cuda in results ], } try: @@ -478,7 +637,7 @@ def _print_summary(results: list[Result]) -> int: print(" SUMMARY ".center(84, "=")) print("=" * 84) counts: dict[str, int] = defaultdict(int) - for _img, status, _note, _instance, _host_cuda in results: + for _img, status, _note, _instance, _host_cuda, _req in results: counts[status] += 1 print( f"totals: {counts['PASS']} PASS, " @@ -486,9 +645,9 @@ def _print_summary(results: list[Result]) -> int: f"{counts['SKIP']} SKIP\n" ) for want in ("FAIL", "SKIP", "PASS"): - for img, status, note, instance, host_cuda in results: + for img, status, note, instance, host_cuda, req_cuda in results: line = _format_result_line( - want, img, status, note, instance, host_cuda + want, img, status, note, instance, host_cuda or req_cuda ) if line is not None: print(line) From bdb883a2412d6b57520b1145733cbe240d52fd68 Mon Sep 17 00:00:00 2001 From: chmokachka Date: Mon, 31 Aug 2026 14:18:00 +0300 Subject: [PATCH 18/33] feat: check all gpu --- .github/actions/smoke-test/action.yml | 22 +++++++++++++---- .github/scripts/generate_test_manifest.py | 17 +++++++++++--- .github/workflows/_tmp-gpu-compat-probe.yml | 26 +++++++++++---------- 3 files changed, 45 insertions(+), 20 deletions(-) diff --git a/.github/actions/smoke-test/action.yml b/.github/actions/smoke-test/action.yml index 44849232..8285912f 100644 --- a/.github/actions/smoke-test/action.yml +++ b/.github/actions/smoke-test/action.yml @@ -33,9 +33,13 @@ inputs: into the pod for the GPU/CUDA functional check and log capture. required: true budget-usd-per-hour: - description: "Max USD/hr for GPU instance selection" + description: | + Max USD/hr for GPU instance selection. Empty (default) means the + generator's own 1.0 default for budget mode, and NO price filter at + all under check-all-gpu — a matrix run is meant to cover every GPU. + Set it explicitly to trim an expensive matrix. required: false - default: "1.0" + default: "" min-vram-gb: description: "Min vRAM (GB) for GPU instance selection" required: false @@ -121,8 +125,12 @@ inputs: default: "" check-all-gpu: description: | - Test every GPU matching the vendor/vRAM filters independently instead - of selecting by hourly budget. This can be expensive. + Test every matching GPU as its own job instead of walking the + candidate list and stopping at the first PASS. Without this, a run + proves "the image works somewhere"; with it you get a per-GPU matrix. + + Covers the whole catalog (subject to vendor/vRAM/exclude filters) + unless budget-usd-per-hour is set explicitly. This can be expensive. required: false default: "false" exclude-instances: @@ -325,6 +333,11 @@ runs: [[ -z "${line}" ]] && continue EXTRA_ARGS+=(--test-port "${line}") done <<< "${TEST_PORTS}" + # Same treatment as --min-cuda-version: an empty budget must not + # reach the generator, or check-all-gpu would always be price-filtered. + if [[ -n "${BUDGET}" ]]; then + EXTRA_ARGS+=(--budget "${BUDGET}") + fi # Only pass --min-cuda-version when the input is non-empty; the # generator treats empty as "no floor" and we want to keep the # argv free of stray empty strings so argparse doesn't choke. @@ -358,7 +371,6 @@ runs: python3 "${GITHUB_ACTION_PATH}/../../scripts/generate_test_manifest.py" \ --profile "${PROFILE}" \ --refs "${REFS}" \ - --budget "${BUDGET}" \ --min-vram-gb "${MIN_VRAM_GB}" \ --manufacturer "${MANUFACTURER}" \ "${EXTRA_ARGS[@]}" \ diff --git a/.github/scripts/generate_test_manifest.py b/.github/scripts/generate_test_manifest.py index 5c66e436..fd111b87 100755 --- a/.github/scripts/generate_test_manifest.py +++ b/.github/scripts/generate_test_manifest.py @@ -125,6 +125,7 @@ def build_groups( exclude_instances: list[str] | None = None, min_cuda_version: str | None = None, cuda_versions: list[str] | None = None, + explicit_budget: bool = False, ) -> dict: """Build the manifest dict for `profile`. @@ -165,6 +166,11 @@ def _decorate(body: dict, *, gpu_group: bool) -> dict: if gpu_group: if check_all_gpu: body["check_all_gpu"] = True + # A matrix run means EVERY GPU by default — no price filter. + # A budget is honoured only when the caller passed one + # explicitly, as a deliberate way to trim an expensive sweep. + if explicit_budget: + body["max_price_per_hour"] = budget else: body["max_price_per_hour"] = budget body["min_vram_gb"] = min_vram_gb @@ -234,8 +240,12 @@ def main() -> int: ap.add_argument( "--budget", type=float, - default=1.0, - help="Max USD/hr for GPU instance selection (default: 1.0)", + default=None, + help=( + "Max USD/hr for GPU instance selection (default: 1.0). Ignored " + "under --check-all-gpu unless passed explicitly, since a matrix " + "run is meant to cover every GPU." + ), ) ap.add_argument( "--min-vram-gb", @@ -323,7 +333,8 @@ def main() -> int: groups = build_groups( args.profile, refs, - budget=args.budget, + budget=1.0 if args.budget is None else args.budget, + explicit_budget=args.budget is not None, min_vram_gb=args.min_vram_gb, manufacturer=args.manufacturer, test_jupyter=args.test_jupyter, diff --git a/.github/workflows/_tmp-gpu-compat-probe.yml b/.github/workflows/_tmp-gpu-compat-probe.yml index 6d0f0e47..f58d9e9c 100644 --- a/.github/workflows/_tmp-gpu-compat-probe.yml +++ b/.github/workflows/_tmp-gpu-compat-probe.yml @@ -10,11 +10,12 @@ name: TEMP GPU compat probe # * the results JSON artifact carrying `requested_cuda` # * the whole pod lifecycle on REST API v2 (create / get / delete) # -# Deliberately the CHEAP mode: no check-all-gpu, so the axis produces one -# job per CUDA version rather than a full GPU x CUDA matrix, and each job -# short-circuits on the first card that passes. With the budget filter the -# candidate list is cheapest-first, so this lands on ~$0.24/hr cards — -# roughly $0.08 for the run instead of the ~32 pods a matrix would spend. +# A REAL matrix: check-all-gpu makes every card its own job, so there is no +# short-circuit and the pivot table actually has rows. Trimmed with an +# explicit budget-usd-per-hour — under check-all-gpu that cap is opt-in, and +# without it this sweep would pull in B300 at $7.89/hr and cost ~$10. +# $0.6/hr cap -> ~11 pods across ~7 GPUs, roughly $0.66. +# Raise or drop the cap to widen it. # ============================================================================ on: @@ -55,16 +56,17 @@ jobs: runpod-api-key: ${{ secrets.TESTING_RUNPOD_API_KEY }} ssh-private-key: ${{ secrets.TESTING_RUNPOD_SSH_PRIVATE_KEY }} - # THE THING UNDER TEST. Two versions -> two jobs. The tag says - # cu1281, so pinning 13.0 also proves the pin beats the - # tag-derived floor. - cuda-versions: "12.8, 13.0" + # THE THING UNDER TEST: every GPU x every CUDA version it reports + # capacity for. The tag says cu1281, so the 13.0 cells also prove + # the pin beats the tag-derived floor. + cuda-versions: all + check-all-gpu: "true" - # No check-all-gpu on purpose — see the header. - budget-usd-per-hour: "1.0" + # Opt-in cap so the matrix stays under a dollar — see the header. + # budget-usd-per-hour: "0.6" manufacturer: Nvidia min-vram-gb: "16" - max-parallel: "2" + max-parallel: "3" upload-results-json: "true" results-artifact-name: gpu-matrix-${{ github.run_id }} From fd3c8e6e0b61b45f7cff419e071d7ee9a6331938 Mon Sep 17 00:00:00 2001 From: chmokachka Date: Mon, 31 Aug 2026 16:27:54 +0300 Subject: [PATCH 19/33] feat: logging --- tests/runpod_smoke/runner.py | 20 +++++++-- tests/test_images.py | 81 +++++++++++++++++++++++++++++++----- 2 files changed, 87 insertions(+), 14 deletions(-) diff --git a/tests/runpod_smoke/runner.py b/tests/runpod_smoke/runner.py index 8e68d7dc..de25b609 100644 --- a/tests/runpod_smoke/runner.py +++ b/tests/runpod_smoke/runner.py @@ -176,6 +176,18 @@ def _create_pod_with_retries( ) +def _over_candidates(count: int) -> str: + """Suffix naming how many candidates were tried, or '' for exactly one. + + With a CUDA axis or check_all_gpu every job carries a single candidate, + and "no capacity on any of 1 candidate instance type(s)" reads like a + bug. Saying nothing is right there: the row already names the GPU. + """ + if count <= 1: + return "" + return f" on any of {count} candidate instance types" + + def _classify_non_running( state: str, detail: str, pod_id: str, image: str, ) -> _Outcome: @@ -578,9 +590,9 @@ def test_image( return ( "SKIP", ( - f"RunPod never assigned an SSH endpoint on " - f"{len(stuck_instances)} instance type(s) — likely a " - "scheduler issue, try again later" + "RunPod never assigned an SSH endpoint" + + _over_candidates(len(stuck_instances)) + + " — likely a scheduler issue, try again later" ), ", ".join(stuck_instances), "", @@ -588,7 +600,7 @@ def test_image( log(f"all {len(instances)} instances unavailable (no capacity)", indent=1) return ( "SKIP", - f"no capacity on any of {len(instances)} candidate instance type(s)", + "no capacity" + _over_candidates(len(instances)), ", ".join(unavailable_instances), "", ) diff --git a/tests/test_images.py b/tests/test_images.py index 1ef06587..9fc38408 100755 --- a/tests/test_images.py +++ b/tests/test_images.py @@ -32,6 +32,7 @@ from runpod_smoke import api, config from runpod_smoke.instances import ( cuda_axis_for, + cuda_versions_offered, discover_gpu_catalog, discover_gpu_id_map, is_known_gpu, @@ -371,9 +372,17 @@ def _cuda_matrix_jobs( for inst in instances: versions = cuda_axis_for(group, inst) if not versions: + offered = cuda_versions_offered(inst, only_available=False) + detail = ( + f"offers {', '.join(offered)} but none had capacity" + if offered else "reports no CUDA versions" + ) + # No version reached a pod-create, so requested_cuda stays empty + # and the pivot shows this GPU as an all-dots row. The note has + # to carry the scope, since the CUDA column has nothing to show. results.append(( image, "SKIP", - "no requested CUDA version has capacity on this GPU", + f"GPU not covered: {detail}", inst, "", "", )) continue @@ -403,6 +412,45 @@ def _cuda_per_version_jobs( ] +def _warn_unviable_cuda_axis( + manifest: dict[str, dict], resolved: dict[str, list[str]], +) -> None: + """Warn when a CUDA axis could never have produced a single job. + + Runs before any pod is created. Without this the run is silent: every + GPU turns into a SKIP row, `ON_SKIP=pass` keeps the job green, and a + sweep that tested nothing looks the same as one that passed. + + Only misconfiguration is reported. "Versions exist and match, but none + has capacity right now" is transient, already visible as per-GPU SKIP + rows, and would fire on healthy runs — so it stays silent here. + """ + for group in manifest: + candidates = resolved.get(group, []) + if not candidates: + continue + all_mode = config.GROUP_CUDA_ALL.get(group, False) + requested = config.GROUP_CUDA_VERSIONS.get(group) or [] + if not (all_mode or requested): + continue + offered: set[str] = set() + for inst in candidates: + offered |= set(cuda_versions_offered(inst, only_available=False)) + if not offered: + log( + f"::warning::group '{group}': cuda_versions is set but none " + f"of the {len(candidates)} candidate GPUs reports any CUDA " + "version, so no pod can be created. The axis only applies " + "to NVIDIA — drop cuda_versions for a ROCm/AMD sweep." + ) + elif requested and not set(requested) & offered: + log( + f"::warning::group '{group}': cuda_versions=" + f"{sorted(requested)} but the candidate GPUs only offer " + f"{sorted(offered)}, so nothing will be tested." + ) + + def _cap_jobs(jobs: list[Job]) -> list[Job]: """Enforce MAX_CUDA_COMBOS so a stray sweep can't run for a day.""" if len(jobs) <= config.MAX_CUDA_COMBOS: @@ -509,21 +557,28 @@ def _md_cell(value: str) -> str: def _emit_cuda_pivot(results: list[Result]) -> list[str]: """GPU-by-CUDA pivot table, or [] when no CUDA axis was requested. - A flat list is unreadable at 34+ rows, and the whole point of the axis + A flat list is unreadable at 30+ rows, and the whole point of the axis is comparing one GPU across versions — so rows are GPUs, columns are - CUDA versions, and a blank cell means that pairing was never attempted - (the catalog reported no capacity for it). + CUDA versions. + + Every resolved GPU gets a row, including ones where no version had + capacity and no pod was ever created. Those come out as a full row of + `·`, which is the honest answer: "not covered". Leaving them out would + make an untested GPU indistinguishable from one that doesn't exist. """ - rows = [r for r in results if r[5]] - if not rows: + attempted = [r for r in results if r[5]] + if not attempted: return [] versions = sorted( - {r[5] for r in rows}, + {r[5] for r in attempted}, key=lambda v: tuple(int(p) for p in v.split(".")) if "." in v else (0,), ) - gpus = sorted({r[3] for r in rows}) + # A comma in the label means a multi-candidate SKIP, which can't be a + # single row — the axis produces one instance per job, so this only + # guards against a non-axis group sneaking into the same run. + gpus = sorted({r[3] for r in results if r[3] and ", " not in r[3]}) cell: dict[tuple[str, str], str] = {} - for _img, status, _note, inst, _host, req in rows: + for _img, status, _note, inst, _host, req in attempted: cell[(inst, req)] = _CELL_ICON.get(status, status) out = ["", "### GPU x CUDA", ""] out.append("| GPU | " + " | ".join(f"CUDA {v}" for v in versions) + " |") @@ -531,7 +586,12 @@ def _emit_cuda_pivot(results: list[Result]) -> list[str]: for gpu in gpus: cells = [cell.get((gpu, v), "·") for v in versions] out.append(f"| {_md_cell(gpu)} | " + " | ".join(cells) + " |") - out += ["", "✅ pass · ❌ fail · ⚠️ skip · · not offered / no capacity", ""] + out += [ + "", + "✅ pass · ❌ fail · ⚠️ skip (pod attempted, no capacity) · " + "· not attempted (no capacity at planning time)", + "", + ] return out @@ -704,6 +764,7 @@ def main() -> int: resolved = _resolve_all_instances(manifest) _warn_unknown_instances(resolved) + _warn_unviable_cuda_axis(manifest, resolved) _log_budget_picks(manifest, resolved) results: list[Result] = [] From ff58bc1baea2c8783b277f88f8acd76e748c0e03 Mon Sep 17 00:00:00 2001 From: chmokachka Date: Mon, 31 Aug 2026 17:15:23 +0300 Subject: [PATCH 20/33] feat: cleaned up and documented --- tests/README.md | 232 ++++++++++++++++++++++++++--------- tests/runpod_smoke/checks.py | 9 -- tests/runpod_smoke/pod.py | 90 -------------- tests/runpod_smoke/runner.py | 5 +- 4 files changed, 177 insertions(+), 159 deletions(-) diff --git a/tests/README.md b/tests/README.md index f2d5c683..a6909162 100644 --- a/tests/README.md +++ b/tests/README.md @@ -22,8 +22,8 @@ tests/ ├── config.py ← env vars, sentinels, shared mutable state ├── log.py ← thread-tagged logging ├── manifest.py ← parser + value normalizers - ├── runpodctl.py ← subprocess wrappers around the `runpodctl` binary - ├── instances.py ← GPU catalog, budget resolution, exclude filter, CUDA detection + ├── api.py ← REST API v2 client: auth, requests, error classification + ├── instances.py ← GPU/CPU catalog, budget resolution, exclude filter, CUDA axis ├── pod.py ← pod create/lifecycle/signals, registry auth ├── checks.py ← SSH/proxy checks, CUDA functional check, REST API v2 log diagnostics ├── comfyui.py ← ComfyUI proxy, model, workflow, and PNG checks @@ -34,32 +34,37 @@ tests/ ## Prerequisites 1. **Python ≥ 3.9** (stdlib only — no pip install needed). -2. **`runpodctl` 2.3.0+ on `$PATH`**, authenticated: +2. **A RunPod API key** with **pod-management** permissions, in either + `RUNPOD_API_KEY` or `~/.runpod/config.toml` as `apikey = '...'`: ```bash - runpodctl config --apiKey - runpodctl user # smoke test — should print your account info + export RUNPOD_API_KEY= ``` - The API key needs **pod-management** permissions. You can find / - generate one at . + Generate one at . There is + no CLI dependency — everything goes through REST API v2 (see + `runpod_smoke/api.py`). The key is validated at startup with + `GET /v2/account/ssh-keys`; a bad key fails fast before any pod is + created. 3. **SSH key registered on your RunPod account.** `test_images.py` probes every pod over SSH for the real readiness signal and the - GPU/CUDA functional check. `runpodctl` writes a managed key pair on - first use; if you already have one in `~/.runpod/ssh/` you're set. - To use a different key, point `RUNPOD_SSH_KEY` at the private half - AND make sure the matching public half is registered at - . - -4. **(Recommended)** A Docker Hub registry auth registered with - `runpodctl registry add`. RunPod datacenters share an anonymous Hub - IP pool that hits the `toomanyrequests` rate limit fast — without - auth, parallel runs in particular will produce a wave of - "image pull backoff" failures that look like image bugs but aren't. - The script auto-discovers the first entry from `runpodctl registry - list`; pin a specific one with `REGISTRY_AUTH_ID` or - `REGISTRY_AUTH_NAME`. + GPU/CUDA functional check. Register the public half at + and keep the + private half at one of `~/.runpod/ssh/runpodctl-ssh-key`, + `~/.ssh/runpodctl-ssh-key`, or point `RUNPOD_SSH_KEY` at it. + + The private key file must be mode `600` — OpenSSH refuses + group/world-readable keys, and the SSH probe will fail every pod + with no obvious reason. + +4. **(Recommended)** A Docker Hub registry credential on the account. + RunPod datacenters share an anonymous Hub IP pool that hits the + `toomanyrequests` rate limit fast — without auth, parallel runs in + particular produce a wave of "image pull backoff" failures that look + like image bugs but aren't. The script auto-discovers the first entry + from `GET /v2/registries`; pin a specific one with `REGISTRY_AUTH_ID` + or `REGISTRY_AUTH_NAME`. ## Quick start @@ -81,7 +86,7 @@ Run it: You should see, in order: -1. `discovered N GPU types from runpodctl` — startup catalog query +1. `loaded N GPU types from GET /v2/catalog/gpus` — startup catalog query 2. `using registry auth: …` — Docker Hub auth resolved (or a warning) 3. `==================== running 1 job(s) with MAX_PARALLEL=1 ===` 4. `attempt: CPU pod …` → `pod p-xxx created, waiting for RUNNING` @@ -112,8 +117,8 @@ runs this sequence and reports the outcome as soon as one step fails. | # | Step | Failure → | |---|------|---| -| 1 | `runpodctl pod create` (with `--gpu-id`, `--container-disk-in-gb`, `--ports`, registry auth, optional `--min-cuda-version`). Transient `5xx` / `Something went wrong` errors are retried silently up to `CREATE_RETRIES` with linear backoff. | `UNAVAILABLE` (no capacity for this instance type — try next) / `CREATE_FAIL` (bad image tag, registry auth, malformed request — any non-capacity, non-transient orchestrator error after retries are exhausted) | -| 2 | Poll `runpodctl pod get` and REST API v2 status until `ssh.ip` / `ssh.port` are assigned and one-shot `ssh root@ip -p port 'echo ready'` succeeds (SSH is the readiness signal; v2 surfaces terminal `ERROR` states the CLI misses) | `FAIL` on a terminal status; `STUCK` if no SSH endpoint within `CREATE_TIMEOUT` | +| 1 | `POST /v2/pods` with `gpu.id` (or an auto-picked `cpu.id` + `vcpuCount`), `disk`, `ports`, `startSsh`, registry credential, and either `gpu.minCudaVersion` or `gpu.allowedCudaVersions`. Transient failures (429, 5xx, transport) are retried up to `CREATE_RETRIES` with linear backoff. | `UNAVAILABLE` (no capacity — try next instance) / `CREATE_FAIL` (bad image tag, auth, malformed request — any non-capacity, non-transient error after retries) | +| 2 | Poll `GET /v2/pods/{id}` until `status` is `RUNNING`, `ssh.direct` is populated, and one-shot `ssh root@host -p port 'echo ready'` succeeds. SSH is the readiness signal; `status` is the real observed `PodStatus`, so terminal `EXITED`/`ERROR`/`TERMINATED` stop the poll immediately. | `FAIL` on a terminal status; `STUCK` if no SSH endpoint within `CREATE_TIMEOUT` | | 3 | **CUDA functional check** over SSH — see [Functional check](#functional-check). Image-driven: pytorch ref → `torch.cuda` + matmul; cuda/rocm ref → `nvidia-smi` + `nvcc`; neither → skip | `FAIL` (image is broken — stop iterating; another GPU won't help) | | 4 | **JupyterLab proxy-first check** (only when `test_jupyter: true`) — checks the public proxy; SSH probes `/api/status` only to diagnose a proxy failure | `FAIL` (Jupyter did not start, or is not exposed as `8888/http`) | | 5 | **Generic proxy-first port checks** (optional `test_ports`) — each service must return HTTP 200 through `https://-.proxy.runpod.net/`; SSH diagnoses failures | `FAIL` (service unavailable or incorrectly exposed) | @@ -123,13 +128,14 @@ runs this sequence and reports the outcome as soon as one step fails. | 9 | Sleep `DWELL_SEC`, re-probe SSH (catches "boots fine then crashes after 30s") | `FAIL` if SSH stops responding | | 10 | Re-probe ComfyUI `/system_stats` after dwell, then re-scan REST API v2 logs | `FAIL` if ComfyUI died during dwell, or logs contain a new error marker | | 11 | `dump_pod_logs` — API container logs, API system-log error markers, and GPU SMI over SSH | _(diagnostic only)_ | -| 12 | `runpodctl pod delete` (always — even on Ctrl-C / exception via `atexit` + signal handlers) | _(diagnostic only)_ | +| 12 | `DELETE /v2/pods/{id}` (always — even on Ctrl-C / exception via `atexit` + signal handlers). A 404 counts as success. | _(diagnostic only)_ | `test_image()` then iterates over the next instance candidate when the result was `UNAVAILABLE` or `STUCK`, and short-circuits on `PASS`, `FAIL`, or `CREATE_FAIL`. With `check_all_gpu: true`, each resolved GPU is instead run as an independent job, so the summary shows compatibility across -the full selected GPU set. +the full selected GPU set. Adding `cuda_versions:` splits it further — one +job per (GPU, CUDA version) — see [CUDA axis](#cuda-axis). ## Outcomes @@ -209,8 +215,8 @@ python3 ./tests/test_images.py ./tests/comfyui/images.example.yaml comfyui ``` If a pod gets stuck (rare), `Ctrl-C` cleans up — `SIGINT`/`SIGTERM` are -trapped and trigger `cleanup_all()`, which `runpodctl pod delete`s -every pod the script created. +trapped and trigger `cleanup_all()`, which deletes every pod the script +created. For pods the script misses, the real safety net is CI-side: a `cancel-in-progress` PR cancel can SIGKILL the runner before @@ -218,12 +224,10 @@ For pods the script misses, the real safety net is CI-side: a sweeps any `smoketest-*` pod older than ~60 min and deletes it. It reads each pod's age from its name, so it never touches a human's pod. -> Each `pod create` also passes `--terminate-after ` (an RFC3339 -> **datetime**, per `runpodctl pod create --help`) as a best-effort -> server-side backstop — but **don't rely on it**: runpodctl drops the -> flag entirely for CPU pods (the REST path has no such field), and even -> GPU pods were observed alive well past their deadline in testing. The -> reaper cron is the real safety net. +> **There is no server-side auto-terminate.** The old CLI accepted +> `--terminate-after `; REST API v2 has no equivalent field, so +> `reap-pods.yml` is now the *only* backstop against a leaked pod billing +> indefinitely. Keep that cron healthy. ## Manifest schema @@ -256,24 +260,123 @@ Field reference: | `min_vram_gb` | Extra filter for budget mode (default 0). | | `manufacturer` | `Nvidia` or `AMD` filter for budget mode (default: any). | | `exclude_instances` | fnmatch-style patterns (case-insensitive) subtracted from the candidate list AFTER `instances:` or budget selection. Useful for blocking known-bad host pairings without rewriting the whole list — e.g. `"*Blackwell*"` skips every Blackwell GPU (sm\_100 / sm\_120 are not in the kernel set of PyTorch ≤ 2.6 wheels). | -| `min_cuda_version` | `X.Y` string passed to `runpodctl pod create --min-cuda-version`. Only used as a **fallback** when the image tag itself doesn't encode a CUDA version (e.g. NGC `nvidia-pytorch:25.11`). Image tags like `cu1281` / `cuda1281` and `cuda13.0` always win. | +| `min_cuda_version` | `X.Y` floor sent as `gpu.minCudaVersion`. Used as a **fallback** when the image tag doesn't encode a CUDA version (e.g. NGC `nvidia-pytorch:25.11`); tags like `cu1281` / `cuda1281` / `cuda13.0` are parsed and win. Superseded by `cuda_versions` — the API rejects both fields on one request. | +| `cuda_versions` | `all`, or a list of exact `X.Y` versions. Turns on the [CUDA axis](#cuda-axis): each candidate GPU is tested once per version, pinned with `gpu.allowedCudaVersions`. Default: unset (no axis). | | `check_all_gpu` | `true` / `false` — use every catalog GPU matching `min_vram_gb` and `manufacturer`, with one independent result row per `(image, GPU)`. Mutually exclusive with budget selection in generated manifests and potentially expensive. Default: `false`. | | `test_jupyter` | `true` / `false` — when true, the pod is created with `JUPYTER_PASSWORD=admin` in env and HTTP port 8888 exposed, then the script SSHes in and verifies JupyterLab is actually listening. Use for groups whose images use `container-template/start.sh` (`runpod/base`, `runpod/pytorch`, `runpod/autoresearch`, `rocm`). Skip for NGC `nvidia-pytorch` (different entrypoint). Default: `false`. | | `test_ports` | Optional list of HTTP ports. Each is exposed as `/http` and must return HTTP 200 through the RunPod public proxy. On a proxy failure, the test probes `127.0.0.1:` over SSH to distinguish a service startup failure from an exposure/configuration error. | | `test_comfyui` | `true` / `false` — exposes `8188/http` and runs a labelled proxy-first ComfyUI reachability check. After dwell it verifies `/system_stats` again because the container can survive a ComfyUI crash. Default: `false`. | | `test_comfyui_functional` | `true` / `false` — implies `test_comfyui`; downloads/verifies the configured model through ComfyUI-RunpodDirect, POSTs the workflow, waits for completion, then validates a non-empty PNG from `/view`. The ComfyUI workflow enables it for both PR and release runs. | -The `base_cpu` group is special: `runpodctl` 2.3.0 does not let us pick -a specific CPU flavor (`--gpu-id` is rejected for `--compute-type CPU`), -so the manifest needs ONLY an `images:` list for that group — no -`instances:` / `max_price_per_hour:` / `min_vram_gb:`. RunPod picks a -CPU flavor for us. +The `base_cpu` group is special: the manifest needs ONLY an `images:` +list for that group — no `instances:` / `max_price_per_hour:` / +`min_vram_gb:`. The flavor is chosen from `GET /v2/catalog/cpus` by +`instances.pick_cpu_flavor()` — cheapest per-vCPU flavor whose range +admits `CPU_VCPU_COUNT`, preferring better-reported availability. Pin one +with `CPU_FLAVOR_ID` if you need a specific tier. The functional workflow and model manifest live under `tests/comfyui/`. Set `COMFYUI_SAVE_DIR` to retain the validated PNG locally; the composite action uploads it when `save-comfyui-images: "true"`. +## CUDA axis + +`--min-cuda-version` / `gpu.minCudaVersion` is only a **floor**: the +scheduler may place the pod on any host at or above it. In practice that +means a `cu1281` image gets tested on whatever driver RunPod happens to +have free — observed values for the same GPU have ranged from 12.8 to +13.2 across two runs an hour apart. So a plain `PASS` says "works +somewhere", not "works on 12.8". + +`cuda_versions:` fixes that by pinning `gpu.allowedCudaVersions`, which +the API matches **exactly**: + +```yaml +base_gpu: + images: + - runpod/pytorch:1.2.0-cu1281-torch2121-ubuntu2404 + check_all_gpu: true + manufacturer: Nvidia + min_vram_gb: 16 + cuda_versions: all # or: a list of exact versions +``` + +```yaml + cuda_versions: + - "12.8" + - "13.0" +``` + +Two expansion modes, depending on `check_all_gpu`: + +| | one job per | short-circuit | +|---|---|---| +| `cuda_versions` + `check_all_gpu: true` | **(GPU, version)** — the full matrix | no | +| `cuda_versions` alone | **version**, candidates = GPUs offering it | yes, first PASS wins | + +The second mode exists so that adding an axis to a budget-filtered group +doesn't silently turn one pod into a product of ~20 cards × N versions. + +**Only pairings the catalog reports capacity for are attempted.** +`GET /v2/catalog/gpus?include=AVAILABILITY&product=POD` returns, per GPU, +`cudaVersions: [{version, available}]`. Because matching is exact, pinning +a version nobody reports yields a capacity error rather than a fallback — +so unavailable pairings are dropped at planning time instead of burning a +pod each. A GPU with no usable version is recorded as a SKIP row and +appears in the matrix as a full row of `·`. + +`MAX_CUDA_COMBOS` (default 120) caps the fan-out; jobs past the cap are +dropped with a warning. + +The axis is NVIDIA-only — AMD cards report no `cudaVersions`, and turning +it on for a ROCm sweep produces zero jobs plus a `::warning::` saying so. + +### Result matrix + +With an axis active, the job's step summary gains a pivot table on top of +the flat list: + +``` +### GPU x CUDA + +| GPU | CUDA 12.4 | CUDA 12.8 | CUDA 13.0 | CUDA 13.2 | +|-------------------|-----------|-----------|-----------|-----------| +| A100 SXM | ✅ | ✅ | ✅ | · | +| B200 | · | · | ✅ | · | +| PRO 6000 MIG 48GB | · | · | ✅ | ✅ | +| RTX A6000 | · | ✅ | ⚠️ | · | +| Tesla V100 | · | · | · | · | + +✅ pass · ❌ fail · ⚠️ skip (pod attempted, no capacity) · · not attempted +``` + +`⚠️` and `·` are different events: yellow means a pod was created for that +exact pairing and RunPod had no capacity; a dot means the pairing never +reached a create because the catalog said so up front. + +Both the table and the flat list are always written to +`$GITHUB_STEP_SUMMARY`, which is readable on the run page without +permission to download job logs. Set `SMOKE_RESULTS_JSON` (the composite +action does, behind `upload-results-json`) to also emit a machine-readable +report for diffing runs: + +```json +{ + "generated_at": "2026-08-31T10:03:03Z", + "totals": { "PASS": 28, "FAIL": 0, "SKIP": 25 }, + "results": [ + { "image": "…", "status": "PASS", "instance": "A40", + "cuda": "13.0", "requested_cuda": "13.0", "note": "" } + ] +} +``` + +`cuda` is what the host reported (`Pod.cudaVersion`); `requested_cuda` is +what we pinned. They differ only if the API ever stops honouring the pin, +which is worth knowing. + + ## Example manifest (the real one used in this repo) Lives outside the repo at `~/tmp/runpod-scripts/testing/images` — the @@ -358,10 +461,14 @@ pytorch: | `CLOUD_TYPE` | `SECURE` | `SECURE` or `COMMUNITY`. | | `DISK_GB` | `100` | Container disk size for GPU pods. | | `CPU_DISK_GB` | `20` | Container disk size for CPU pods. RunPod caps this per CPU flavor (20 GB on the cheapest, 30 GB on larger ones); 20 is the universal safe value. | -| `CPU_CANDIDATES` | `""` (uses `cpu-secure,cpu-community`) | CPU "instance candidates". `runpodctl pod create` doesn't accept `--vcpu` / `--mem` / `--cpu-flavor`, so we vary the axes it DOES expose for CPU: `--cloud-type` (SECURE vs COMMUNITY) and optional `--data-center-ids`. Each label becomes one candidate iterated by the same per-instance retry loop GPU groups use, so when SECURE is saturated COMMUNITY almost always has free CPU capacity. Format: `label:CLOUD[:DC1+DC2+…],label:CLOUD[:DC_CSV],…` (use `+` not `,` to separate DC ids inside one candidate so the outer csv stays unambiguous). CLOUD must be SECURE or COMMUNITY. Malformed entries are silently dropped; an empty/all-broken value falls back to the default 2-candidate list. | -| `RUNPOD_API_KEY` | _(from `~/.runpod/config.toml`)_ | Used for the GraphQL GPU pricing query. Set this in CI / containers without a config file. | +| `CPU_CANDIDATES` | `""` (uses `cpu-secure,cpu-community`) | CPU "instance candidates". The flavor comes from `pick_cpu_flavor()`; what varies between candidates is placement — cloud (SECURE vs COMMUNITY) and optional data centres. Each label becomes one candidate iterated by the same per-instance retry loop GPU groups use, so when SECURE is saturated COMMUNITY almost always has free CPU capacity. Format: `label:CLOUD[:DC1+DC2+…],label:CLOUD[:DC_CSV],…` (use `+` not `,` to separate DC ids inside one candidate so the outer csv stays unambiguous). CLOUD must be SECURE or COMMUNITY. Malformed entries are silently dropped; an empty/all-broken value falls back to the default 2-candidate list. | +| `RUNPOD_API_KEY` | _(from `~/.runpod/config.toml`)_ | Bearer token for every REST API v2 call. Set this in CI / containers without a config file. | +| `MAX_CUDA_COMBOS` | `120` | Cap on the (GPU, CUDA) fan-out. Jobs past it are dropped with a warning so a stray `cuda_versions: all` can't run for a day. | +| `CPU_VCPU_COUNT` | `4` | vCPUs requested for CPU pods. Must be a power of two inside the chosen flavor's `vcpu.min..max`. | +| `CPU_FLAVOR_ID` | _(empty)_ | Pin a CPU flavor (e.g. `cpu3c`) instead of auto-picking the cheapest fitting one from `GET /v2/catalog/cpus`. | +| `SMOKE_RESULTS_JSON` | _(empty)_ | Path to write the machine-readable result report to. Empty = don't write it. The markdown step summary is written regardless. | | `REGISTRY_AUTH_ID` | _(empty)_ | Explicit Docker Hub registry auth id to pass as `--registry-auth-id`. Overrides auto-discovery. | -| `REGISTRY_AUTH_NAME` | _(empty)_ | Display name to look up via `runpodctl registry list` when `REGISTRY_AUTH_ID` is not set. Falls back to the first entry. | +| `REGISTRY_AUTH_NAME` | _(empty)_ | Display name to look up via `GET /v2/registries` when `REGISTRY_AUTH_ID` is not set. Falls back to the first entry. | | `DWELL_SEC` | `60` | Extra seconds to wait after SSH becomes reachable, then re-probe SSH to catch containers that boot, accept SSH, then crash. Set 0 to skip the re-probe. | | `CREATE_TIMEOUT` | `600` | Max seconds to wait for SSH to become reachable. Raise for ROCm workflows (`create-timeout: "1200"` on the action) — the official `rocm/pytorch:*` base images are 30-50GB and routinely take 8-15 minutes to pull. | | `POLL_INTERVAL` | `10` | Poll cadence for SSH probes. | @@ -374,7 +481,7 @@ pytorch: | `LOG_API_TAIL` | `1000` | Number of historical lines to fetch from the REST API v2 log stream. | | `SYS_LOG_ERROR_PATTERN` | error/failure/crash regex | Case-insensitive regex for host-side REST API system-log diagnostics during a failed boot. | | `SSH_LOG_FETCH` | `1` | `1`/`0` — fetch only the GPU SMI diagnostic over SSH. Container logs use REST API v2. | -| `RUNPOD_SSH_KEY` | _(empty)_ | Path to private key matching the `PUBLIC_KEY` `runpodctl` injects into pods. Auto-discovered from common locations if not set. | +| `RUNPOD_SSH_KEY` | _(empty)_ | Path to the private key matching a public key registered on the account (`startSsh` injects those as `PUBLIC_KEY`). Auto-discovered from `~/.runpod/ssh/` and `~/.ssh/` if unset. Must be mode `600`. | | `JUPYTER_WAIT_TIMEOUT` | `30` | Seconds the in-pod Jupyter probe waits for `:8888` to bind. | | `JUPYTER_PROXY_TIMEOUT` | `60` | Seconds the proxy probe retries while RunPod's ingress registers the new pod. | | `PORT_WAIT_TIMEOUT` | `300` | Seconds the SSH diagnostic probe waits for a `test_ports` service to bind and return HTTP 200. | @@ -427,19 +534,17 @@ The composite action at [`.github/actions/smoke-test/action.yml`](../.github/actions/smoke-test/action.yml) wraps everything in this script needs for a clean CI run: -1. Installs the pinned `runpodctl` binary (`runpodctl-version`, - `runpodctl-sha256` inputs). -2. Configures the RunPod API key (`runpod-api-key` input) into - `~/.runpod/config.toml`. -3. Writes the `ssh-private-key` input to `~/.ssh/id_runpod` and exports +1. Exports the RunPod API key (`runpod-api-key` input) as + `RUNPOD_API_KEY` for every later step. No CLI to install. +2. Writes the `ssh-private-key` input to `~/.ssh/id_runpod` and exports `RUNPOD_SSH_KEY` so the in-pod CUDA probe and log fetch work. -4. Generates a manifest from the `image-refs` JSON array using +3. Generates a manifest from the `image-refs` JSON array using `.github/scripts/generate_test_manifest.py`, applying the `profile`, `budget-usd-per-hour`, `min-vram-gb`, `manufacturer`, `test-jupyter`, `test-ports`, `test-comfyui`, - `test-comfyui-functional`, `check-all-gpu`, and - `exclude-instances` inputs. -5. Invokes `python3 tests/test_images.py ` with + `test-comfyui-functional`, `check-all-gpu`, `cuda-versions`, + `min-cuda-version`, and `exclude-instances` inputs. +4. Invokes `python3 tests/test_images.py ` with `MAX_PARALLEL=`. A failed image makes the smoke-test action fail, which prevents a release from being created. @@ -456,9 +561,16 @@ Typical caller (from a per-image-family build workflow): min-vram-gb: "16" manufacturer: Nvidia test-jupyter: "true" - exclude-instances: | - *Blackwell* + exclude-instances: | # fnmatch on GPU DISPLAY names + B200 # ("*Blackwell*" matches nothing — + B300 # display names don't contain it) + RTX PRO * max-parallel: "3" + + # Opt in to the GPU x CUDA matrix (see "CUDA axis"): + # check-all-gpu: "true" + # cuda-versions: all + # upload-results-json: "true" ``` The full input reference lives in the action's own `description:` @@ -469,10 +581,12 @@ fields. | symptom in logs | likely cause | fix | |---|---|---| -| `runpodctl not found in PATH` | `runpodctl` binary missing | install from , put on `$PATH` | -| `runpodctl is not authenticated. Run 'runpodctl doctor'` | API key not configured or expired | `runpodctl config --apiKey ` | -| `warn: no GPU pricing data` | `RUNPOD_API_KEY` not set and no `~/.runpod/config.toml` | set `RUNPOD_API_KEY` or run `runpodctl config --apiKey` | -| `warn: no registry auth configured` | no Docker Hub auth registered | `runpodctl registry add` (paid Hub account strongly recommended for parallel runs) | +| `no RunPod API key — set RUNPOD_API_KEY…` | key missing from env and `~/.runpod/config.toml` | `export RUNPOD_API_KEY=` | +| `RunPod API rejected the key (HTTP 401…)` | key expired or lacks pod-management permission | regenerate at | +| `warn: no GPU catalog` | `GET /v2/catalog/gpus` failed — usually a bad/absent key | fix the key; budget and `check_all_gpu` selection are disabled without it | +| `warn: no registry auth configured` | no Docker Hub credential on the account | add one in the RunPod console (paid Hub account strongly recommended for parallel runs) | +| every pod SKIPs with an SSH failure | private key not mode `600`, or its public half isn't registered | `chmod 600 `; verify the fingerprint appears in `GET /v2/account/ssh-keys` | +| `cuda_versions is set but none of the N candidate GPUs reports any CUDA version` | CUDA axis on a ROCm/AMD sweep | drop `cuda_versions` — the axis is NVIDIA-only | | every group says `no capacity on any of N candidate instance type(s)` | budget too low / VRAM too high / region saturated | raise `max_price_per_hour`, drop `min_vram_gb`, or set explicit `instances:` | | only the `base_cpu` group says `no capacity` while GPU groups pass | the cloud(s) you target don't have CPU capacity right now | by default we already try SECURE then COMMUNITY. If both are full, add DC-pinned candidates: `CPU_CANDIDATES="cpu-secure:SECURE,cpu-community:COMMUNITY,cpu-eu:COMMUNITY:EU-RO-1+EU-NL-1,cpu-us:COMMUNITY:US-OR-1"` | | pod stays in `ssh endpoint not assigned yet` past `STALL_HINT_AFTER` | slow image pull or Docker Hub `toomanyrequests` | add registry auth, reduce `MAX_PARALLEL`, or wait 6 h for the Hub rate limit to reset | diff --git a/tests/runpod_smoke/checks.py b/tests/runpod_smoke/checks.py index d76140ff..70194910 100644 --- a/tests/runpod_smoke/checks.py +++ b/tests/runpod_smoke/checks.py @@ -602,15 +602,6 @@ def fetch_pod_logs_api( return lines -def pod_status_api(pod_id: str) -> Optional[str]: - """Return lifecycle status from `GET /v2/pods/{id}`, if available.""" - status, data = api.request("GET", f"/pods/{pod_id}", timeout=10) - if not (200 <= status < 300) or not isinstance(data, dict): - return None - value = data.get("status") - return value if isinstance(value, str) else None - - def system_log_errors(pod_id: str, max_lines: int = 20) -> Optional[list[str]]: """Return error-marker lines from the host-side REST system-log stream.""" lines = fetch_pod_logs_api(pod_id, source="system") diff --git a/tests/runpod_smoke/pod.py b/tests/runpod_smoke/pod.py index 3a702cf6..eac80a31 100644 --- a/tests/runpod_smoke/pod.py +++ b/tests/runpod_smoke/pod.py @@ -11,7 +11,6 @@ from __future__ import annotations import atexit -import re import signal import sys import threading @@ -24,25 +23,6 @@ from .log import log -# --------------------------------------------------------------------------- -# Error-classification regexes -# --------------------------------------------------------------------------- - -# Capacity / transient classification of API failures lives in -# api.classify_error. This one is different: it scans pod-get FIELDS for a -# container-runtime failure that happened before the pod reached RUNNING. -RUNTIME_ERROR_RE = re.compile( - r"toomanyrequests" - r"|rate\s+limit" - r"|failed\s+to\s+pull\s+image" - r"|error\s+creating\s+container" - r"|manifest\s+(?:unknown|not\s+found)" - r"|access\s+denied" - r"|no\s+such\s+image", - re.IGNORECASE, -) - - # --------------------------------------------------------------------------- # Active-pod tracking + signal-safe cleanup # --------------------------------------------------------------------------- @@ -296,76 +276,6 @@ def pod_state(pod_id: str) -> dict: } -def pod_status(pod_id: str) -> Optional[str]: - """Observed PodStatus, or None when the pod could not be read.""" - return pod_state(pod_id).get("status") - - -# Fields on the pod object that may carry a runtime error message directly. -_DIRECT_ERROR_FIELDS = ("lastError", "errorMessage", "statusMessage") - -# Same, on the nested `runtime` dict. -_RUNTIME_ERROR_FIELDS = ("lastError", "errorMessage", "statusMessage") - -# Fields whose value is a list of event objects (or strings); each item's -# `message` is harvested. -_EVENT_LIST_FIELDS = ("events", "statusEvents", "containerEvents") - -# Fields whose value is a single block of log lines that may contain -# pull-time errors not surfaced anywhere else. -_LOG_BLOCK_FIELDS = ("containerLogs", "logs") - - -def _collect_string_field(target: list[str], src: dict, key: str) -> None: - val = src.get(key) - if isinstance(val, str) and val: - target.append(val) - - -def _collect_event_messages(target: list[str], events: object) -> None: - if not isinstance(events, list): - return - for ev in events: - msg = ev.get("message") if isinstance(ev, dict) else str(ev) - if isinstance(msg, str) and msg: - target.append(msg) - - -def _gather_runtime_error_candidates(data: dict) -> list[str]: - """Walk every plausible place RunPod stuffs a runtime/pull error, - return a flat list of candidate lines. Doesn't filter — that's - `pod_runtime_error`'s job.""" - runtime = data.get("runtime") or {} - if not isinstance(runtime, dict): - runtime = {} - candidates: list[str] = [] - for key in _DIRECT_ERROR_FIELDS: - _collect_string_field(candidates, data, key) - for key in _RUNTIME_ERROR_FIELDS: - _collect_string_field(candidates, runtime, key) - for key in _EVENT_LIST_FIELDS: - _collect_event_messages(candidates, data.get(key) or runtime.get(key)) - for key in _LOG_BLOCK_FIELDS: - val = data.get(key) or runtime.get(key) - if isinstance(val, str): - candidates.extend(val.splitlines()) - return candidates - - -def pod_runtime_error(pod_id: str) -> Optional[str]: - """Inspect the pod object for container-runtime errors (pull failures, - bad images, etc.) that appear *before* the pod ever reaches RUNNING. - Returns a short error string or None.""" - state = pod_state(pod_id) - data = state.get("raw") - if not isinstance(data, dict): - return None - for line in _gather_runtime_error_candidates(data): - if RUNTIME_ERROR_RE.search(line): - return line.strip()[:300] - return None - - # --------------------------------------------------------------------------- # Wait for the pod to become reachable # --------------------------------------------------------------------------- diff --git a/tests/runpod_smoke/runner.py b/tests/runpod_smoke/runner.py index de25b609..11b30ee3 100644 --- a/tests/runpod_smoke/runner.py +++ b/tests/runpod_smoke/runner.py @@ -483,7 +483,10 @@ def test_pair( host = st.get("ssh_ip") or "" port = int(st.get("ssh_port") or 0) - host_cuda = fetch_pod_cuda_version(pod_id) + # pod_state already carries cudaVersion, so the common path costs no + # extra request. It is nullable until the scheduler has assigned a + # machine, hence the retrying fallback. + host_cuda = st.get("cuda_version") or fetch_pod_cuda_version(pod_id) if host_cuda: _set_host_cuda(host_cuda) log(f"host CUDA: {host_cuda}", indent=2) From 6af3bac317b941b86fc6a992c821db4535667b9d Mon Sep 17 00:00:00 2001 From: chmokachka Date: Tue, 1 Sep 2026 15:47:20 +0300 Subject: [PATCH 21/33] feat: cloud-type --- .github/actions/smoke-test/action.yml | 13 ++ .github/workflows/_tmp-gpu-compat-probe.yml | 76 --------- .github/workflows/gpu-compatibility.yml | 9 +- tests/README.md | 37 +++- tests/runpod_smoke/checks.py | 47 ++++- tests/runpod_smoke/config.py | 18 +- tests/runpod_smoke/instances.py | 47 ++++- tests/runpod_smoke/runner.py | 45 +++-- tests/test_images.py | 180 ++++++++++++++------ 9 files changed, 321 insertions(+), 151 deletions(-) delete mode 100644 .github/workflows/_tmp-gpu-compat-probe.yml diff --git a/.github/actions/smoke-test/action.yml b/.github/actions/smoke-test/action.yml index 8285912f..ef77add1 100644 --- a/.github/actions/smoke-test/action.yml +++ b/.github/actions/smoke-test/action.yml @@ -133,6 +133,16 @@ inputs: unless budget-usd-per-hour is set explicitly. This can be expensive. required: false default: "false" + cloud-type: + description: | + Cloud tier(s) to sweep: SECURE, COMMUNITY, a comma list of both, or + ALL. The catalog's CUDA versions and availability are scoped per tier + and the tiers do not nest — 16 of 47 GPUs are community-only (every + GeForce card, both V100s) while A40, L4, H100 SXM, B200 and B300 have + no community hosts — so full catalog coverage needs both. Extra tiers + multiply the pod count; MAX_CUDA_COMBOS caps the combined total. + required: false + default: "SECURE" exclude-instances: description: | Newline-separated list of fnmatch-style GPU-display-name patterns to @@ -383,6 +393,9 @@ runs: # RUNPOD_API_KEY and RUNPOD_SSH_KEY come in via $GITHUB_ENV from the # 'Configure RunPod credentials' and 'Write SSH private key' steps. MAX_PARALLEL: ${{ inputs.max-parallel }} + # Unknown tier names are dropped by config._parse_cloud_types, which + # falls back to SECURE, so the raw input is safe to propagate. + CLOUD_TYPE: ${{ inputs.cloud-type }} # ON_SKIP is validated by config._coerce_on_skip (unknown values # collapse to 'fail') so we just propagate the raw input. ON_SKIP: ${{ inputs.on-skip }} diff --git a/.github/workflows/_tmp-gpu-compat-probe.yml b/.github/workflows/_tmp-gpu-compat-probe.yml deleted file mode 100644 index f58d9e9c..00000000 --- a/.github/workflows/_tmp-gpu-compat-probe.yml +++ /dev/null @@ -1,76 +0,0 @@ -name: TEMP GPU compat probe - -# ============================================================================ -# TEMPORARY — DELETE BEFORE MERGING THIS PR. -# -# Verifies the new pieces end-to-end in CI: -# * the CUDA axis (`cuda-versions`) expanding into per-version jobs -# * `gpu.allowedCudaVersions` pinning the host to an exact version -# * the GPU x CUDA pivot in the job's step summary -# * the results JSON artifact carrying `requested_cuda` -# * the whole pod lifecycle on REST API v2 (create / get / delete) -# -# A REAL matrix: check-all-gpu makes every card its own job, so there is no -# short-circuit and the pivot table actually has rows. Trimmed with an -# explicit budget-usd-per-hour — under check-all-gpu that cap is opt-in, and -# without it this sweep would pull in B300 at $7.89/hr and cost ~$10. -# $0.6/hr cap -> ~11 pods across ~7 GPUs, roughly $0.66. -# Raise or drop the cap to widen it. -# ============================================================================ - -on: - pull_request: - paths: - - '.github/workflows/_tmp-gpu-compat-probe.yml' - - '.github/actions/smoke-test/**' - - '.github/scripts/generate_test_manifest.py' - - 'tests/**' - -permissions: - contents: read - -# Queue rather than cancel: a cancel SIGKILLs the runner before -# test_images.py can delete its pods, and v2 has no server-side -# terminate-after to fall back on. -concurrency: - group: tmp-gpu-compat-probe - cancel-in-progress: false - -jobs: - probe: - runs-on: blacksmith-4vcpu-ubuntu-2404 - timeout-minutes: 120 - steps: - - name: Checkout - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 - with: - fetch-depth: 0 - - - name: GPU compat probe - uses: ./.github/actions/smoke-test - with: - # torch 2.12 supports every current architecture, so a FAIL here - # means our plumbing, not the image. - image-refs: '["docker.io/runpod/pytorch:1.2.0-rc.162-cu1281-torch2121-ubuntu2404"]' - profile: gpu - runpod-api-key: ${{ secrets.TESTING_RUNPOD_API_KEY }} - ssh-private-key: ${{ secrets.TESTING_RUNPOD_SSH_PRIVATE_KEY }} - - # THE THING UNDER TEST: every GPU x every CUDA version it reports - # capacity for. The tag says cu1281, so the 13.0 cells also prove - # the pin beats the tag-derived floor. - cuda-versions: all - check-all-gpu: "true" - - # Opt-in cap so the matrix stays under a dollar — see the header. - # budget-usd-per-hour: "0.6" - manufacturer: Nvidia - min-vram-gb: "16" - max-parallel: "3" - - upload-results-json: "true" - results-artifact-name: gpu-matrix-${{ github.run_id }} - - test-jupyter: "true" - on-skip: pass - create-timeout: "600" diff --git a/.github/workflows/gpu-compatibility.yml b/.github/workflows/gpu-compatibility.yml index db87a862..aeef8b0b 100644 --- a/.github/workflows/gpu-compatibility.yml +++ b/.github/workflows/gpu-compatibility.yml @@ -10,7 +10,7 @@ on: workflow_dispatch: inputs: image: - description: "Image ref WITH tag, e.g. runpod/comfyui:1.0.7-comfyuiv0.30.0-cuda12.8" + description: "Image ref WITH tag, e.g. runpod/pytorch:1.1.0-cu1290-torch280-ubuntu2404-cluster" type: string required: true test-ports: @@ -175,6 +175,7 @@ jobs: echo "| --- | --- |" echo "| Image | \`${IMAGE}\` |" echo "| CUDA axis | ${CUDA:-none (floor from tag)} |" + echo "| Cloud tiers | SECURE + COMMUNITY |" echo "| Vendor | ${VENDOR} |" echo "| Min vRAM | ${MIN_VRAM} GB |" echo "| Ports | ${ports_1l:-none} |" @@ -201,6 +202,12 @@ jobs: min-vram-gb: ${{ inputs.min-vram-gb }} exclude-instances: ${{ steps.prep.outputs.exclude }} + # Both tiers, always: a compatibility sweep that silently skipped + # every community-only GPU wouldn't be a catalog sweep. Not an + # input because workflow_dispatch is capped at 10 of them; narrow + # with exclude-instances if you need one tier only. + cloud-type: "ALL" + # Empty = no axis; one job per GPU with the floor derived from the # image tag by instances.detect_cuda_version. cuda-versions: ${{ inputs.cuda-versions }} diff --git a/tests/README.md b/tests/README.md index a6909162..96888375 100644 --- a/tests/README.md +++ b/tests/README.md @@ -118,7 +118,7 @@ runs this sequence and reports the outcome as soon as one step fails. | # | Step | Failure → | |---|------|---| | 1 | `POST /v2/pods` with `gpu.id` (or an auto-picked `cpu.id` + `vcpuCount`), `disk`, `ports`, `startSsh`, registry credential, and either `gpu.minCudaVersion` or `gpu.allowedCudaVersions`. Transient failures (429, 5xx, transport) are retried up to `CREATE_RETRIES` with linear backoff. | `UNAVAILABLE` (no capacity — try next instance) / `CREATE_FAIL` (bad image tag, auth, malformed request — any non-capacity, non-transient error after retries) | -| 2 | Poll `GET /v2/pods/{id}` until `status` is `RUNNING`, `ssh.direct` is populated, and one-shot `ssh root@host -p port 'echo ready'` succeeds. SSH is the readiness signal; `status` is the real observed `PodStatus`, so terminal `EXITED`/`ERROR`/`TERMINATED` stop the poll immediately. | `FAIL` on a terminal status; `STUCK` if no SSH endpoint within `CREATE_TIMEOUT` | +| 2 | Poll `GET /v2/pods/{id}` until `status` is `RUNNING`, `ssh.direct` is populated, and one-shot `ssh root@host -p port 'echo ready'` succeeds. SSH is the readiness signal; `status` is the real observed `PodStatus`, so terminal `EXITED`/`ERROR`/`TERMINATED` stop the poll immediately. | `FAIL` on a terminal status, or when the system log shows a container-init rejection; `STUCK` if no SSH endpoint within `CREATE_TIMEOUT` | | 3 | **CUDA functional check** over SSH — see [Functional check](#functional-check). Image-driven: pytorch ref → `torch.cuda` + matmul; cuda/rocm ref → `nvidia-smi` + `nvcc`; neither → skip | `FAIL` (image is broken — stop iterating; another GPU won't help) | | 4 | **JupyterLab proxy-first check** (only when `test_jupyter: true`) — checks the public proxy; SSH probes `/api/status` only to diagnose a proxy failure | `FAIL` (Jupyter did not start, or is not exposed as `8888/http`) | | 5 | **Generic proxy-first port checks** (optional `test_ports`) — each service must return HTTP 200 through `https://-.proxy.runpod.net/`; SSH diagnoses failures | `FAIL` (service unavailable or incorrectly exposed) | @@ -148,6 +148,7 @@ The granular per-pod outcomes below collapse into them: | `PASS` | `PASS` | Image booted, all checks passed, survived dwell. | nothing | | `FAIL` | `FAIL` | Pod was created and the container itself proved broken (CUDA check failed, JupyterLab didn't start, crashed during dwell, etc.). Moving to another GPU won't help — the image is the problem. | fix the image | | `FAIL` | `CREATE_FAIL` | Pod-create returned a non-capacity, non-transient orchestrator error (bad image tag, registry auth, malformed request, missing CUDA version). | fix the manifest / image ref / auth | +| `FAIL` | `FAIL` (container init) | `nvidia-container-cli` rejected the container in the prestart hook — typically the image's `NVIDIA_REQUIRE_CUDA` floor is above the host driver, e.g. a `cu1290` image pinned to CUDA 12.4. Deterministic, so no other instance type is tried. | pin a CUDA version the image supports, or fix the image's requirement | | `SKIP` | all `UNAVAILABLE` | RunPod had no capacity on **any** candidate instance type. | retry later, expand `instances:` list, or raise `max_price_per_hour` | | `SKIP` | some `STUCK` + rest `UNAVAILABLE` | At least one instance was scheduled but RunPod never assigned an SSH endpoint within `CREATE_TIMEOUT` (slow pull / dead host). | retry later — usually transient | @@ -326,6 +327,35 @@ so unavailable pairings are dropped at planning time instead of burning a pod each. A GPU with no usable version is recorded as a SKIP row and appears in the matrix as a full row of `·`. +**`cudaVersions` is scoped to one cloud tier.** The catalog is fetched with +`cloud=$CLOUD_TYPE` (default `SECURE`), and a GPU with no host in that tier +comes back with an empty version list — 16 of 47 catalog GPUs are +community-only, including every GeForce card and both V100s. Pods are +created in the same tier, so those GPUs are genuinely untestable in that +run, and the SKIP note names the tier rather than claiming the API reports +nothing. The tiers do not nest (`A40`, `L4`, `H100 SXM`, `B200` and `B300` +have no community hosts), so covering the whole catalog needs both. + +`CLOUD_TYPE` therefore takes a list, or `ALL`, and one invocation sweeps +each tier: + +```sh +CLOUD_TYPE=ALL ON_SKIP=pass python3 tests/test_images.py +``` + +Planning happens once per tier, since availability, CUDA versions and +prices are all scoped to one; the resulting jobs then share a single worker +pool, and each job carries its tier through to `POST /v2/pods`. Practical +consequences: + +* `MAX_CUDA_COMBOS` caps the **combined** job count, so a two-tier sweep + can't silently double the pod count. +* The summary labels every row with its tier, and the step summary emits + one GPU × CUDA pivot per tier — the same (GPU, CUDA) pairing can be + tested in both, and one grid would drop one of the two outcomes. +* CPU groups are planned only once. Their candidate labels already encode a + tier each (see `CPU_CANDIDATES`), so they ignore this axis. + `MAX_CUDA_COMBOS` (default 120) caps the fan-out; jobs past the cap are dropped with a warning. @@ -458,7 +488,7 @@ pytorch: | var | default | description | |---|---|---| -| `CLOUD_TYPE` | `SECURE` | `SECURE` or `COMMUNITY`. | +| `CLOUD_TYPE` | `SECURE` | `SECURE`, `COMMUNITY`, both as a comma list, or `ALL`. Multiple tiers are planned and swept in one invocation — see [CUDA axis](#cuda-axis). | | `DISK_GB` | `100` | Container disk size for GPU pods. | | `CPU_DISK_GB` | `20` | Container disk size for CPU pods. RunPod caps this per CPU flavor (20 GB on the cheapest, 30 GB on larger ones); 20 is the universal safe value. | | `CPU_CANDIDATES` | `""` (uses `cpu-secure,cpu-community`) | CPU "instance candidates". The flavor comes from `pick_cpu_flavor()`; what varies between candidates is placement — cloud (SECURE vs COMMUNITY) and optional data centres. Each label becomes one candidate iterated by the same per-instance retry loop GPU groups use, so when SECURE is saturated COMMUNITY almost always has free CPU capacity. Format: `label:CLOUD[:DC1+DC2+…],label:CLOUD[:DC_CSV],…` (use `+` not `,` to separate DC ids inside one candidate so the outer csv stays unambiguous). CLOUD must be SECURE or COMMUNITY. Malformed entries are silently dropped; an empty/all-broken value falls back to the default 2-candidate list. | @@ -586,7 +616,8 @@ fields. | `warn: no GPU catalog` | `GET /v2/catalog/gpus` failed — usually a bad/absent key | fix the key; budget and `check_all_gpu` selection are disabled without it | | `warn: no registry auth configured` | no Docker Hub credential on the account | add one in the RunPod console (paid Hub account strongly recommended for parallel runs) | | every pod SKIPs with an SSH failure | private key not mode `600`, or its public half isn't registered | `chmod 600 `; verify the fingerprint appears in `GET /v2/account/ssh-keys` | -| `cuda_versions is set but none of the N candidate GPUs reports any CUDA version` | CUDA axis on a ROCm/AMD sweep | drop `cuda_versions` — the axis is NVIDIA-only | +| `cuda_versions is set but none of the N candidate GPUs reports any CUDA version in the SECURE cloud` | CUDA axis on a ROCm/AMD sweep, or every candidate lives in the other cloud tier | drop `cuda_versions` for ROCm — the axis is NVIDIA-only; otherwise rerun with the other `CLOUD_TYPE` | +| `GPU not covered: not offered in the SECURE cloud` | community-only GPU (all GeForce cards, both V100s, `A100 SXM 40GB`) — `cudaVersions` is scoped to `CLOUD_TYPE` | use `CLOUD_TYPE=ALL` to sweep both tiers in one run; the note says `(covered by the COMMUNITY pass)` when it already did | | every group says `no capacity on any of N candidate instance type(s)` | budget too low / VRAM too high / region saturated | raise `max_price_per_hour`, drop `min_vram_gb`, or set explicit `instances:` | | only the `base_cpu` group says `no capacity` while GPU groups pass | the cloud(s) you target don't have CPU capacity right now | by default we already try SECURE then COMMUNITY. If both are full, add DC-pinned candidates: `CPU_CANDIDATES="cpu-secure:SECURE,cpu-community:COMMUNITY,cpu-eu:COMMUNITY:EU-RO-1+EU-NL-1,cpu-us:COMMUNITY:US-OR-1"` | | pod stays in `ssh endpoint not assigned yet` past `STALL_HINT_AFTER` | slow image pull or Docker Hub `toomanyrequests` | add registry auth, reduce `MAX_PARALLEL`, or wait 6 h for the Hub rate limit to reset | diff --git a/tests/runpod_smoke/checks.py b/tests/runpod_smoke/checks.py index 70194910..0e0ddc70 100644 --- a/tests/runpod_smoke/checks.py +++ b/tests/runpod_smoke/checks.py @@ -611,6 +611,33 @@ def system_log_errors(pod_id: str, max_lines: int = 20) -> Optional[list[str]]: return [line for line in lines if pattern.search(line)][:max_lines] +# `nvidia-container-cli` aborts the prestart hook when the image's +# NVIDIA_REQUIRE_CUDA floor is above the host driver. +_HOST_INCOMPATIBLE_RE = re.compile( + r"nvidia-container-cli:[^\n]*requirement error" + r"|unsatisfied condition:\s*cuda", + re.IGNORECASE, +) +_CUDA_CONDITION_RE = re.compile( + r"unsatisfied condition:\s*(cuda\s*[<>=!]+\s*[\d.]+)", re.IGNORECASE +) + + +def host_incompatibility(sys_errors: Optional[list[str]]) -> str: + """Summarize a container-init rejection, or '' if there wasn't one. + + Unlike a dead host, this verdict is deterministic: the same image on the + same pinned CUDA is rejected by every host, so the caller must not retry + other instance types. + """ + for line in sys_errors or []: + if not _HOST_INCOMPATIBLE_RE.search(line): + continue + match = _CUDA_CONDITION_RE.search(line) + return match.group(1) if match else line.strip()[:200] + return "" + + _LOG_SCAN_ATTEMPTS = 3 _LOG_SCAN_RETRY_SLEEP_SEC = 10 @@ -709,12 +736,17 @@ def fetch_logs_via_ssh( return f"__SSH_FAILED__\nreturncode={r.returncode}\nstderr: {r.stderr.strip()[:400]}" -def dump_pod_logs(pod_id: str, image: str) -> None: - """Print metadata, API container logs, system errors, and GPU SMI.""" +def dump_pod_logs(pod_id: str, image: str) -> list[str]: + """Print metadata, API container logs, system errors, and GPU SMI. + + Returns the system-log error markers so a caller classifying the failure + can read them without fetching the stream a second time. + """ + sys_errors: list[str] = [] status, data = api.request("GET", f"/pods/{pod_id}", timeout=30) if not (200 <= status < 300) or not isinstance(data, dict): api.log_error("(could not fetch pod state)", status, data, indent=2) - return + return sys_errors ssh = data.get("ssh") or {} direct = ssh.get("direct") or {} proxy = ssh.get("proxy") or {} @@ -739,7 +771,7 @@ def dump_pod_logs(pod_id: str, image: str) -> None: for line in api_lines: log(f" {line}", indent=2) - sys_errors = system_log_errors(pod_id) + sys_errors = system_log_errors(pod_id) or [] if sys_errors: log( f"--- system-log error markers via API ({len(sys_errors)}) ---", @@ -751,17 +783,18 @@ def dump_pod_logs(pod_id: str, image: str) -> None: if not (host and port): log(" (no SSH endpoint yet — skipping GPU SMI fetch)", indent=2) log(f" inspect via UI: https://www.runpod.io/console/pods/{pod_id}", indent=2) - return + return sys_errors logs = fetch_logs_via_ssh(host, int(port), image) if logs is None: - return + return sys_errors log(f"--- GPU SMI via SSH (root@{host}:{port}) ---", indent=2) if logs.startswith("__SSH_FAILED__"): log(" SSH could not reach the pod:", indent=2) for line in logs.splitlines()[1:]: log(f" {line}", indent=2) log(f" inspect via UI: https://www.runpod.io/console/pods/{pod_id}", indent=2) - return + return sys_errors for line in logs.splitlines(): log(f" {line}", indent=2) + return sys_errors diff --git a/tests/runpod_smoke/config.py b/tests/runpod_smoke/config.py index 3cc0cae0..6d9f3764 100644 --- a/tests/runpod_smoke/config.py +++ b/tests/runpod_smoke/config.py @@ -19,7 +19,23 @@ # Pod / scheduling # --------------------------------------------------------------------------- -CLOUD_TYPE = os.environ.get("CLOUD_TYPE", "SECURE") +VALID_CLOUDS = ("SECURE", "COMMUNITY") + + +def _parse_cloud_types(raw: str) -> list[str]: + """One tier, a comma-separated list, or `ALL`. Unknown names are dropped.""" + wanted = [part.strip().upper() for part in raw.split(",") if part.strip()] + if "ALL" in wanted: + return list(VALID_CLOUDS) + ordered = [c for c in VALID_CLOUDS if c in wanted] + return ordered or ["SECURE"] + + +# The catalog's `cudaVersions` and availability are scoped to one tier and the +# tiers do not nest, so full coverage needs a planning pass each. CLOUD_TYPES +# is every requested tier; CLOUD_TYPE is the one being planned right now. +CLOUD_TYPES = _parse_cloud_types(os.environ.get("CLOUD_TYPE", "SECURE")) +CLOUD_TYPE = CLOUD_TYPES[0] DISK_GB = int(os.environ.get("DISK_GB", "100")) # CPU pods on RunPod cap container disk by flavor: the cheapest flavors # (cpu3c-2-4 and similar) reject >20 GB outright; larger ones cap at 30 GB. diff --git a/tests/runpod_smoke/instances.py b/tests/runpod_smoke/instances.py index e93ed48a..daf447d5 100644 --- a/tests/runpod_smoke/instances.py +++ b/tests/runpod_smoke/instances.py @@ -107,6 +107,10 @@ def discover_gpu_catalog() -> list[dict]: for gpu in data.get("gpus") or []: if not isinstance(gpu, dict): continue + # Present in neither tier means no pod can ever land on it — the + # catalog carries such a placeholder entry named 'unknown'. + if not (gpu.get("secure") or gpu.get("community")): + continue price = gpu.get("price") or {} cuda = [ cv.get("version") @@ -128,6 +132,11 @@ def discover_gpu_catalog() -> list[dict]: "availability": gpu.get("availability") or "", "cudaVersions": cuda, "cudaVersionsAvailable": cuda_available, + # Which tier the GPU exists in. `cudaVersions` is scoped to the + # requested cloud, so these say whether an empty list means + # "wrong tier" or "no capacity". + "secure": bool(gpu.get("secure")), + "community": bool(gpu.get("community")), }) return out @@ -156,11 +165,45 @@ def cuda_versions_offered(display_name: str, *, only_available: bool = True) -> CUDA sweep from being mostly wasted attempts. """ key = "cudaVersionsAvailable" if only_available else "cudaVersions" + entry = catalog_entry(display_name) + return list(entry.get(key) or []) if entry else [] + + +def catalog_entry(display_name: str) -> Optional[dict]: + """Catalog row for one GPU display name, or None if unknown.""" lowered = display_name.lower() for gpu in config.GPU_CATALOG: if (gpu.get("displayName") or "").lower() == lowered: - return list(gpu.get(key) or []) - return [] + return gpu + return None + + +def uncovered_reason(display_name: str) -> str: + """Why a GPU yielded no (GPU, CUDA) pairing to test. + + `cudaVersions` is scoped to `CLOUD_TYPE`, so an empty list has three + different causes needing three different actions: sweep the other tier, + accept that the vendor has no CUDA, or retry later. + """ + cloud = config.CLOUD_TYPE.upper() + entry = catalog_entry(display_name) + if entry is None: + return "not in the GPU catalog" + offered = list(entry.get("cudaVersions") or []) + if offered: + return f"offers {', '.join(offered)} but none had capacity" + other = "COMMUNITY" if cloud == "SECURE" else "SECURE" + if not entry.get(cloud.lower(), True): + if other in config.CLOUD_TYPES: + return f"not offered in the {cloud} cloud (covered by the {other} pass)" + return ( + f"not offered in the {cloud} cloud — " + f"rerun with CLOUD_TYPE={other} to cover it" + ) + manufacturer = (entry.get("manufacturer") or "").upper() + if manufacturer and manufacturer != "NVIDIA": + return f"no CUDA versions ({entry.get('manufacturer')} GPU)" + return f"no CUDA versions in the {cloud} cloud right now" def cuda_axis_for(group: str, display_name: str) -> list[str]: diff --git a/tests/runpod_smoke/runner.py b/tests/runpod_smoke/runner.py index 11b30ee3..b925fd29 100644 --- a/tests/runpod_smoke/runner.py +++ b/tests/runpod_smoke/runner.py @@ -22,6 +22,7 @@ cuda_check_command, dump_pod_logs, fetch_pod_cuda_version, + host_incompatibility, run_cuda_check, run_jupyter_check, run_jupyter_proxy_check, @@ -65,6 +66,7 @@ def _take_host_cuda() -> str: def _log_attempt_header( image: str, instance: str, group: str, cuda_pin: str = "", + cloud: str = "", ) -> tuple[bool, str]: """Log the per-attempt header line and resolve the gpu_id. @@ -92,8 +94,10 @@ def _log_attempt_header( # Same precedence as create_pod: explicit request wins, tag is fallback. cuda = config.GROUP_MIN_CUDA.get(group) or detect_cuda_version(image) cuda_note = f", min-cuda={cuda}" if cuda else "" + cloud_note = f", cloud={cloud}" if cloud and len(config.CLOUD_TYPES) > 1 else "" log( - f"attempt: instance='{instance}' (--gpu-id '{gpu_id}'){cuda_note}", + f"attempt: instance='{instance}' (--gpu-id '{gpu_id}')" + f"{cuda_note}{cloud_note}", indent=1, ) return False, gpu_id @@ -101,7 +105,7 @@ def _log_attempt_header( def _create_pod_with_retries( image: str, instance: str, gpu_id: str, is_cpu: bool, group: str, - cuda_pin: str = "", + cuda_pin: str = "", cloud: str = "", ) -> tuple[Optional[str], str, str]: """Drive `create_pod` through the transient-error retry budget. @@ -116,13 +120,15 @@ def _create_pod_with_retries( We back off and retry a few times before falling through to CREATE_FAIL. """ # CPU candidate (cloud_type, data_center_ids) is encoded in the - # instance label (see config.CPU_CANDIDATES). For GPU instances both - # overrides are absent → pod.create_pod falls back to the global - # config.CLOUD_TYPE and skips --data-center-ids. + # instance label (see config.CPU_CANDIDATES). GPU jobs carry the tier + # they were planned against, since a multi-tier sweep runs both from one + # pool and the global config.CLOUD_TYPE no longer identifies either. cpu_candidate = ( config.cpu_candidate_for(instance) if is_cpu else None ) - cloud_override = cpu_candidate.cloud_type if cpu_candidate else None + cloud_override = ( + cpu_candidate.cloud_type if cpu_candidate else (cloud or None) + ) dc_ids = cpu_candidate.data_center_ids if cpu_candidate else "" raw = "" for attempt in range(1, config.CREATE_RETRIES + 1): @@ -197,19 +203,32 @@ def _classify_non_running( scheduler/host issue, not the image: a different GPU type lands on a different host pool and usually works. Anything else (EXITED, TERMINATED, FAILED, RUNNING-then-died) is a container problem — the - image is broken, another GPU won't help.""" + image is broken, another GPU won't help. + + A container-init rejection overrides that heuristic. It looks identical + from the outside — no SSH, no RUNNING — but it is a verdict about the + image, so it must not be reported as a retryable host problem.""" st = pod_state(pod_id) ever_had_ssh = bool(st.get("ssh_ip") and st.get("ssh_port")) + # Dumped before the verdict so the classification can use its findings. + sys_errors = dump_pod_logs(pod_id, image) + blocker = host_incompatibility(sys_errors) + if blocker: + log( + f"{state.lower()} -- container init rejected the image " + f"({blocker}) -- FAIL (deterministic; not retrying other " + "instance types)", + indent=2, + ) + return "FAIL", f"container init rejected the image: {blocker}" if state == "TIMEOUT" and not ever_had_ssh: log( f"{state.lower()} -- {detail} -- STUCK (no SSH endpoint " "was ever assigned; trying next instance type)", indent=2, ) - dump_pod_logs(pod_id, image) return "STUCK", "" log(f"{state.lower()} -- {detail} -- FAIL", indent=2) - dump_pod_logs(pod_id, image) return "FAIL", f"pod entered {state} state: {detail}" @@ -431,6 +450,7 @@ def _run_post_dwell_steps( def test_pair( image: str, instance: str, group: str, cuda_pin: str = "", + cloud: str = "", ) -> _Outcome: """Returns (status, detail). Statuses: 'PASS' — image booted, CUDA check OK, survived dwell @@ -455,10 +475,10 @@ def test_pair( # Clear first so a label from a previous instance can't leak into an # attempt that never reaches the probe (UNAVAILABLE, STUCK). _set_host_cuda("") - is_cpu, gpu_id = _log_attempt_header(image, instance, group, cuda_pin) + is_cpu, gpu_id = _log_attempt_header(image, instance, group, cuda_pin, cloud) pod_id, early, early_detail = _create_pod_with_retries( - image, instance, gpu_id, is_cpu, group, cuda_pin, + image, instance, gpu_id, is_cpu, group, cuda_pin, cloud, ) if early: return early, early_detail @@ -526,6 +546,7 @@ def test_pair( def test_image( image: str, instances: list[str], group: str, cuda_pin: str = "", + cloud: str = "", ) -> tuple[str, str, str, str]: """Returns (status, note, instance_used, host_cuda). @@ -551,7 +572,7 @@ def test_image( for inst in instances: set_worker_context(inst) try: - result, detail = test_pair(image, inst, group, cuda_pin) + result, detail = test_pair(image, inst, group, cuda_pin, cloud) finally: set_worker_context(None) host_cuda = _take_host_cuda() diff --git a/tests/test_images.py b/tests/test_images.py index 9fc38408..a2c60f8a 100755 --- a/tests/test_images.py +++ b/tests/test_images.py @@ -37,6 +37,7 @@ discover_gpu_id_map, is_known_gpu, resolve_instances, + uncovered_reason, ) from runpod_smoke.log import ensure_worker_tag, log from runpod_smoke.manifest import ( @@ -48,17 +49,19 @@ from runpod_smoke.runner import test_image -# Each entry: (image, group, instances-to-try, cuda_pin). One pod is created -# per entry; the runner iterates instances internally until something -# settles. `cuda_pin` is "" unless the group has a CUDA axis. -Job = tuple[str, str, list[str], str] +# Each entry: (image, group, instances-to-try, cuda_pin, cloud). One pod is +# created per entry; the runner iterates instances internally until something +# settles. `cuda_pin` is "" unless the group has a CUDA axis. `cloud` is the +# tier the job was planned against — it must travel with the job, since the +# catalog is re-fetched per tier and jobs from both run in one pool. +Job = tuple[str, str, list[str], str, str] # Per-attempt outcome: -# (image, status, note, instance_used, host_cuda, requested_cuda) +# (image, status, note, instance_used, host_cuda, requested_cuda, cloud) # `requested_cuda` is the pinned version, kept separately so a SKIPped # attempt still says WHICH cell of the matrix it belongs to. A list avoids # overwriting rows when one GPU produces several jobs. -Result = tuple[str, str, str, str, str, str] +Result = tuple[str, str, str, str, str, str, str] # --------------------------------------------------------------------------- @@ -97,13 +100,16 @@ def _check_prereqs(manifest_path: Path) -> Optional[int]: def _init_gpu_catalog() -> None: # One request now serves both the id map and the price/vRAM filters. - config.GPU_CATALOG.extend(discover_gpu_catalog()) + # Availability and CUDA versions are scoped to config.CLOUD_TYPE, so this + # is re-run per tier; GPU ids are tier-independent and just get refreshed. + config.GPU_CATALOG[:] = discover_gpu_catalog() config.GPU_ID_MAP.update(discover_gpu_id_map()) if config.GPU_CATALOG: with_cuda = sum(1 for g in config.GPU_CATALOG if g.get("cudaVersions")) log( f"loaded {len(config.GPU_CATALOG)} GPU types from " - f"GET /v2/catalog/gpus ({with_cuda} reporting CUDA availability)" + f"GET /v2/catalog/gpus for cloud={config.CLOUD_TYPE} " + f"({with_cuda} reporting CUDA availability)" ) else: log( @@ -321,24 +327,36 @@ def _build_jobs( resolved: dict[str, list[str]], group_filter: Optional[str], results: list[Result], + cloud: str, ) -> list[Job]: """Flatten the manifest into a list of `(image, group, instances)` jobs that can run independently. Groups with no resolvable instances are recorded directly into `results` as SKIPs (caller handles the - summary print).""" + summary print). + + Called once per cloud tier; `cloud` is stamped on every job produced. + CPU groups are planned only in the first pass — their candidate labels + already carry a tier each (see `config.CPU_CANDIDATES`).""" jobs: list[Job] = [] + first_pass = cloud == config.CLOUD_TYPES[0] for group, contents in manifest.items(): if group_filter and group != group_filter: continue + if group in config.CPU_GROUP_NAMES and not first_pass: + continue instances = resolved.get(group, []) if not instances: + if not first_pass: + continue log( f"skipping group '{group}': no instances resolved " "(none of 'instances:', 'max_price_per_hour:' or " "'check_all_gpu:' produced candidates)" ) for img in contents.get("images", []): - results.append((img, "SKIP", "no instances configured", "", "", "")) + results.append( + (img, "SKIP", "no instances configured", "", "", "", cloud) + ) continue check_all = config.GROUP_CHECK_ALL_GPU.get(group, False) cuda_axis = bool( @@ -347,18 +365,21 @@ def _build_jobs( ) for img in contents.get("images", []): if cuda_axis and check_all: - jobs.extend(_cuda_matrix_jobs(img, group, instances, results)) + jobs.extend( + _cuda_matrix_jobs(img, group, instances, results, cloud) + ) elif cuda_axis: - jobs.extend(_cuda_per_version_jobs(img, group, instances)) + jobs.extend(_cuda_per_version_jobs(img, group, instances, cloud)) elif check_all: - jobs.extend((img, group, [inst], "") for inst in instances) + jobs.extend((img, group, [inst], "", cloud) for inst in instances) else: - jobs.append((img, group, instances, "")) - return _cap_jobs(jobs) + jobs.append((img, group, instances, "", cloud)) + return jobs def _cuda_matrix_jobs( image: str, group: str, instances: list[str], results: list[Result], + cloud: str, ) -> list[Job]: """One job per (GPU, CUDA version) — the full matrix, for check_all_gpu. @@ -372,26 +393,21 @@ def _cuda_matrix_jobs( for inst in instances: versions = cuda_axis_for(group, inst) if not versions: - offered = cuda_versions_offered(inst, only_available=False) - detail = ( - f"offers {', '.join(offered)} but none had capacity" - if offered else "reports no CUDA versions" - ) # No version reached a pod-create, so requested_cuda stays empty # and the pivot shows this GPU as an all-dots row. The note has # to carry the scope, since the CUDA column has nothing to show. results.append(( image, "SKIP", - f"GPU not covered: {detail}", - inst, "", "", + f"GPU not covered: {uncovered_reason(inst)}", + inst, "", "", cloud, )) continue - jobs.extend((image, group, [inst], v) for v in versions) + jobs.extend((image, group, [inst], v, cloud) for v in versions) return jobs def _cuda_per_version_jobs( - image: str, group: str, instances: list[str], + image: str, group: str, instances: list[str], cloud: str, ) -> list[Job]: """One job per CUDA version, each keeping the full candidate list. @@ -407,7 +423,7 @@ def _cuda_per_version_jobs( for version in cuda_axis_for(group, inst): per_version.setdefault(version, []).append(inst) return [ - (image, group, candidates, version) + (image, group, candidates, version, cloud) for version, candidates in sorted(per_version.items(), reverse=True) ] @@ -440,8 +456,10 @@ def _warn_unviable_cuda_axis( log( f"::warning::group '{group}': cuda_versions is set but none " f"of the {len(candidates)} candidate GPUs reports any CUDA " - "version, so no pod can be created. The axis only applies " - "to NVIDIA — drop cuda_versions for a ROCm/AMD sweep." + f"version in the {config.CLOUD_TYPE.upper()} cloud, so no " + "pod can be created. Either the candidates live in the other " + "cloud tier, or they are AMD — the axis only applies to " + "NVIDIA, so drop cuda_versions for a ROCm sweep." ) elif requested and not set(requested) & offered: log( @@ -473,28 +491,40 @@ def _run_jobs_serial(jobs: list[Job], results: list[Result]) -> None: """Single-threaded run — no worker tags, simpler logs, group-header banner each time the group changes.""" current_group: Optional[str] = None - for img, group, instances, cuda_pin in jobs: + for img, group, instances, cuda_pin, cloud in jobs: if group != current_group: print() log(f"---------- group: {group} ----------") current_group = group status, note, instance, host_cuda = test_image( - img, instances, group, cuda_pin + img, instances, group, cuda_pin, cloud ) - results.append((img, status, note, instance, host_cuda, cuda_pin)) + results.append((img, status, note, instance, host_cuda, cuda_pin, cloud)) + + +def _job_note(cuda_pin: str, cloud: str) -> str: + """Trailing detail for the start/done lines — only what varies.""" + parts = [] + if cuda_pin: + parts.append(f"cuda={cuda_pin}") + if len(config.CLOUD_TYPES) > 1: + parts.append(f"cloud={cloud}") + return (" " + " ".join(parts)) if parts else "" def _run_one_tagged_job(job: Job) -> Result: """ThreadPool worker. The W tag is assigned to the THREAD (not the job), so e.g. with 5 jobs and 3 workers you still see only W1/W2/W3, each handling 1-2 jobs sequentially.""" - img, grp, insts, cuda_pin = job + img, grp, insts, cuda_pin, cloud = job ensure_worker_tag() - pin_note = f" cuda={cuda_pin}" if cuda_pin else "" + pin_note = _job_note(cuda_pin, cloud) log(f"start [group={grp}] image={img}{pin_note}") - status, note, instance, host_cuda = test_image(img, insts, grp, cuda_pin) + status, note, instance, host_cuda = test_image( + img, insts, grp, cuda_pin, cloud + ) log(f"done [group={grp}] image={img}{pin_note} -> {status}") - return img, status, note, instance, host_cuda, cuda_pin + return img, status, note, instance, host_cuda, cuda_pin, cloud def _run_jobs_parallel(jobs: list[Job], results: list[Result]) -> None: @@ -527,7 +557,8 @@ def _run_jobs(jobs: list[Job], results: list[Result]) -> None: def _format_result_line(want: str, img: str, status: str, note: str, - instance: str, host_cuda: str = "") -> Optional[str]: + instance: str, host_cuda: str = "", + cloud: str = "") -> Optional[str]: """Format one row of the summary, or None when this result doesn't belong in the `want` bucket. CPU labels ('cpu-secure', 'cpu-community', …) are already human-readable, so they go to the summary verbatim. @@ -538,6 +569,10 @@ def _format_result_line(want: str, img: str, status: str, note: str, if status != want: return None label = f"{instance} - CUDA {host_cuda}" if instance and host_cuda else instance + # Only when the run swept more than one tier — otherwise it's noise on + # every single line. + if cloud and len(config.CLOUD_TYPES) > 1: + label = f"{label} - {cloud}" if label else cloud inst_str = f" [{label}]" if label else "" note_str = f" -- {note}" if note else "" return f" {want:6s} {img}{inst_str}{note_str}" @@ -555,6 +590,22 @@ def _md_cell(value: str) -> str: def _emit_cuda_pivot(results: list[Result]) -> list[str]: + """One GPU-by-CUDA pivot per cloud tier that produced results. + + A tier gets its own table rather than a column: the same (GPU, CUDA) + pairing can be tested in both tiers, and merging them into one grid + would silently drop one of the two outcomes. + """ + tiers = [c for c in config.CLOUD_TYPES if any(r[6] == c for r in results)] + if len(tiers) <= 1: + return _pivot_table(results, "") + out: list[str] = [] + for tier in tiers: + out += _pivot_table([r for r in results if r[6] == tier], tier) + return out + + +def _pivot_table(results: list[Result], tier: str) -> list[str]: """GPU-by-CUDA pivot table, or [] when no CUDA axis was requested. A flat list is unreadable at 30+ rows, and the whole point of the axis @@ -578,9 +629,10 @@ def _emit_cuda_pivot(results: list[Result]) -> list[str]: # guards against a non-axis group sneaking into the same run. gpus = sorted({r[3] for r in results if r[3] and ", " not in r[3]}) cell: dict[tuple[str, str], str] = {} - for _img, status, _note, inst, _host, req in attempted: + for _img, status, _note, inst, _host, req, _cloud in attempted: cell[(inst, req)] = _CELL_ICON.get(status, status) - out = ["", "### GPU x CUDA", ""] + heading = f"### GPU x CUDA — {tier}" if tier else "### GPU x CUDA" + out = ["", heading, ""] out.append("| GPU | " + " | ".join(f"CUDA {v}" for v in versions) + " |") out.append("|" + "|".join(["---"] * (len(versions) + 1)) + "|") for gpu in gpus: @@ -607,7 +659,10 @@ def _emit_step_summary(results: list[Result], counts: dict[str, int]) -> None: return images = {r[0] for r in results} single = next(iter(images)) if len(images) == 1 else "" + multi_cloud = len(config.CLOUD_TYPES) > 1 head = ["Status", "Instance", "CUDA", "Note"] + if multi_cloud: + head.insert(2, "Cloud") if not single: head.insert(1, "Image") lines = [ @@ -622,7 +677,7 @@ def _emit_step_summary(results: list[Result], counts: dict[str, int]) -> None: lines.append("| " + " | ".join(head) + " |") lines.append("|" + "|".join(["---"] * len(head)) + "|") for want in ("FAIL", "SKIP", "PASS"): - for img, status, note, instance, host_cuda, req_cuda in results: + for img, status, note, instance, host_cuda, req_cuda, cloud in results: if status != want: continue row = [ @@ -631,6 +686,8 @@ def _emit_step_summary(results: list[Result], counts: dict[str, int]) -> None: _md_cell(host_cuda or req_cuda), _md_cell(note), ] + if multi_cloud: + row.insert(2, _md_cell(cloud)) if not single: row.insert(1, f"`{img}`") lines.append("| " + " | ".join(row) + " |") @@ -655,11 +712,13 @@ def _write_results_json(results: list[Result], counts: dict[str, int]) -> None: "image": img, "status": status, "instance": instance, + "cloud": cloud, "cuda": host_cuda, "requested_cuda": req_cuda, "note": note, } - for img, status, note, instance, host_cuda, req_cuda in results + for img, status, note, instance, host_cuda, req_cuda, cloud + in results ], } try: @@ -697,7 +756,7 @@ def _print_summary(results: list[Result]) -> int: print(" SUMMARY ".center(84, "=")) print("=" * 84) counts: dict[str, int] = defaultdict(int) - for _img, status, _note, _instance, _host_cuda, _req in results: + for _img, status, _note, _instance, _host_cuda, _req, _cloud in results: counts[status] += 1 print( f"totals: {counts['PASS']} PASS, " @@ -705,9 +764,9 @@ def _print_summary(results: list[Result]) -> int: f"{counts['SKIP']} SKIP\n" ) for want in ("FAIL", "SKIP", "PASS"): - for img, status, note, instance, host_cuda, req_cuda in results: + for img, status, note, instance, host_cuda, req_cuda, cloud in results: line = _format_result_line( - want, img, status, note, instance, host_cuda or req_cuda + want, img, status, note, instance, host_cuda or req_cuda, cloud ) if line is not None: print(line) @@ -752,7 +811,6 @@ def main() -> int: if rc is not None: return rc - _init_gpu_catalog() _init_registry_auth() manifest = parse_manifest(manifest_path) @@ -762,17 +820,41 @@ def main() -> int: log(f"error: {exc}") return 1 - resolved = _resolve_all_instances(manifest) - _warn_unknown_instances(resolved) - _warn_unviable_cuda_axis(manifest, resolved) - _log_budget_picks(manifest, resolved) - results: list[Result] = [] - jobs = _build_jobs(manifest, resolved, group_filter, results) + jobs = _plan_all_clouds(manifest, group_filter, results) _run_jobs(jobs, results) return _print_summary(results) +def _plan_all_clouds( + manifest: dict[str, dict], + group_filter: Optional[str], + results: list[Result], +) -> list[Job]: + """Plan every requested cloud tier, then cap the combined job list. + + Planning is per tier because the catalog's CUDA versions, availability + and prices are all scoped to one; execution is shared so both tiers use + the same worker pool. MAX_CUDA_COMBOS applies to the total, so a + two-tier sweep can't quietly double the pod count. + """ + if len(config.CLOUD_TYPES) > 1: + log(f"cloud tiers to sweep, in order: {', '.join(config.CLOUD_TYPES)}") + jobs: list[Job] = [] + for cloud in config.CLOUD_TYPES: + config.CLOUD_TYPE = cloud + if len(config.CLOUD_TYPES) > 1: + print() + log(f"---------- planning cloud: {cloud} ----------") + _init_gpu_catalog() + resolved = _resolve_all_instances(manifest) + _warn_unknown_instances(resolved) + _warn_unviable_cuda_axis(manifest, resolved) + _log_budget_picks(manifest, resolved) + jobs += _build_jobs(manifest, resolved, group_filter, results, cloud) + return _cap_jobs(jobs) + + if __name__ == "__main__": sys.exit(main()) From 1639d181daa2aa2214f96de29330c7dd17152d44 Mon Sep 17 00:00:00 2001 From: chmokachka Date: Tue, 1 Sep 2026 21:02:06 +0300 Subject: [PATCH 22/33] feat: incompatibility fixes --- .github/actions/smoke-test/action.yml | 20 ++ .github/scripts/generate_test_manifest.py | 38 ++- .github/workflows/base.yml | 8 +- .github/workflows/gpu-compatibility.yml | 148 +++++++----- .../zz-temp-gpu-compat-check-all.yml | 202 ++++++++++++++++ .../zz-temp-gpu-compat-instances.yml | 219 ++++++++++++++++++ tests/README.md | 62 ++++- tests/gpu-compat.example.yaml | 29 +++ tests/runpod_smoke/checks.py | 74 +++++- tests/runpod_smoke/config.py | 7 + tests/runpod_smoke/pod.py | 138 ++++++++--- tests/runpod_smoke/runner.py | 38 +-- 12 files changed, 859 insertions(+), 124 deletions(-) create mode 100644 .github/workflows/zz-temp-gpu-compat-check-all.yml create mode 100644 .github/workflows/zz-temp-gpu-compat-instances.yml create mode 100644 tests/gpu-compat.example.yaml diff --git a/.github/actions/smoke-test/action.yml b/.github/actions/smoke-test/action.yml index ef77add1..e9eae929 100644 --- a/.github/actions/smoke-test/action.yml +++ b/.github/actions/smoke-test/action.yml @@ -133,6 +133,17 @@ inputs: unless budget-usd-per-hour is set explicitly. This can be expensive. required: false default: "false" + instances: + description: | + Newline-separated GPU display names, one per line, exactly as + `GET /v2/catalog/gpus` reports them ('RTX A4000', 'A100 SXM'). Emitted + as `instances:`, which wins over catalog selection — mutually exclusive + with check-all-gpu, and it makes budget-usd-per-hour / min-vram-gb / + manufacturer inert. Empty (default) leaves selection to the catalog. + The list is a fallback chain, not a matrix: one pod per image lands on + the first entry with capacity. One pod per GPU needs check-all-gpu. + required: false + default: "" cloud-type: description: | Cloud tier(s) to sweep: SECURE, COMMUNITY, a comma list of both, or @@ -310,6 +321,7 @@ runs: TEST_COMFYUI: ${{ inputs.test-comfyui }} TEST_COMFYUI_FUNCTIONAL: ${{ inputs.test-comfyui-functional }} CHECK_ALL_GPU: ${{ inputs.check-all-gpu }} + INSTANCES: ${{ inputs.instances }} EXCLUDE_INSTANCES: ${{ inputs.exclude-instances }} MIN_CUDA_VERSION: ${{ inputs.min-cuda-version }} CUDA_VERSIONS: ${{ inputs.cuda-versions }} @@ -378,6 +390,14 @@ runs: [[ -z "${line}" || "${line}" == \#* ]] && continue EXTRA_ARGS+=(--exclude-instance "${line}") done <<< "${EXCLUDE_INSTANCES}" + # Same treatment for INSTANCES. Display names contain spaces + # ('RTX A4000'), so each line must stay one argv element. + while IFS= read -r line; do + line="${line#"${line%%[![:space:]]*}"}" + line="${line%"${line##*[![:space:]]}"}" + [[ -z "${line}" || "${line}" == \#* ]] && continue + EXTRA_ARGS+=(--instance "${line}") + done <<< "${INSTANCES}" python3 "${GITHUB_ACTION_PATH}/../../scripts/generate_test_manifest.py" \ --profile "${PROFILE}" \ --refs "${REFS}" \ diff --git a/.github/scripts/generate_test_manifest.py b/.github/scripts/generate_test_manifest.py index fd111b87..0c794fa8 100755 --- a/.github/scripts/generate_test_manifest.py +++ b/.github/scripts/generate_test_manifest.py @@ -122,6 +122,7 @@ def build_groups( test_comfyui: bool = False, test_comfyui_functional: bool = False, check_all_gpu: bool = False, + instances: list[str] | None = None, exclude_instances: list[str] | None = None, min_cuda_version: str | None = None, cuda_versions: list[str] | None = None, @@ -158,12 +159,25 @@ def build_groups( CUDA 13.0 and refuses to run on hosts with a 12.x driver. """ exclude_instances = list(exclude_instances or []) + instances = list(instances or []) test_ports = list(test_ports or []) cuda_versions = list(cuda_versions or []) wants_all_cuda = any(v.strip().lower() == "all" for v in cuda_versions) + if instances and check_all_gpu: + raise ValueError( + "instances and check_all_gpu are mutually exclusive: an explicit " + "list wins over catalog selection, so passing both silently " + "ignores one of them" + ) + def _decorate(body: dict, *, gpu_group: bool) -> dict: - if gpu_group: + if gpu_group and instances: + # An explicit list wins over catalog selection in + # instances.resolve_instances, so the budget / vRAM / vendor + # filters would be dead weight in the manifest. + body["instances"] = list(instances) + elif gpu_group: if check_all_gpu: body["check_all_gpu"] = True # A matrix run means EVERY GPU by default — no price filter. @@ -317,6 +331,19 @@ def main() -> int: "gpu.allowedCudaVersions, so this supersedes --min-cuda-version." ), ) + ap.add_argument( + "--instance", + action="append", + default=[], + dest="instances", + metavar="DISPLAY_NAME", + help=( + "Test exactly this GPU display name (repeat for several). " + "Emitted as `instances:`, which wins over catalog selection — so " + "it cannot be combined with --check-all-gpu, and the budget / " + "vRAM / vendor filters no longer apply." + ), + ) ap.add_argument("--output", required=True, type=Path) args = ap.parse_args() @@ -330,6 +357,14 @@ def main() -> int: print("--refs must be a non-empty JSON array", file=sys.stderr) return 1 + if args.instances and args.check_all_gpu: + print( + "--instance and --check-all-gpu are mutually exclusive: an " + "explicit instance list wins over catalog selection", + file=sys.stderr, + ) + return 1 + groups = build_groups( args.profile, refs, @@ -342,6 +377,7 @@ def main() -> int: test_comfyui=args.test_comfyui, test_comfyui_functional=args.test_comfyui_functional, check_all_gpu=args.check_all_gpu, + instances=args.instances, exclude_instances=args.exclude_instance, min_cuda_version=(args.min_cuda_version or None), cuda_versions=args.cuda_versions, diff --git a/.github/workflows/base.yml b/.github/workflows/base.yml index 0035f4ca..9413dc42 100644 --- a/.github/workflows/base.yml +++ b/.github/workflows/base.yml @@ -451,8 +451,14 @@ jobs: runpod-api-key: ${{ secrets.TESTING_RUNPOD_API_KEY }} ssh-private-key: ${{ secrets.TESTING_RUNPOD_SSH_PRIVATE_KEY }} test-jupyter: true + # Patterns match catalog DISPLAY names, which never contain the + # word "Blackwell" — only the ids do. Enumerate the cards. exclude-instances: | - *Blackwell* + B200 + B300 + RTX 5090 + RTX PRO * + PRO 6000 MIG * RTX 2000 Ada max-parallel: "4" diff --git a/.github/workflows/gpu-compatibility.yml b/.github/workflows/gpu-compatibility.yml index aeef8b0b..e364dab0 100644 --- a/.github/workflows/gpu-compatibility.yml +++ b/.github/workflows/gpu-compatibility.yml @@ -1,16 +1,18 @@ name: GPU Compatibility Matrix -# Manual sweep for any image: boots it on EVERY GPU in the RunPod catalog -# independently and reports a per-GPU pass/fail matrix. +# Manual sweep for any image: boots it on every NVIDIA GPU in the RunPod +# catalog independently and reports a per-GPU pass/fail matrix. AMD is out +# of scope here — the CUDA axis has no meaning for it; use a local manifest. # -# COST: ~30 NVIDIA types x 5-10 min per pod, far longer with the ComfyUI -# functional check. Narrow with min-vram-gb / exclude-instances. +# COST: ~45 NVIDIA types x 5-10 min per pod, multiplied by the images and +# the CUDA axis, far longer with the ComfyUI functional check. Narrow with +# exclude-instances, or name exact GPUs in instances. on: workflow_dispatch: inputs: image: - description: "Image ref WITH tag, e.g. runpod/pytorch:1.1.0-cu1290-torch280-ubuntu2404-cluster" + description: "Image refs WITH tags, comma-separated. Every image is tested on every selected GPU, so N images means N times the pods." type: string required: true test-ports: @@ -43,24 +45,25 @@ on: type: boolean required: false default: false - manufacturer: - description: "GPU vendor to sweep (AMD for ROCm images)" + cloud: + description: "Cloud tier. ALL sweeps both — needed for full catalog coverage, since the tiers don't overlap." type: choice required: false - default: "Nvidia" + default: "SECURE" options: - - Nvidia - - AMD - min-vram-gb: - description: "Skip GPUs below this vRAM. 0 = sweep the whole catalog." - type: string + - SECURE + - COMMUNITY + - ALL + check-all-gpu: + description: "Sweep every GPU in the catalog. Mutually exclusive with 'instances'." + type: boolean required: false - default: "0" - max-parallel: - description: "Pods under test at once (each worker holds one pod — this is the cost throttle)" + default: true + instances: + description: "Exact GPUs, comma-separated (e.g. A100 SXM, RTX 4090) — a fallback chain, so one pod per image lands on the first one with capacity, NOT one pod per GPU. For a matrix over a subset, tick check-all-gpu and exclude the rest. Mutually exclusive with 'check-all-gpu'." type: string required: false - default: "2" + default: "" permissions: contents: read @@ -74,6 +77,12 @@ concurrency: jobs: compatibility: runs-on: blacksmith-4vcpu-ubuntu-2404 + env: + # Not a dispatch input: workflow_dispatch is capped at 10 and the GPU + # selection inputs are worth more. Parallelism doesn't change cost + # anyway (you pay per pod-minute) — it trades wall-clock against the + # risk of throttled concurrent image pulls. + MAX_PARALLEL: "4" # A full sweep outlives the 360-minute default. timeout-minutes: 720 steps: @@ -89,6 +98,7 @@ jobs: IMAGE: ${{ inputs.image }} PORTS: ${{ inputs.test-ports }} EXCLUDE: ${{ inputs.exclude-instances }} + INSTANCES: ${{ inputs.instances }} run: | set -euo pipefail @@ -110,35 +120,59 @@ jobs: } >> "$GITHUB_OUTPUT" } - IMAGE_TRIMMED=$(printf '%s' "${IMAGE}" | tr -d '[:space:]') - if [[ -z "${IMAGE_TRIMMED}" ]]; then + # Image refs never contain whitespace, so strip it all and split on + # commas. Every ref is validated against a charset before it goes + # into the JSON array, which is what keeps the hand-built JSON from + # being escapable. + IMAGES_CSV=$(printf '%s' "${IMAGE}" | tr -d '[:space:]') + if [[ -z "${IMAGES_CSV}" ]]; then echo "::error::image input is empty" exit 1 fi - if [[ "${IMAGE_TRIMMED}" != *:* ]]; then - echo "::error::image '${IMAGE_TRIMMED}' has no tag — pass an explicit tag, not a bare repository" - exit 1 - fi - printf 'image-refs=["%s"]\n' "${IMAGE_TRIMMED}" >> "$GITHUB_OUTPUT" - printf 'image=%s\n' "${IMAGE_TRIMMED}" >> "$GITHUB_OUTPUT" + refs_json="" + refs_human="" + image_count=0 + while IFS= read -r ref; do + [[ -z "${ref}" ]] && continue + if [[ ! "${ref}" =~ ^[A-Za-z0-9._:/@-]+$ ]]; then + echo "::error::image '${ref}' has characters that aren't valid in an image ref" + exit 1 + fi + if [[ "${ref}" != *:* ]]; then + echo "::error::image '${ref}' has no tag — pass an explicit tag, not a bare repository" + exit 1 + fi + refs_json+="${refs_json:+,}\"${ref}\"" + refs_human+="${refs_human:+, }${ref}" + image_count=$((image_count + 1)) + done <<< "$(printf '%s' "${IMAGES_CSV}" | tr ',' '\n')" + printf 'image-refs=[%s]\n' "${refs_json}" >> "$GITHUB_OUTPUT" + printf 'images=%s\n' "${refs_human}" >> "$GITHUB_OUTPUT" + printf 'image-count=%s\n' "${image_count}" >> "$GITHUB_OUTPUT" emit_multiline ports "$(printf '%s' "${PORTS}" | tr ' ' ',')" emit_multiline exclude "${EXCLUDE}" + emit_multiline instances "${INSTANCES}" - - name: Validate numeric inputs + - name: Validate inputs shell: bash env: - MIN_VRAM: ${{ inputs.min-vram-gb }} - MAX_PARALLEL: ${{ inputs.max-parallel }} CUDA: ${{ inputs.cuda-versions }} + INSTANCES: ${{ inputs.instances }} + CHECK_ALL: ${{ inputs.check-all-gpu }} run: | set -euo pipefail - # Fail before the catalog fetch, not with an int()/float() - # traceback deep in the harness. - [[ "${MIN_VRAM}" =~ ^[0-9]+$ ]] \ - || { echo "::error::min-vram-gb must be a whole number, got '${MIN_VRAM}'"; exit 1; } - [[ "${MAX_PARALLEL}" =~ ^[1-9][0-9]*$ ]] \ - || { echo "::error::max-parallel must be a positive integer, got '${MAX_PARALLEL}'"; exit 1; } + # Fail before the catalog fetch, not with a traceback deep in the + # harness or, worse, a sweep that silently ignored an input. + instances_trimmed=$(printf '%s' "${INSTANCES}" | tr -d '[:space:]') + if [[ -n "${instances_trimmed}" && "${CHECK_ALL,,}" == "true" ]]; then + echo "::error::choose one: 'instances' lists exact GPUs, 'check-all-gpu' sweeps the catalog. An explicit list wins in the harness, so check-all-gpu would be silently ignored. Clear the instances field, or untick check-all-gpu." + exit 1 + fi + if [[ -z "${instances_trimmed}" && "${CHECK_ALL,,}" != "true" ]]; then + echo "::error::nothing to test: set 'instances', or tick 'check-all-gpu'." + exit 1 + fi # 'all', or a comma list of X.Y. Rejecting a bare major here saves # a whole sweep that would pin a version nothing reports. if [[ -n "${CUDA}" && "${CUDA,,}" != "all" ]]; then @@ -152,13 +186,15 @@ jobs: env: # Free-text inputs go through env so shell metacharacters can't # break out of the echo. - IMAGE: ${{ steps.prep.outputs.image }} + IMAGES: ${{ steps.prep.outputs.images }} + IMAGE_COUNT: ${{ steps.prep.outputs.image-count }} PORTS: ${{ steps.prep.outputs.ports }} EXCLUDE: ${{ steps.prep.outputs.exclude }} + INSTANCES: ${{ steps.prep.outputs.instances }} CUDA: ${{ inputs.cuda-versions }} - VENDOR: ${{ inputs.manufacturer }} - MIN_VRAM: ${{ inputs.min-vram-gb }} - MAX_PARALLEL: ${{ inputs.max-parallel }} + CLOUD: ${{ inputs.cloud }} + CHECK_ALL: ${{ inputs.check-all-gpu }} + MAX_PARALLEL: ${{ env.MAX_PARALLEL }} JUPYTER: ${{ inputs.test-jupyter }} COMFY_SMOKE: ${{ inputs.test-comfyui || inputs.test-comfyui-functional }} COMFY_FUNC: ${{ inputs.test-comfyui-functional }} @@ -168,16 +204,16 @@ jobs: # through it, yielding '8080,8888 9000'. ports_1l=$(printf '%s' "${PORTS}" | paste -sd, - | sed 's/,/, /g') exclude_1l=$(printf '%s' "${EXCLUDE}" | paste -sd, - | sed 's/,/, /g') + instances_1l=$(printf '%s' "${INSTANCES}" | paste -sd, - | sed 's/,/, /g') { echo "### GPU compatibility sweep" echo echo "| Setting | Value |" echo "| --- | --- |" - echo "| Image | \`${IMAGE}\` |" + echo "| Images (${IMAGE_COUNT}) | \`${IMAGES//, /\`, \`}\` |" + echo "| GPUs | ${instances_1l:-whole catalog (check-all-gpu=${CHECK_ALL})} |" + echo "| Cloud tier | ${CLOUD} |" echo "| CUDA axis | ${CUDA:-none (floor from tag)} |" - echo "| Cloud tiers | SECURE + COMMUNITY |" - echo "| Vendor | ${VENDOR} |" - echo "| Min vRAM | ${MIN_VRAM} GB |" echo "| Ports | ${ports_1l:-none} |" echo "| Excluded GPUs | ${exclude_1l:-none} |" echo "| Jupyter | ${JUPYTER} |" @@ -194,19 +230,23 @@ jobs: runpod-api-key: ${{ secrets.TESTING_RUNPOD_API_KEY }} ssh-private-key: ${{ secrets.TESTING_RUNPOD_SSH_PRIVATE_KEY }} - # One pod per GPU, no first-PASS short-circuit. This also drops the - # budget filter, so min-vram-gb and exclude-instances are the only - # ways to narrow the sweep. - check-all-gpu: "true" - manufacturer: ${{ inputs.manufacturer }} - min-vram-gb: ${{ inputs.min-vram-gb }} + # Exactly one of these is set — the validate step rejects both and + # neither. With check-all-gpu it's one pod per catalog GPU and no + # budget filter; with instances it's one pod per named GPU. + check-all-gpu: ${{ inputs.check-all-gpu }} + instances: ${{ steps.prep.outputs.instances }} exclude-instances: ${{ steps.prep.outputs.exclude }} - # Both tiers, always: a compatibility sweep that silently skipped - # every community-only GPU wouldn't be a catalog sweep. Not an - # input because workflow_dispatch is capped at 10 of them; narrow - # with exclude-instances if you need one tier only. - cloud-type: "ALL" + # The action defaults to a 16 GB floor, which would silently drop + # RTX 3070 and RTX A2000 from a sweep that claims to cover every + # GPU. Vendor stays at the action's Nvidia default: the CUDA axis + # is NVIDIA-only, so MI300X belongs in a separate ROCm run. + min-vram-gb: "0" + + # ALL sweeps both tiers. Needed for full catalog coverage: they + # don't overlap — every GeForce card is community-only, while A40, + # L4, H100 SXM, B200 and B300 have no community hosts. + cloud-type: ${{ inputs.cloud }} # Empty = no axis; one job per GPU with the floor derived from the # image tag by instances.detect_cuda_version. @@ -234,4 +274,4 @@ jobs: # Worst case across images: rocm/* bases are 30-50 GB. create-timeout: "1200" - max-parallel: ${{ inputs.max-parallel }} + max-parallel: ${{ env.MAX_PARALLEL }} diff --git a/.github/workflows/zz-temp-gpu-compat-check-all.yml b/.github/workflows/zz-temp-gpu-compat-check-all.yml new file mode 100644 index 00000000..c756a6db --- /dev/null +++ b/.github/workflows/zz-temp-gpu-compat-check-all.yml @@ -0,0 +1,202 @@ +name: "TEMP: gpu-compat via check-all-gpu" + +# ============================================================================ +# TEMPORARY — DELETE BEFORE MERGING THIS BRANCH. +# +# Same purpose as zz-temp-gpu-compat-instances.yml, but the other selection +# branch: check-all-gpu on the SECURE tier. The Normalize / Validate / +# Summarize shell is copied verbatim from gpu-compatibility.yml with +# `inputs.X` swapped for `env.X`. +# +# Trigger: pushing a change to THIS file. +# +# The catalog holds 46 NVIDIA types, so the exclude list below cuts it to 3: +# L4, RTX A4000, RTX A4500 — all SECURE, all around $0.25-0.49/hr. Patterns +# match DISPLAY names, so 'RTX 50*' also swallows 'RTX 5000 Ada' and +# 'PRO 6000 MIG *' is spelled out because those names lack the RTX prefix. +# +# Cost: 3 pods, ~7 min each -> about $0.12. Widen by deleting exclude lines. +# ============================================================================ + +on: + push: + paths: + - .github/workflows/zz-temp-gpu-compat-check-all.yml + +permissions: + contents: read + +concurrency: + group: zz-temp-gpu-compat-check-all + cancel-in-progress: false + +jobs: + compatibility: + runs-on: blacksmith-4vcpu-ubuntu-2404 + env: + # ---- stand-ins for the dispatch inputs ------------------------------- + IN_IMAGE: runpod/pytorch:1.2.0-rc.162-cu1281-torch2121-ubuntu2204 + IN_TEST_PORTS: "" + # Empty on purpose: no CUDA axis means one pod per GPU with the floor + # taken from the tag. Set to 'all' only if you want to pay for a matrix. + IN_CUDA_VERSIONS: "" + IN_EXCLUDE_INSTANCES: >- + A100*, A40, B200, B300, + RTX 30*, RTX 40*, RTX 50*, + H100*, H200*, L40*, + RTX 2000 Ada, RTX 6000 Ada, + RTX A2000, RTX A5000, RTX A6000, + RTX PRO *, PRO 6000 MIG *, + Tesla V100, V100 SXM2 + IN_CLOUD: SECURE + IN_INSTANCES: "" + IN_CHECK_ALL_GPU: "true" + IN_TEST_JUPYTER: "true" + IN_TEST_COMFYUI: "false" + IN_TEST_COMFYUI_FUNCTIONAL: "false" + MAX_PARALLEL: "3" + timeout-minutes: 90 + steps: + - name: Checkout + uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 + with: + fetch-depth: 0 + + - name: Normalize inputs + id: prep + shell: bash + env: + IMAGE: ${{ env.IN_IMAGE }} + PORTS: ${{ env.IN_TEST_PORTS }} + EXCLUDE: ${{ env.IN_EXCLUDE_INSTANCES }} + INSTANCES: ${{ env.IN_INSTANCES }} + run: | + set -euo pipefail + + to_lines() { + printf '%s' "$1" \ + | tr ',' '\n' \ + | sed 's/^[[:space:]]*//; s/[[:space:]]*$//' \ + | grep -v '^$' || true + } + + emit_multiline() { + { + echo "$1<<__GH_EOF__" + to_lines "$2" + echo "__GH_EOF__" + } >> "$GITHUB_OUTPUT" + } + + IMAGES_CSV=$(printf '%s' "${IMAGE}" | tr -d '[:space:]') + if [[ -z "${IMAGES_CSV}" ]]; then + echo "::error::image input is empty" + exit 1 + fi + refs_json="" + refs_human="" + image_count=0 + while IFS= read -r ref; do + [[ -z "${ref}" ]] && continue + if [[ ! "${ref}" =~ ^[A-Za-z0-9._:/@-]+$ ]]; then + echo "::error::image '${ref}' has characters that aren't valid in an image ref" + exit 1 + fi + if [[ "${ref}" != *:* ]]; then + echo "::error::image '${ref}' has no tag — pass an explicit tag, not a bare repository" + exit 1 + fi + refs_json+="${refs_json:+,}\"${ref}\"" + refs_human+="${refs_human:+, }${ref}" + image_count=$((image_count + 1)) + done <<< "$(printf '%s' "${IMAGES_CSV}" | tr ',' '\n')" + printf 'image-refs=[%s]\n' "${refs_json}" >> "$GITHUB_OUTPUT" + printf 'images=%s\n' "${refs_human}" >> "$GITHUB_OUTPUT" + printf 'image-count=%s\n' "${image_count}" >> "$GITHUB_OUTPUT" + + emit_multiline ports "$(printf '%s' "${PORTS}" | tr ' ' ',')" + emit_multiline exclude "${EXCLUDE}" + emit_multiline instances "${INSTANCES}" + + - name: Validate inputs + shell: bash + env: + CUDA: ${{ env.IN_CUDA_VERSIONS }} + INSTANCES: ${{ env.IN_INSTANCES }} + CHECK_ALL: ${{ env.IN_CHECK_ALL_GPU }} + run: | + set -euo pipefail + instances_trimmed=$(printf '%s' "${INSTANCES}" | tr -d '[:space:]') + if [[ -n "${instances_trimmed}" && "${CHECK_ALL,,}" == "true" ]]; then + echo "::error::choose one: 'instances' lists exact GPUs, 'check-all-gpu' sweeps the catalog. An explicit list wins in the harness, so check-all-gpu would be silently ignored. Clear the instances field, or untick check-all-gpu." + exit 1 + fi + if [[ -z "${instances_trimmed}" && "${CHECK_ALL,,}" != "true" ]]; then + echo "::error::nothing to test: set 'instances', or tick 'check-all-gpu'." + exit 1 + fi + if [[ -n "${CUDA}" && "${CUDA,,}" != "all" ]]; then + normalized=$(printf '%s' "${CUDA}" | tr -d '[:space:]') + [[ "${normalized}" =~ ^[0-9]+\.[0-9]+(,[0-9]+\.[0-9]+)*$ ]] \ + || { echo "::error::cuda-versions must be 'all' or X.Y[,X.Y...] (e.g. '12.8, 13.0'), got '${CUDA}'"; exit 1; } + fi + + - name: Summarize run parameters + shell: bash + env: + IMAGES: ${{ steps.prep.outputs.images }} + IMAGE_COUNT: ${{ steps.prep.outputs.image-count }} + PORTS: ${{ steps.prep.outputs.ports }} + EXCLUDE: ${{ steps.prep.outputs.exclude }} + INSTANCES: ${{ steps.prep.outputs.instances }} + CUDA: ${{ env.IN_CUDA_VERSIONS }} + CLOUD: ${{ env.IN_CLOUD }} + CHECK_ALL: ${{ env.IN_CHECK_ALL_GPU }} + MAX_PARALLEL: ${{ env.MAX_PARALLEL }} + JUPYTER: ${{ env.IN_TEST_JUPYTER }} + run: | + set -euo pipefail + ports_1l=$(printf '%s' "${PORTS}" | paste -sd, - | sed 's/,/, /g') + exclude_1l=$(printf '%s' "${EXCLUDE}" | paste -sd, - | sed 's/,/, /g') + instances_1l=$(printf '%s' "${INSTANCES}" | paste -sd, - | sed 's/,/, /g') + { + echo "### TEMP: gpu-compat via check-all-gpu (SECURE)" + echo + echo "| Setting | Value |" + echo "| --- | --- |" + echo "| Images (${IMAGE_COUNT}) | \`${IMAGES//, /\`, \`}\` |" + echo "| GPUs | ${instances_1l:-whole catalog (check-all-gpu=${CHECK_ALL})} |" + echo "| Cloud tier | ${CLOUD} |" + echo "| CUDA axis | ${CUDA:-none (floor from tag)} |" + echo "| Ports | ${ports_1l:-none} |" + echo "| Excluded GPUs | ${exclude_1l:-none} |" + echo "| Jupyter | ${JUPYTER} |" + echo "| Parallel pods | ${MAX_PARALLEL} |" + } >> "$GITHUB_STEP_SUMMARY" + + - name: Run compatibility matrix + uses: ./.github/actions/smoke-test + with: + image-refs: ${{ steps.prep.outputs.image-refs }} + profile: gpu + runpod-api-key: ${{ secrets.TESTING_RUNPOD_API_KEY }} + ssh-private-key: ${{ secrets.TESTING_RUNPOD_SSH_PRIVATE_KEY }} + + check-all-gpu: ${{ env.IN_CHECK_ALL_GPU }} + instances: ${{ steps.prep.outputs.instances }} + exclude-instances: ${{ steps.prep.outputs.exclude }} + min-vram-gb: "0" + cloud-type: ${{ env.IN_CLOUD }} + cuda-versions: ${{ env.IN_CUDA_VERSIONS }} + + test-ports: ${{ steps.prep.outputs.ports }} + test-jupyter: ${{ env.IN_TEST_JUPYTER }} + test-comfyui: ${{ env.IN_TEST_COMFYUI }} + test-comfyui-functional: ${{ env.IN_TEST_COMFYUI_FUNCTIONAL }} + + upload-results-json: "true" + results-artifact-name: zz-temp-check-all-${{ github.run_id }} + + on-skip: pass + create-timeout: "1200" + max-parallel: ${{ env.MAX_PARALLEL }} diff --git a/.github/workflows/zz-temp-gpu-compat-instances.yml b/.github/workflows/zz-temp-gpu-compat-instances.yml new file mode 100644 index 00000000..c51c0874 --- /dev/null +++ b/.github/workflows/zz-temp-gpu-compat-instances.yml @@ -0,0 +1,219 @@ +name: "TEMP: gpu-compat via instances" + +# ============================================================================ +# TEMPORARY — DELETE BEFORE MERGING THIS BRANCH. +# +# gpu-compatibility.yml is workflow_dispatch-only, and GitHub does not show +# a dispatch button for a workflow that isn't on the default branch yet. This +# file exists to exercise the same steps from a branch push: the Normalize / +# Validate / Summarize shell is copied verbatim, with `inputs.X` swapped for +# `env.X` so the values are hardcoded below instead of typed into a form. +# +# Trigger: pushing a change to THIS file. Nothing else runs it, so ordinary +# commits on the branch don't spend money. To run it again without editing, +# use "Re-run all jobs" on the run in the Actions tab. +# +# Covers: multi-image splitting, explicit `instances`, cloud tier, and the +# mutual-exclusion guard. +# +# `instances` is a fallback chain, not a matrix — so this is 2 pods (one per +# image), each landing on the first listed GPU with capacity. +# +# Cost: 2 pods, ~7 min each, ~$0.25/hr -> about $0.06. +# ============================================================================ + +on: + push: + paths: + - .github/workflows/zz-temp-gpu-compat-instances.yml + +permissions: + contents: read + +concurrency: + group: zz-temp-gpu-compat-instances + cancel-in-progress: false + +jobs: + compatibility: + runs-on: blacksmith-4vcpu-ubuntu-2404 + env: + # ---- stand-ins for the dispatch inputs ------------------------------- + IN_IMAGE: >- + runpod/pytorch:1.2.0-rc.162-cu1281-torch2121-ubuntu2204, + runpod/pytorch:1.2.0-rc.162-cu1290-torch2130-ubuntu2404 + IN_TEST_PORTS: "" + IN_CUDA_VERSIONS: "" + IN_EXCLUDE_INSTANCES: "" + IN_CLOUD: SECURE + # Both are SECURE and ~$0.25/hr, and both names contain a space, which + # is what the --instance argv loop has to keep intact. + IN_INSTANCES: "RTX A4000, RTX A4500" + IN_CHECK_ALL_GPU: "false" + IN_TEST_JUPYTER: "true" + IN_TEST_COMFYUI: "false" + IN_TEST_COMFYUI_FUNCTIONAL: "false" + MAX_PARALLEL: "2" + timeout-minutes: 90 + steps: + - name: Checkout + uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 + with: + fetch-depth: 0 + + - name: Normalize inputs + id: prep + shell: bash + env: + IMAGE: ${{ env.IN_IMAGE }} + PORTS: ${{ env.IN_TEST_PORTS }} + EXCLUDE: ${{ env.IN_EXCLUDE_INSTANCES }} + INSTANCES: ${{ env.IN_INSTANCES }} + run: | + set -euo pipefail + + to_lines() { + printf '%s' "$1" \ + | tr ',' '\n' \ + | sed 's/^[[:space:]]*//; s/[[:space:]]*$//' \ + | grep -v '^$' || true + } + + emit_multiline() { + { + echo "$1<<__GH_EOF__" + to_lines "$2" + echo "__GH_EOF__" + } >> "$GITHUB_OUTPUT" + } + + IMAGES_CSV=$(printf '%s' "${IMAGE}" | tr -d '[:space:]') + if [[ -z "${IMAGES_CSV}" ]]; then + echo "::error::image input is empty" + exit 1 + fi + refs_json="" + refs_human="" + image_count=0 + while IFS= read -r ref; do + [[ -z "${ref}" ]] && continue + if [[ ! "${ref}" =~ ^[A-Za-z0-9._:/@-]+$ ]]; then + echo "::error::image '${ref}' has characters that aren't valid in an image ref" + exit 1 + fi + if [[ "${ref}" != *:* ]]; then + echo "::error::image '${ref}' has no tag — pass an explicit tag, not a bare repository" + exit 1 + fi + refs_json+="${refs_json:+,}\"${ref}\"" + refs_human+="${refs_human:+, }${ref}" + image_count=$((image_count + 1)) + done <<< "$(printf '%s' "${IMAGES_CSV}" | tr ',' '\n')" + printf 'image-refs=[%s]\n' "${refs_json}" >> "$GITHUB_OUTPUT" + printf 'images=%s\n' "${refs_human}" >> "$GITHUB_OUTPUT" + printf 'image-count=%s\n' "${image_count}" >> "$GITHUB_OUTPUT" + + emit_multiline ports "$(printf '%s' "${PORTS}" | tr ' ' ',')" + emit_multiline exclude "${EXCLUDE}" + emit_multiline instances "${INSTANCES}" + + - name: Validate inputs + shell: bash + env: + CUDA: ${{ env.IN_CUDA_VERSIONS }} + INSTANCES: ${{ env.IN_INSTANCES }} + CHECK_ALL: ${{ env.IN_CHECK_ALL_GPU }} + run: | + set -euo pipefail + instances_trimmed=$(printf '%s' "${INSTANCES}" | tr -d '[:space:]') + if [[ -n "${instances_trimmed}" && "${CHECK_ALL,,}" == "true" ]]; then + echo "::error::choose one: 'instances' lists exact GPUs, 'check-all-gpu' sweeps the catalog. An explicit list wins in the harness, so check-all-gpu would be silently ignored. Clear the instances field, or untick check-all-gpu." + exit 1 + fi + if [[ -z "${instances_trimmed}" && "${CHECK_ALL,,}" != "true" ]]; then + echo "::error::nothing to test: set 'instances', or tick 'check-all-gpu'." + exit 1 + fi + if [[ -n "${CUDA}" && "${CUDA,,}" != "all" ]]; then + normalized=$(printf '%s' "${CUDA}" | tr -d '[:space:]') + [[ "${normalized}" =~ ^[0-9]+\.[0-9]+(,[0-9]+\.[0-9]+)*$ ]] \ + || { echo "::error::cuda-versions must be 'all' or X.Y[,X.Y...] (e.g. '12.8, 13.0'), got '${CUDA}'"; exit 1; } + fi + + # TEMP-only: the mutual exclusion is enforced twice — by the Validate + # step above and by the generator the composite action calls. Assert + # the generator half here, since no run can exercise both branches. + - name: Assert instances + check-all-gpu is rejected + shell: bash + run: | + set -uo pipefail + out=$(python3 .github/scripts/generate_test_manifest.py \ + --profile gpu --refs '["runpod/pytorch:1.2.0-rc.162-cu1281-torch2121-ubuntu2204"]' \ + --instance "RTX A4000" --check-all-gpu \ + --output "${RUNNER_TEMP}/should-not-exist.yaml" 2>&1) && rc=0 || rc=$? + echo "${out}" + if [[ "${rc}" -eq 0 ]]; then + echo "::error::generator accepted --instance together with --check-all-gpu" + exit 1 + fi + echo "OK: generator exited ${rc}" + + - name: Summarize run parameters + shell: bash + env: + IMAGES: ${{ steps.prep.outputs.images }} + IMAGE_COUNT: ${{ steps.prep.outputs.image-count }} + PORTS: ${{ steps.prep.outputs.ports }} + EXCLUDE: ${{ steps.prep.outputs.exclude }} + INSTANCES: ${{ steps.prep.outputs.instances }} + CUDA: ${{ env.IN_CUDA_VERSIONS }} + CLOUD: ${{ env.IN_CLOUD }} + CHECK_ALL: ${{ env.IN_CHECK_ALL_GPU }} + MAX_PARALLEL: ${{ env.MAX_PARALLEL }} + JUPYTER: ${{ env.IN_TEST_JUPYTER }} + run: | + set -euo pipefail + ports_1l=$(printf '%s' "${PORTS}" | paste -sd, - | sed 's/,/, /g') + exclude_1l=$(printf '%s' "${EXCLUDE}" | paste -sd, - | sed 's/,/, /g') + instances_1l=$(printf '%s' "${INSTANCES}" | paste -sd, - | sed 's/,/, /g') + { + echo "### TEMP: gpu-compat via explicit instances" + echo + echo "| Setting | Value |" + echo "| --- | --- |" + echo "| Images (${IMAGE_COUNT}) | \`${IMAGES//, /\`, \`}\` |" + echo "| GPUs | ${instances_1l:-whole catalog (check-all-gpu=${CHECK_ALL})} |" + echo "| Cloud tier | ${CLOUD} |" + echo "| CUDA axis | ${CUDA:-none (floor from tag)} |" + echo "| Ports | ${ports_1l:-none} |" + echo "| Excluded GPUs | ${exclude_1l:-none} |" + echo "| Jupyter | ${JUPYTER} |" + echo "| Parallel pods | ${MAX_PARALLEL} |" + } >> "$GITHUB_STEP_SUMMARY" + + - name: Run compatibility matrix + uses: ./.github/actions/smoke-test + with: + image-refs: ${{ steps.prep.outputs.image-refs }} + profile: gpu + runpod-api-key: ${{ secrets.TESTING_RUNPOD_API_KEY }} + ssh-private-key: ${{ secrets.TESTING_RUNPOD_SSH_PRIVATE_KEY }} + + check-all-gpu: ${{ env.IN_CHECK_ALL_GPU }} + instances: ${{ steps.prep.outputs.instances }} + exclude-instances: ${{ steps.prep.outputs.exclude }} + min-vram-gb: "0" + cloud-type: ${{ env.IN_CLOUD }} + cuda-versions: ${{ env.IN_CUDA_VERSIONS }} + + test-ports: ${{ steps.prep.outputs.ports }} + test-jupyter: ${{ env.IN_TEST_JUPYTER }} + test-comfyui: ${{ env.IN_TEST_COMFYUI }} + test-comfyui-functional: ${{ env.IN_TEST_COMFYUI_FUNCTIONAL }} + + upload-results-json: "true" + results-artifact-name: zz-temp-instances-${{ github.run_id }} + + on-skip: pass + create-timeout: "1200" + max-parallel: ${{ env.MAX_PARALLEL }} diff --git a/tests/README.md b/tests/README.md index 96888375..3f535f84 100644 --- a/tests/README.md +++ b/tests/README.md @@ -118,7 +118,7 @@ runs this sequence and reports the outcome as soon as one step fails. | # | Step | Failure → | |---|------|---| | 1 | `POST /v2/pods` with `gpu.id` (or an auto-picked `cpu.id` + `vcpuCount`), `disk`, `ports`, `startSsh`, registry credential, and either `gpu.minCudaVersion` or `gpu.allowedCudaVersions`. Transient failures (429, 5xx, transport) are retried up to `CREATE_RETRIES` with linear backoff. | `UNAVAILABLE` (no capacity — try next instance) / `CREATE_FAIL` (bad image tag, auth, malformed request — any non-capacity, non-transient error after retries) | -| 2 | Poll `GET /v2/pods/{id}` until `status` is `RUNNING`, `ssh.direct` is populated, and one-shot `ssh root@host -p port 'echo ready'` succeeds. SSH is the readiness signal; `status` is the real observed `PodStatus`, so terminal `EXITED`/`ERROR`/`TERMINATED` stop the poll immediately. | `FAIL` on a terminal status, or when the system log shows a container-init rejection; `STUCK` if no SSH endpoint within `CREATE_TIMEOUT` | +| 2 | Poll `GET /v2/pods/{id}` until `status` is `RUNNING` and a one-shot `ssh @host -p port 'echo ready'` succeeds against **either** `ssh.direct` or `ssh.proxy` — see [SSH endpoints](#ssh-endpoints). SSH is the readiness signal; `status` is the real observed `PodStatus`, so terminal `EXITED`/`ERROR`/`TERMINATED` stop the poll immediately. | `FAIL` on a terminal status, or when the system log shows a container-init rejection; `STUCK` if neither endpoint answers within `CREATE_TIMEOUT` | | 3 | **CUDA functional check** over SSH — see [Functional check](#functional-check). Image-driven: pytorch ref → `torch.cuda` + matmul; cuda/rocm ref → `nvidia-smi` + `nvcc`; neither → skip | `FAIL` (image is broken — stop iterating; another GPU won't help) | | 4 | **JupyterLab proxy-first check** (only when `test_jupyter: true`) — checks the public proxy; SSH probes `/api/status` only to diagnose a proxy failure | `FAIL` (Jupyter did not start, or is not exposed as `8888/http`) | | 5 | **Generic proxy-first port checks** (optional `test_ports`) — each service must return HTTP 200 through `https://-.proxy.runpod.net/`; SSH diagnoses failures | `FAIL` (service unavailable or incorrectly exposed) | @@ -138,6 +138,44 @@ the full selected GPU set. Adding `cuda_versions:` splits it further — one job per (GPU, CUDA version) — see [CUDA axis](#cuda-axis). +## SSH endpoints + +`GET /v2/pods/{id}` returns two ways in, and the harness tries both: + +| | `ssh.direct` | `ssh.proxy` | +|---|---|---| +| target | the container's sshd, straight over TCP | `ssh.runpod.io`, relayed | +| login | `root` | an opaque routing token from `username` | +| needs | RunPod to allocate a public port for `22/tcp` | only a machine assignment | +| supports | everything | interactive shell and remote commands only — no SCP, SFTP or port forwarding | + +**RunPod does not always allocate the direct port.** Such a pod shows only +"SSH" in the console, with no "SSH over exposed TCP" block, and +`ssh.direct` stays null however long you wait. Before the fallback existed +those pods burned the whole `CREATE_TIMEOUT` and were reported as stuck +initializing, which was wrong — they were ready in a minute. + +Direct is tried first (one fewer hop), the proxy on the same poll if direct +is absent or refuses. Whichever answers is reused by every later check, so a +pod validated through the proxy runs the same CUDA, Jupyter, port and +ComfyUI checks — none of them need SCP or port forwarding. The readiness +line names the endpoint and its kind, so the log shows which path was used. + +The proxy invocation comes from `ssh.proxy.command` rather than being +assembled by hand, plus the `-i` and `-o StrictHostKeyChecking=no` that the +field's own description says to add, plus `-tt`: the proxy rejects a +connection with `Your SSH client doesn't support PTY` when no terminal is +allocated, and a single `-t` declines to allocate one because the harness's +stdin is not a terminal. Output is stripped of the `\r` a PTY introduces so +it parses the same as a direct connection. + +**Proxy-only pods give up early.** When only `ssh.proxy` exists and it has +already refused, nothing will change by waiting, so the poll stops at +`DIRECT_PORT_TIMEOUT` (default 300s) instead of billing the pod until +`CREATE_TIMEOUT`. The outcome is `STUCK`, so the next instance type is +tried — a different host usually does get a port. + + ## Outcomes The summary at the end of every run groups results into three buckets. @@ -256,11 +294,11 @@ Field reference: | field | description | |---|---| | `images` | Docker images to test. **Required.** | -| `instances` | Explicit list of GPU display names, tried in order. One of `instances:` or `max_price_per_hour:` is required (except for `base_cpu`). | +| `instances` | Explicit list of GPU display names, tried in order as a **fallback chain** — the group gets one pod per image, on the first candidate with capacity. One pod per GPU is `check_all_gpu`, not this. One of `instances:` or `max_price_per_hour:` is required (except for `base_cpu`). | | `max_price_per_hour` | USD/hr budget — auto-pick any GPU at this price or below, cheapest first. Loses to explicit `instances:` if both are set. | | `min_vram_gb` | Extra filter for budget mode (default 0). | | `manufacturer` | `Nvidia` or `AMD` filter for budget mode (default: any). | -| `exclude_instances` | fnmatch-style patterns (case-insensitive) subtracted from the candidate list AFTER `instances:` or budget selection. Useful for blocking known-bad host pairings without rewriting the whole list — e.g. `"*Blackwell*"` skips every Blackwell GPU (sm\_100 / sm\_120 are not in the kernel set of PyTorch ≤ 2.6 wheels). | +| `exclude_instances` | fnmatch-style patterns (case-insensitive) subtracted from the candidate list AFTER `instances:` or budget selection. Useful for blocking known-bad host pairings without rewriting the whole list. Patterns match **display** names, not catalog ids, so Blackwell needs `B200`, `B300`, `RTX 5090`, `RTX PRO *`, `PRO 6000 MIG *` — `"*Blackwell*"` matches nothing. | | `min_cuda_version` | `X.Y` floor sent as `gpu.minCudaVersion`. Used as a **fallback** when the image tag doesn't encode a CUDA version (e.g. NGC `nvidia-pytorch:25.11`); tags like `cu1281` / `cuda1281` / `cuda13.0` are parsed and win. Superseded by `cuda_versions` — the API rejects both fields on one request. | | `cuda_versions` | `all`, or a list of exact `X.Y` versions. Turns on the [CUDA axis](#cuda-axis): each candidate GPU is tested once per version, pinned with `gpu.allowedCudaVersions`. Default: unset (no axis). | | `check_all_gpu` | `true` / `false` — use every catalog GPU matching `min_vram_gb` and `manufacturer`, with one independent result row per `(image, GPU)`. Mutually exclusive with budget selection in generated manifests and potentially expensive. Default: `false`. | @@ -343,6 +381,9 @@ each tier: CLOUD_TYPE=ALL ON_SKIP=pass python3 tests/test_images.py ``` +[`gpu-compat.example.yaml`](gpu-compat.example.yaml) is a ready full-catalog +sweep in this shape — every GPU, every CUDA version it reports capacity for. + Planning happens once per tier, since availability, CUDA versions and prices are all scoped to one; the resulting jobs then share a single worker pool, and each job carries its tier through to `POST /v2/pods`. Practical @@ -501,6 +542,7 @@ pytorch: | `REGISTRY_AUTH_NAME` | _(empty)_ | Display name to look up via `GET /v2/registries` when `REGISTRY_AUTH_ID` is not set. Falls back to the first entry. | | `DWELL_SEC` | `60` | Extra seconds to wait after SSH becomes reachable, then re-probe SSH to catch containers that boot, accept SSH, then crash. Set 0 to skip the re-probe. | | `CREATE_TIMEOUT` | `600` | Max seconds to wait for SSH to become reachable. Raise for ROCm workflows (`create-timeout: "1200"` on the action) — the official `rocm/pytorch:*` base images are 30-50GB and routinely take 8-15 minutes to pull. | +| `DIRECT_PORT_TIMEOUT` | `300` | Give up this early when only `ssh.proxy` exists and it has already refused — RunPod never allocated a port for `22/tcp` and waiting out `CREATE_TIMEOUT` only bills an unreachable pod. `0` waits the full `CREATE_TIMEOUT`. See [SSH endpoints](#ssh-endpoints). | | `POLL_INTERVAL` | `10` | Poll cadence for SSH probes. | | `MAX_PARALLEL` | `1` | How many images to smoke-test concurrently. Each worker holds at most one pod, so this caps simultaneous live pods. Keep modest to avoid RunPod rate limits and surprise bills. | | `CREATE_RETRIES` | `3` | Retry pod-create up to N times on transient RunPod 5xx errors (`Something went wrong`, 502/503). Capacity shortages are NOT retried. | @@ -572,11 +614,14 @@ wraps everything in this script needs for a clean CI run: `.github/scripts/generate_test_manifest.py`, applying the `profile`, `budget-usd-per-hour`, `min-vram-gb`, `manufacturer`, `test-jupyter`, `test-ports`, `test-comfyui`, - `test-comfyui-functional`, `check-all-gpu`, `cuda-versions`, - `min-cuda-version`, and `exclude-instances` inputs. + `test-comfyui-functional`, `check-all-gpu`, `instances`, + `cuda-versions`, `min-cuda-version`, and `exclude-instances` inputs. + `instances` and `check-all-gpu` are mutually exclusive — passing both + fails the generator instead of silently ignoring one. 4. Invokes `python3 tests/test_images.py ` with - `MAX_PARALLEL=`. A failed image makes the smoke-test - action fail, which prevents a release from being created. + `MAX_PARALLEL=` and `CLOUD_TYPE=`. A failed + image makes the smoke-test action fail, which prevents a release from + being created. Typical caller (from a per-image-family build workflow): @@ -616,13 +661,14 @@ fields. | `warn: no GPU catalog` | `GET /v2/catalog/gpus` failed — usually a bad/absent key | fix the key; budget and `check_all_gpu` selection are disabled without it | | `warn: no registry auth configured` | no Docker Hub credential on the account | add one in the RunPod console (paid Hub account strongly recommended for parallel runs) | | every pod SKIPs with an SSH failure | private key not mode `600`, or its public half isn't registered | `chmod 600 `; verify the fingerprint appears in `GET /v2/account/ssh-keys` | +| `no ssh endpoint assigned yet` for the whole timeout | the pod never got a machine, so neither `ssh.direct` nor `ssh.proxy` exists | genuine provisioning failure — retry, or check the pod in the console. A missing *direct* port alone no longer causes this: the proxy is used instead | | `cuda_versions is set but none of the N candidate GPUs reports any CUDA version in the SECURE cloud` | CUDA axis on a ROCm/AMD sweep, or every candidate lives in the other cloud tier | drop `cuda_versions` for ROCm — the axis is NVIDIA-only; otherwise rerun with the other `CLOUD_TYPE` | | `GPU not covered: not offered in the SECURE cloud` | community-only GPU (all GeForce cards, both V100s, `A100 SXM 40GB`) — `cudaVersions` is scoped to `CLOUD_TYPE` | use `CLOUD_TYPE=ALL` to sweep both tiers in one run; the note says `(covered by the COMMUNITY pass)` when it already did | | every group says `no capacity on any of N candidate instance type(s)` | budget too low / VRAM too high / region saturated | raise `max_price_per_hour`, drop `min_vram_gb`, or set explicit `instances:` | | only the `base_cpu` group says `no capacity` while GPU groups pass | the cloud(s) you target don't have CPU capacity right now | by default we already try SECURE then COMMUNITY. If both are full, add DC-pinned candidates: `CPU_CANDIDATES="cpu-secure:SECURE,cpu-community:COMMUNITY,cpu-eu:COMMUNITY:EU-RO-1+EU-NL-1,cpu-us:COMMUNITY:US-OR-1"` | | pod stays in `ssh endpoint not assigned yet` past `STALL_HINT_AFTER` | slow image pull or Docker Hub `toomanyrequests` | add registry auth, reduce `MAX_PARALLEL`, or wait 6 h for the Hub rate limit to reset | | `ssh_probe=FAIL — Permission denied (publickey)` | wrong SSH key | export `RUNPOD_SSH_KEY=/path/to/private/key` whose public half is on the RunPod account | -| `pod entered TIMEOUT state` repeatedly on Blackwell GPUs for a `pytorch` group | PyTorch ≤ 2.6 has no `sm_100`/`sm_120` kernels | add `exclude_instances: ["*Blackwell*"]` to the group | +| `pod entered TIMEOUT state` repeatedly on Blackwell GPUs for a `pytorch` group | PyTorch ≤ 2.6 has no `sm_100`/`sm_120` kernels | add `exclude_instances: ["B200", "B300", "RTX 5090", "RTX PRO *", "PRO 6000 MIG *"]` — the display names never contain the word `Blackwell` | | `nvidia-container-cli: requirement error: unsatisfied condition: cuda>=X.Y` in pod logs | image needs a newer driver than the host has | set `min_cuda_version: "X.Y"` in the manifest (only needed for tags without a `cuXYZW`/`cudaXYZW` marker) | | `jupyter check (in-pod) FAILED -- start.sh did not bring up JupyterLab` | `start.sh` is launching Jupyter with the wrong Python interpreter (classic Ubuntu 22.04 `python3` → 3.10 vs `python` → 3.12) | fix `container-template/start.sh` to use `python -m jupyter lab` | | `jupyter check (public proxy) FAILED` but in-pod check passed | port exposed as `8888/tcp` instead of `8888/http`, OR proxy hasn't registered the pod yet | check `pod create --ports` arg; bump `JUPYTER_PROXY_TIMEOUT` if proxy is just slow | diff --git a/tests/gpu-compat.example.yaml b/tests/gpu-compat.example.yaml new file mode 100644 index 00000000..147167df --- /dev/null +++ b/tests/gpu-compat.example.yaml @@ -0,0 +1,29 @@ +# Local GPU × CUDA compatibility sweep — every catalog GPU, every CUDA +# version that GPU reports capacity for. Prerequisites in ./README.md +# (RUNPOD_API_KEY + an SSH key registered on the account). +# +# CLOUD_TYPE=ALL ON_SKIP=pass MAX_PARALLEL=4 \ +# python3 tests/test_images.py tests/gpu-compat.example.yaml pytorch +# +# CUDA versions are scoped per cloud tier and the tiers don't nest, so +# CLOUD_TYPE=ALL is what reaches both the datacenter cards and the +# community-only ones (every GeForce, both V100s). Drop it for SECURE only. +# +# Do not add `max_price_per_hour` — it takes precedence over +# check_all_gpu and would filter GPUs out of the sweep. +pytorch: + images: + - runpod/pytorch:1.1.0-cu1290-torch280-ubuntu2404-cluster + check_all_gpu: true + cuda_versions: all + # Narrow the sweep instead of the whole catalog: + # cuda_versions: + # - "12.8" + # - "13.0" + # min_vram_gb: 24 + # manufacturer: Nvidia + # exclude_instances: + # - "*B200*" + # + # Drop this to test bootability only — it adds the :8888 Jupyter probes. + test_jupyter: true diff --git a/tests/runpod_smoke/checks.py b/tests/runpod_smoke/checks.py index 0e0ddc70..d1749583 100644 --- a/tests/runpod_smoke/checks.py +++ b/tests/runpod_smoke/checks.py @@ -15,6 +15,7 @@ import json import os import re +import shlex import subprocess import threading import time @@ -59,16 +60,64 @@ def _resolve_ssh_identity() -> Optional[str]: return None +# How to address each endpoint. Direct SSH lands in the container as root; +# the RunPod proxy needs an opaque routing token and a pseudo-terminal, and +# the API hands us the exact invocation in `ssh.proxy.command`, so we build +# on that instead of guessing. `pod.wait_for_running` registers what it +# found; everything else keeps identifying endpoints by (host, port). +_SSH_ENDPOINTS: dict[tuple[str, int], tuple[str, str, bool]] = {} + + +def set_ssh_endpoint( + host: str, port: int, user: str, command: str = "", *, pty: bool = False, +) -> None: + if host and port and user: + _SSH_ENDPOINTS[(host, int(port))] = (user, command or "", pty) + + +def ssh_user_for(host: str, port: int) -> str: + entry = _SSH_ENDPOINTS.get((host, int(port))) + return entry[0] if entry else "root" + + def _ssh_command_prefix(host: str, port: int) -> list[str]: - """Build the `ssh ... root@ -p ` prefix common to all SSH calls.""" - cmd = ["ssh", *config.SSH_OPTS, "-p", str(port)] + """Build the `ssh ... @` prefix common to all SSH calls. + + For a registered endpoint the API's own invocation is the base, with the + flags its description tells us to add (`-i`, `-o StrictHostKeyChecking`) + plus `-tt` when the endpoint insists on a terminal. Everything else gets + the plain `root@host -p port` form. + """ + user, api_command, pty = _SSH_ENDPOINTS.get( + (host, int(port)), ("root", "", False) + ) identity = _resolve_ssh_identity() + target = f"{user}@{host}" + if api_command: + # Keep only the target from the API string: the flags we add below + # are the ones it documents as missing, and re-using its argv + # verbatim would fight with SSH_OPTS. + parts = shlex.split(api_command) + target = next((p for p in parts[1:] if "@" in p), target) + cmd = ["ssh", *config.SSH_OPTS] + if pty: + # -tt, not -t: the local stdin is not a terminal under subprocess, + # and single -t silently declines to allocate one in that case. + cmd.append("-tt") + if int(port) != 22: + cmd.extend(["-p", str(port)]) if identity: cmd.extend(["-i", identity]) - cmd.append(f"root@{host}") + cmd.append(target) return cmd +def _strip_cr(text: str) -> str: + """A PTY turns every \\n into \\r\\n; undo that so parsing and logs match + what a direct, terminal-less connection would have produced.""" + return text.replace("\r\n", "\n").replace("\r", "\n") + + def ssh_probe(host: str, port: int, timeout: int = 8) -> tuple[bool, str]: """One-shot SSH connection attempt. Returns (success, stderr_excerpt). Used as the real container-readiness signal.""" @@ -263,7 +312,7 @@ def run_cuda_check(host: str, port: int, image: str) -> tuple[bool, str]: return False, "cuda check timed out after 60s" except FileNotFoundError: return False, _SSH_BINARY_NOT_FOUND - combined = (r.stdout + r.stderr).strip() + combined = _strip_cr(r.stdout + r.stderr).strip() return (r.returncode == 0), combined @@ -340,7 +389,7 @@ def run_jupyter_check(host: str, port: int) -> tuple[bool, str]: return False, f"jupyter check timed out after {outer_timeout}s" except FileNotFoundError: return False, _SSH_BINARY_NOT_FOUND - combined = (r.stdout + r.stderr).strip() + combined = _strip_cr(r.stdout + r.stderr).strip() return (r.returncode == 0), combined @@ -732,7 +781,7 @@ def fetch_logs_via_ssh( except FileNotFoundError: return None if r.returncode == 0 and r.stdout.strip(): - return r.stdout + return _strip_cr(r.stdout) return f"__SSH_FAILED__\nreturncode={r.returncode}\nstderr: {r.stderr.strip()[:400]}" @@ -751,6 +800,14 @@ def dump_pod_logs(pod_id: str, image: str) -> list[str]: direct = ssh.get("direct") or {} proxy = ssh.get("proxy") or {} host, port = direct.get("host"), direct.get("port") + # Fall back to the proxy so the SMI snapshot still gets fetched on pods + # that never received a direct TCP port. + if not (host and port) and proxy.get("host") and proxy.get("username"): + host, port = proxy.get("host"), proxy.get("port") + set_ssh_endpoint( + host, int(port or 0), str(proxy["username"]), + str(proxy.get("command") or ""), pty=True, + ) log(f"--- pod metadata for {pod_id} ---", indent=2) for key, val in [ @@ -788,7 +845,10 @@ def dump_pod_logs(pod_id: str, image: str) -> list[str]: logs = fetch_logs_via_ssh(host, int(port), image) if logs is None: return sys_errors - log(f"--- GPU SMI via SSH (root@{host}:{port}) ---", indent=2) + log( + f"--- GPU SMI via SSH ({ssh_user_for(host, int(port))}@{host}:{port}) ---", + indent=2, + ) if logs.startswith("__SSH_FAILED__"): log(" SSH could not reach the pod:", indent=2) for line in logs.splitlines()[1:]: diff --git a/tests/runpod_smoke/config.py b/tests/runpod_smoke/config.py index 6d9f3764..7f058535 100644 --- a/tests/runpod_smoke/config.py +++ b/tests/runpod_smoke/config.py @@ -94,6 +94,13 @@ def _coerce_on_skip(raw: str) -> str: # — just an informational note in the logs. STALL_HINT_AFTER = int(os.environ.get("STALL_HINT_AFTER", "180")) +# RunPod sometimes never allocates the public port for 22/tcp — such a pod +# shows only `ssh.proxy` in the console and `ssh.direct` stays null forever. +# Once the proxy has also refused, waiting out CREATE_TIMEOUT just bills a +# pod that will never be reachable, so give up at this mark instead. Set to +# 0 to wait the full CREATE_TIMEOUT. +DIRECT_PORT_TIMEOUT = int(os.environ.get("DIRECT_PORT_TIMEOUT", "300")) + # Docker Hub authenticated pulls — without this, RunPod datacenters share # an anonymous IP pool that hits Docker Hub's `toomanyrequests` rate limit # fast. Either set REGISTRY_AUTH_ID explicitly, or REGISTRY_AUTH_NAME to diff --git a/tests/runpod_smoke/pod.py b/tests/runpod_smoke/pod.py index eac80a31..9fb47c1e 100644 --- a/tests/runpod_smoke/pod.py +++ b/tests/runpod_smoke/pod.py @@ -18,7 +18,12 @@ from typing import Optional from . import api, config -from .checks import ssh_probe, system_log_errors +from .checks import ( + set_ssh_endpoint, + ssh_probe, + ssh_user_for, + system_log_errors, +) from .instances import detect_cuda_version, pick_cpu_flavor from .log import log @@ -257,17 +262,24 @@ def pod_state(pod_id: str) -> dict: terminal states. `ssh.direct` is null until the pod has a machine assignment and a public - port for `22/tcp`; that transition is the readiness signal we poll for. + port for `22/tcp`. RunPod does not always allocate that port — the pod + then shows only `ssh.proxy` in the console — so both endpoints are + returned and the readiness poll accepts whichever answers. """ status, data = api.request_with_retries("GET", f"/pods/{pod_id}", timeout=30) if not (200 <= status < 300) or not isinstance(data, dict): return {} ssh = data.get("ssh") or {} direct = ssh.get("direct") or {} + proxy = ssh.get("proxy") or {} return { "status": data.get("status"), "ssh_ip": direct.get("host") or "", "ssh_port": int(direct.get("port") or 0), + "proxy_host": proxy.get("host") or "", + "proxy_port": int(proxy.get("port") or 0), + "proxy_user": proxy.get("username") or "", + "proxy_command": proxy.get("command") or "", "cuda_version": data.get("cudaVersion") or "", "cost": data.get("cost"), "data_center": data.get("dataCenterId") or "", @@ -320,26 +332,52 @@ def _print_stall_hint(pod_id: str, elapsed: int) -> None: ) +def _ssh_endpoints(st: dict) -> list[tuple[str, str, int]]: + """Reachable-SSH candidates for this pod, best first: (kind, host, port). + + Direct comes first because it is a plain TCP hop to the container's sshd. + The proxy adds a relay, is documented as carrying an interactive shell + only, and rejects a connection with "Your SSH client doesn't support PTY" + unless a terminal is allocated — so it is registered with the API's own + invocation and a forced `-tt`. Registration keys on (host, port) so every + later SSH call can keep addressing endpoints the same way. + """ + out: list[tuple[str, str, int]] = [] + host, port = st.get("ssh_ip") or "", int(st.get("ssh_port") or 0) + if host and port: + out.append(("direct", host, port)) + p_host, p_port = st.get("proxy_host") or "", int(st.get("proxy_port") or 0) + if p_host and p_port and st.get("proxy_user"): + set_ssh_endpoint( + p_host, p_port, str(st["proxy_user"]), + str(st.get("proxy_command") or ""), pty=True, + ) + out.append(("proxy", p_host, p_port)) + return out + + def _probe_ssh_endpoint( + kind: str, host: str, port: int, pod_status_value: object, elapsed: int, ssh_attempts: int, last_summary: Optional[tuple], -) -> tuple[Optional[tuple[str, str]], tuple]: +) -> tuple[Optional[tuple[str, str, tuple[str, int]]], tuple]: """One SSH probe against an assigned endpoint. Returns: (outcome | None, summary_for_dedup) - `outcome` is `("RUNNING", detail)` when the probe succeeds; otherwise - None — caller keeps polling. `summary_for_dedup` is the value the - caller compares against `last_summary` to dedup the log line. + `outcome` is `("RUNNING", detail, (host, port))` when the probe succeeds; + otherwise None — caller keeps polling. `summary_for_dedup` is the value + the caller compares against `last_summary` to dedup the log line. """ + label = f"{ssh_user_for(host, port)}@{host}:{port}" ok, err = ssh_probe(host, port, timeout=8) summary = (pod_status_value, host, port, ok) if summary != last_summary: log( - f"t+{elapsed}s endpoint=root@{host}:{port} " + f"t+{elapsed}s endpoint={label} ({kind}) " f"ssh_probe={'OK' if ok else 'FAIL'} (#{ssh_attempts})" + (f" — {err}" if not ok and err else ""), indent=2, @@ -348,29 +386,32 @@ def _probe_ssh_endpoint( return ( "RUNNING", f"ssh probe succeeded after {elapsed}s " - f"({ssh_attempts} attempts, endpoint root@{host}:{port})", + f"({ssh_attempts} attempts, {kind} endpoint {label})", + (host, port), ), summary return None, summary -def wait_for_running(pod_id: str) -> tuple[str, str]: - """Returns (outcome, detail). Outcome is one of: - 'RUNNING' SSH probe to root@: succeeded — - the container's sshd is up, which means it has fully - booted and we can trust it as healthy. +def wait_for_running(pod_id: str) -> tuple[str, str, tuple[str, int]]: + """Returns (outcome, detail, endpoint). `endpoint` is the (host, port) + that answered — direct or proxy — and ('', 0) when nothing did. Outcome + is one of: + 'RUNNING' an SSH probe succeeded, so the container's sshd is up, + which means it has fully booted and we can trust it as + healthy. Every later check reuses this endpoint. 'TERMINAL' status reached EXITED / ERROR / TERMINATED. 'TIMEOUT' SSH never reachable within CREATE_TIMEOUT — pod stuck initializing (capacity issue or image broken). SSH probing is the real health-check. We poll `GET /v2/pods/{id}` for - `ssh.direct` (populated once a machine is allocated and `22/tcp` gets a - public port), then try `ssh root@host -p port 'echo ready'` until it - succeeds. A successful SSH means the container booted and sshd started — - a much stronger signal than any status field. + either endpoint, then try `ssh @host 'echo ready'` against each + until one succeeds. A successful SSH means the container booted and sshd + started — a much stronger signal than any status field. """ start = time.time() deadline = start + config.CREATE_TIMEOUT - last_summary: Optional[tuple] = None + last_summary: Optional[tuple] = None # the "no endpoint yet" line + last_summaries: dict[tuple[str, int], tuple] = {} # one per endpoint last_status: Optional[str] = None ssh_attempts = 0 stall_hinted = False # one-time hint when pod has no ssh endpoint for a while @@ -394,22 +435,47 @@ def wait_for_running(pod_id: str) -> tuple[str, str]: _log_system_errors(pod_id, f"pod entered {pod_status_value}") return "TERMINAL", ( f"pod entered {pod_status_value} after {elapsed}s" - ) - - if host and port: - ssh_attempts += 1 - outcome, last_summary = _probe_ssh_endpoint( - host, int(port), pod_status_value, elapsed, ssh_attempts, - last_summary, - ) - if outcome is not None: - return outcome + ), ("", 0) + + endpoints = _ssh_endpoints(st) + kinds = {kind for kind, _h, _p in endpoints} + # Proxy-only for this long means RunPod never allocated the direct + # port and the proxy has already refused us, so nothing will change + # by waiting — stop billing the pod. + if ( + config.DIRECT_PORT_TIMEOUT + and kinds == {"proxy"} + and ssh_attempts + and elapsed >= config.DIRECT_PORT_TIMEOUT + ): + _log_system_errors(pod_id, f"proxy-only after {elapsed}s") + return "TIMEOUT", ( + f"RunPod never allocated a public port for 22/tcp in " + f"{elapsed}s and the SSH proxy refused {ssh_attempts} " + "probe(s) — the pod is running but unreachable. Giving up " + f"early (DIRECT_PORT_TIMEOUT={config.DIRECT_PORT_TIMEOUT}s) " + f"instead of waiting out CREATE_TIMEOUT=" + f"{config.CREATE_TIMEOUT}s" + ), ("", 0) + if endpoints: + for kind, e_host, e_port in endpoints: + ssh_attempts += 1 + # Dedup per endpoint: two endpoints alternating would each + # look like a change to a single shared `last_summary`, so a + # stuck pod would log every probe instead of just the first. + key = (e_host, e_port) + outcome, last_summaries[key] = _probe_ssh_endpoint( + kind, e_host, e_port, pod_status_value, elapsed, + ssh_attempts, last_summaries.get(key), + ) + if outcome is not None: + return outcome else: summary = (pod_status_value, host, port, False) if summary != last_summary: log( f"t+{elapsed}s status={pod_status_value!r} " - "ssh endpoint not assigned yet", + "no ssh endpoint assigned yet (neither direct nor proxy)", indent=2, ) last_summary = summary @@ -422,10 +488,10 @@ def wait_for_running(pod_id: str) -> tuple[str, str]: _log_system_errors(pod_id, f"timeout after {config.CREATE_TIMEOUT}s") return "TIMEOUT", ( - f"SSH endpoint never became reachable in {config.CREATE_TIMEOUT}s " - f"({ssh_attempts} probes) — pod stuck initializing. Likely causes: " - "(1) slow/throttled image pull (check UI for pull progress), " - "(2) Docker Hub rate limit if many parallel pulls of the same image, " - "(3) host scheduling delay on a saturated DC — " - "see system-log error markers above (if any)" - ) + f"no SSH endpoint became reachable in {config.CREATE_TIMEOUT}s " + f"({ssh_attempts} probes, direct and proxy) — pod stuck " + "initializing. Likely causes: (1) slow/throttled image pull (check " + "UI for pull progress), (2) Docker Hub rate limit if many parallel " + "pulls of the same image, (3) host scheduling delay on a saturated " + "DC — see system-log error markers above (if any)" + ), ("", 0) diff --git a/tests/runpod_smoke/runner.py b/tests/runpod_smoke/runner.py index b925fd29..73292638 100644 --- a/tests/runpod_smoke/runner.py +++ b/tests/runpod_smoke/runner.py @@ -199,17 +199,20 @@ def _classify_non_running( ) -> _Outcome: """Map a non-RUNNING terminal state to STUCK or FAIL. - TIMEOUT with no SSH endpoint ever assigned is almost always a - scheduler/host issue, not the image: a different GPU type lands on - a different host pool and usually works. Anything else (EXITED, - TERMINATED, FAILED, RUNNING-then-died) is a container problem — the - image is broken, another GPU won't help. - - A container-init rejection overrides that heuristic. It looks identical - from the outside — no SSH, no RUNNING — but it is a verdict about the - image, so it must not be reported as a retryable host problem.""" - st = pod_state(pod_id) - ever_had_ssh = bool(st.get("ssh_ip") and st.get("ssh_port")) + TIMEOUT is a host/infrastructure verdict, so it retries on another + instance type. Every timeout observed so far was one: RunPod not + allocating a direct TCP port, an unreachable assigned port, or + provisioning that never finished. It used to be split by whether an + endpoint had been assigned, and the assigned-but-silent half was called + a broken image — 7 timeouts in one catalog sweep, none of them an image + fault, is what retired that theory. + + Anything else (EXITED, TERMINATED, FAILED, RUNNING-then-died) is a + container problem — the image is broken and another GPU won't help. + + A container-init rejection overrides all of it. It looks identical from + the outside — no SSH, no RUNNING — but it is a verdict about the image, + so it must not be reported as a retryable host problem.""" # Dumped before the verdict so the classification can use its findings. sys_errors = dump_pod_logs(pod_id, image) blocker = host_incompatibility(sys_errors) @@ -221,10 +224,9 @@ def _classify_non_running( indent=2, ) return "FAIL", f"container init rejected the image: {blocker}" - if state == "TIMEOUT" and not ever_had_ssh: + if state == "TIMEOUT": log( - f"{state.lower()} -- {detail} -- STUCK (no SSH endpoint " - "was ever assigned; trying next instance type)", + f"{state.lower()} -- {detail} -- STUCK (trying next instance type)", indent=2, ) return "STUCK", "" @@ -494,14 +496,16 @@ def test_pair( ) try: - state, wait_detail = wait_for_running(pod_id) + state, wait_detail, endpoint = wait_for_running(pod_id) if state != "RUNNING": return _classify_non_running(state, wait_detail, pod_id, image) log(f"smoke check passed: {wait_detail}", indent=2) st = pod_state(pod_id) - host = st.get("ssh_ip") or "" - port = int(st.get("ssh_port") or 0) + # The endpoint that answered the readiness probe, which may be the + # proxy — RunPod does not always allocate a direct TCP port, so + # re-reading ssh.direct here would throw away a working connection. + host, port = endpoint # pod_state already carries cudaVersion, so the common path costs no # extra request. It is nullable until the scheduler has assigned a From dacd07d6d0919df6e68b7a073e171a3a01ebd4ce Mon Sep 17 00:00:00 2001 From: chmokachka Date: Tue, 1 Sep 2026 21:11:56 +0300 Subject: [PATCH 23/33] feat: removed tmp workflows --- .../zz-temp-gpu-compat-check-all.yml | 202 ---------------- .../zz-temp-gpu-compat-instances.yml | 219 ------------------ 2 files changed, 421 deletions(-) delete mode 100644 .github/workflows/zz-temp-gpu-compat-check-all.yml delete mode 100644 .github/workflows/zz-temp-gpu-compat-instances.yml diff --git a/.github/workflows/zz-temp-gpu-compat-check-all.yml b/.github/workflows/zz-temp-gpu-compat-check-all.yml deleted file mode 100644 index c756a6db..00000000 --- a/.github/workflows/zz-temp-gpu-compat-check-all.yml +++ /dev/null @@ -1,202 +0,0 @@ -name: "TEMP: gpu-compat via check-all-gpu" - -# ============================================================================ -# TEMPORARY — DELETE BEFORE MERGING THIS BRANCH. -# -# Same purpose as zz-temp-gpu-compat-instances.yml, but the other selection -# branch: check-all-gpu on the SECURE tier. The Normalize / Validate / -# Summarize shell is copied verbatim from gpu-compatibility.yml with -# `inputs.X` swapped for `env.X`. -# -# Trigger: pushing a change to THIS file. -# -# The catalog holds 46 NVIDIA types, so the exclude list below cuts it to 3: -# L4, RTX A4000, RTX A4500 — all SECURE, all around $0.25-0.49/hr. Patterns -# match DISPLAY names, so 'RTX 50*' also swallows 'RTX 5000 Ada' and -# 'PRO 6000 MIG *' is spelled out because those names lack the RTX prefix. -# -# Cost: 3 pods, ~7 min each -> about $0.12. Widen by deleting exclude lines. -# ============================================================================ - -on: - push: - paths: - - .github/workflows/zz-temp-gpu-compat-check-all.yml - -permissions: - contents: read - -concurrency: - group: zz-temp-gpu-compat-check-all - cancel-in-progress: false - -jobs: - compatibility: - runs-on: blacksmith-4vcpu-ubuntu-2404 - env: - # ---- stand-ins for the dispatch inputs ------------------------------- - IN_IMAGE: runpod/pytorch:1.2.0-rc.162-cu1281-torch2121-ubuntu2204 - IN_TEST_PORTS: "" - # Empty on purpose: no CUDA axis means one pod per GPU with the floor - # taken from the tag. Set to 'all' only if you want to pay for a matrix. - IN_CUDA_VERSIONS: "" - IN_EXCLUDE_INSTANCES: >- - A100*, A40, B200, B300, - RTX 30*, RTX 40*, RTX 50*, - H100*, H200*, L40*, - RTX 2000 Ada, RTX 6000 Ada, - RTX A2000, RTX A5000, RTX A6000, - RTX PRO *, PRO 6000 MIG *, - Tesla V100, V100 SXM2 - IN_CLOUD: SECURE - IN_INSTANCES: "" - IN_CHECK_ALL_GPU: "true" - IN_TEST_JUPYTER: "true" - IN_TEST_COMFYUI: "false" - IN_TEST_COMFYUI_FUNCTIONAL: "false" - MAX_PARALLEL: "3" - timeout-minutes: 90 - steps: - - name: Checkout - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 - with: - fetch-depth: 0 - - - name: Normalize inputs - id: prep - shell: bash - env: - IMAGE: ${{ env.IN_IMAGE }} - PORTS: ${{ env.IN_TEST_PORTS }} - EXCLUDE: ${{ env.IN_EXCLUDE_INSTANCES }} - INSTANCES: ${{ env.IN_INSTANCES }} - run: | - set -euo pipefail - - to_lines() { - printf '%s' "$1" \ - | tr ',' '\n' \ - | sed 's/^[[:space:]]*//; s/[[:space:]]*$//' \ - | grep -v '^$' || true - } - - emit_multiline() { - { - echo "$1<<__GH_EOF__" - to_lines "$2" - echo "__GH_EOF__" - } >> "$GITHUB_OUTPUT" - } - - IMAGES_CSV=$(printf '%s' "${IMAGE}" | tr -d '[:space:]') - if [[ -z "${IMAGES_CSV}" ]]; then - echo "::error::image input is empty" - exit 1 - fi - refs_json="" - refs_human="" - image_count=0 - while IFS= read -r ref; do - [[ -z "${ref}" ]] && continue - if [[ ! "${ref}" =~ ^[A-Za-z0-9._:/@-]+$ ]]; then - echo "::error::image '${ref}' has characters that aren't valid in an image ref" - exit 1 - fi - if [[ "${ref}" != *:* ]]; then - echo "::error::image '${ref}' has no tag — pass an explicit tag, not a bare repository" - exit 1 - fi - refs_json+="${refs_json:+,}\"${ref}\"" - refs_human+="${refs_human:+, }${ref}" - image_count=$((image_count + 1)) - done <<< "$(printf '%s' "${IMAGES_CSV}" | tr ',' '\n')" - printf 'image-refs=[%s]\n' "${refs_json}" >> "$GITHUB_OUTPUT" - printf 'images=%s\n' "${refs_human}" >> "$GITHUB_OUTPUT" - printf 'image-count=%s\n' "${image_count}" >> "$GITHUB_OUTPUT" - - emit_multiline ports "$(printf '%s' "${PORTS}" | tr ' ' ',')" - emit_multiline exclude "${EXCLUDE}" - emit_multiline instances "${INSTANCES}" - - - name: Validate inputs - shell: bash - env: - CUDA: ${{ env.IN_CUDA_VERSIONS }} - INSTANCES: ${{ env.IN_INSTANCES }} - CHECK_ALL: ${{ env.IN_CHECK_ALL_GPU }} - run: | - set -euo pipefail - instances_trimmed=$(printf '%s' "${INSTANCES}" | tr -d '[:space:]') - if [[ -n "${instances_trimmed}" && "${CHECK_ALL,,}" == "true" ]]; then - echo "::error::choose one: 'instances' lists exact GPUs, 'check-all-gpu' sweeps the catalog. An explicit list wins in the harness, so check-all-gpu would be silently ignored. Clear the instances field, or untick check-all-gpu." - exit 1 - fi - if [[ -z "${instances_trimmed}" && "${CHECK_ALL,,}" != "true" ]]; then - echo "::error::nothing to test: set 'instances', or tick 'check-all-gpu'." - exit 1 - fi - if [[ -n "${CUDA}" && "${CUDA,,}" != "all" ]]; then - normalized=$(printf '%s' "${CUDA}" | tr -d '[:space:]') - [[ "${normalized}" =~ ^[0-9]+\.[0-9]+(,[0-9]+\.[0-9]+)*$ ]] \ - || { echo "::error::cuda-versions must be 'all' or X.Y[,X.Y...] (e.g. '12.8, 13.0'), got '${CUDA}'"; exit 1; } - fi - - - name: Summarize run parameters - shell: bash - env: - IMAGES: ${{ steps.prep.outputs.images }} - IMAGE_COUNT: ${{ steps.prep.outputs.image-count }} - PORTS: ${{ steps.prep.outputs.ports }} - EXCLUDE: ${{ steps.prep.outputs.exclude }} - INSTANCES: ${{ steps.prep.outputs.instances }} - CUDA: ${{ env.IN_CUDA_VERSIONS }} - CLOUD: ${{ env.IN_CLOUD }} - CHECK_ALL: ${{ env.IN_CHECK_ALL_GPU }} - MAX_PARALLEL: ${{ env.MAX_PARALLEL }} - JUPYTER: ${{ env.IN_TEST_JUPYTER }} - run: | - set -euo pipefail - ports_1l=$(printf '%s' "${PORTS}" | paste -sd, - | sed 's/,/, /g') - exclude_1l=$(printf '%s' "${EXCLUDE}" | paste -sd, - | sed 's/,/, /g') - instances_1l=$(printf '%s' "${INSTANCES}" | paste -sd, - | sed 's/,/, /g') - { - echo "### TEMP: gpu-compat via check-all-gpu (SECURE)" - echo - echo "| Setting | Value |" - echo "| --- | --- |" - echo "| Images (${IMAGE_COUNT}) | \`${IMAGES//, /\`, \`}\` |" - echo "| GPUs | ${instances_1l:-whole catalog (check-all-gpu=${CHECK_ALL})} |" - echo "| Cloud tier | ${CLOUD} |" - echo "| CUDA axis | ${CUDA:-none (floor from tag)} |" - echo "| Ports | ${ports_1l:-none} |" - echo "| Excluded GPUs | ${exclude_1l:-none} |" - echo "| Jupyter | ${JUPYTER} |" - echo "| Parallel pods | ${MAX_PARALLEL} |" - } >> "$GITHUB_STEP_SUMMARY" - - - name: Run compatibility matrix - uses: ./.github/actions/smoke-test - with: - image-refs: ${{ steps.prep.outputs.image-refs }} - profile: gpu - runpod-api-key: ${{ secrets.TESTING_RUNPOD_API_KEY }} - ssh-private-key: ${{ secrets.TESTING_RUNPOD_SSH_PRIVATE_KEY }} - - check-all-gpu: ${{ env.IN_CHECK_ALL_GPU }} - instances: ${{ steps.prep.outputs.instances }} - exclude-instances: ${{ steps.prep.outputs.exclude }} - min-vram-gb: "0" - cloud-type: ${{ env.IN_CLOUD }} - cuda-versions: ${{ env.IN_CUDA_VERSIONS }} - - test-ports: ${{ steps.prep.outputs.ports }} - test-jupyter: ${{ env.IN_TEST_JUPYTER }} - test-comfyui: ${{ env.IN_TEST_COMFYUI }} - test-comfyui-functional: ${{ env.IN_TEST_COMFYUI_FUNCTIONAL }} - - upload-results-json: "true" - results-artifact-name: zz-temp-check-all-${{ github.run_id }} - - on-skip: pass - create-timeout: "1200" - max-parallel: ${{ env.MAX_PARALLEL }} diff --git a/.github/workflows/zz-temp-gpu-compat-instances.yml b/.github/workflows/zz-temp-gpu-compat-instances.yml deleted file mode 100644 index c51c0874..00000000 --- a/.github/workflows/zz-temp-gpu-compat-instances.yml +++ /dev/null @@ -1,219 +0,0 @@ -name: "TEMP: gpu-compat via instances" - -# ============================================================================ -# TEMPORARY — DELETE BEFORE MERGING THIS BRANCH. -# -# gpu-compatibility.yml is workflow_dispatch-only, and GitHub does not show -# a dispatch button for a workflow that isn't on the default branch yet. This -# file exists to exercise the same steps from a branch push: the Normalize / -# Validate / Summarize shell is copied verbatim, with `inputs.X` swapped for -# `env.X` so the values are hardcoded below instead of typed into a form. -# -# Trigger: pushing a change to THIS file. Nothing else runs it, so ordinary -# commits on the branch don't spend money. To run it again without editing, -# use "Re-run all jobs" on the run in the Actions tab. -# -# Covers: multi-image splitting, explicit `instances`, cloud tier, and the -# mutual-exclusion guard. -# -# `instances` is a fallback chain, not a matrix — so this is 2 pods (one per -# image), each landing on the first listed GPU with capacity. -# -# Cost: 2 pods, ~7 min each, ~$0.25/hr -> about $0.06. -# ============================================================================ - -on: - push: - paths: - - .github/workflows/zz-temp-gpu-compat-instances.yml - -permissions: - contents: read - -concurrency: - group: zz-temp-gpu-compat-instances - cancel-in-progress: false - -jobs: - compatibility: - runs-on: blacksmith-4vcpu-ubuntu-2404 - env: - # ---- stand-ins for the dispatch inputs ------------------------------- - IN_IMAGE: >- - runpod/pytorch:1.2.0-rc.162-cu1281-torch2121-ubuntu2204, - runpod/pytorch:1.2.0-rc.162-cu1290-torch2130-ubuntu2404 - IN_TEST_PORTS: "" - IN_CUDA_VERSIONS: "" - IN_EXCLUDE_INSTANCES: "" - IN_CLOUD: SECURE - # Both are SECURE and ~$0.25/hr, and both names contain a space, which - # is what the --instance argv loop has to keep intact. - IN_INSTANCES: "RTX A4000, RTX A4500" - IN_CHECK_ALL_GPU: "false" - IN_TEST_JUPYTER: "true" - IN_TEST_COMFYUI: "false" - IN_TEST_COMFYUI_FUNCTIONAL: "false" - MAX_PARALLEL: "2" - timeout-minutes: 90 - steps: - - name: Checkout - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 - with: - fetch-depth: 0 - - - name: Normalize inputs - id: prep - shell: bash - env: - IMAGE: ${{ env.IN_IMAGE }} - PORTS: ${{ env.IN_TEST_PORTS }} - EXCLUDE: ${{ env.IN_EXCLUDE_INSTANCES }} - INSTANCES: ${{ env.IN_INSTANCES }} - run: | - set -euo pipefail - - to_lines() { - printf '%s' "$1" \ - | tr ',' '\n' \ - | sed 's/^[[:space:]]*//; s/[[:space:]]*$//' \ - | grep -v '^$' || true - } - - emit_multiline() { - { - echo "$1<<__GH_EOF__" - to_lines "$2" - echo "__GH_EOF__" - } >> "$GITHUB_OUTPUT" - } - - IMAGES_CSV=$(printf '%s' "${IMAGE}" | tr -d '[:space:]') - if [[ -z "${IMAGES_CSV}" ]]; then - echo "::error::image input is empty" - exit 1 - fi - refs_json="" - refs_human="" - image_count=0 - while IFS= read -r ref; do - [[ -z "${ref}" ]] && continue - if [[ ! "${ref}" =~ ^[A-Za-z0-9._:/@-]+$ ]]; then - echo "::error::image '${ref}' has characters that aren't valid in an image ref" - exit 1 - fi - if [[ "${ref}" != *:* ]]; then - echo "::error::image '${ref}' has no tag — pass an explicit tag, not a bare repository" - exit 1 - fi - refs_json+="${refs_json:+,}\"${ref}\"" - refs_human+="${refs_human:+, }${ref}" - image_count=$((image_count + 1)) - done <<< "$(printf '%s' "${IMAGES_CSV}" | tr ',' '\n')" - printf 'image-refs=[%s]\n' "${refs_json}" >> "$GITHUB_OUTPUT" - printf 'images=%s\n' "${refs_human}" >> "$GITHUB_OUTPUT" - printf 'image-count=%s\n' "${image_count}" >> "$GITHUB_OUTPUT" - - emit_multiline ports "$(printf '%s' "${PORTS}" | tr ' ' ',')" - emit_multiline exclude "${EXCLUDE}" - emit_multiline instances "${INSTANCES}" - - - name: Validate inputs - shell: bash - env: - CUDA: ${{ env.IN_CUDA_VERSIONS }} - INSTANCES: ${{ env.IN_INSTANCES }} - CHECK_ALL: ${{ env.IN_CHECK_ALL_GPU }} - run: | - set -euo pipefail - instances_trimmed=$(printf '%s' "${INSTANCES}" | tr -d '[:space:]') - if [[ -n "${instances_trimmed}" && "${CHECK_ALL,,}" == "true" ]]; then - echo "::error::choose one: 'instances' lists exact GPUs, 'check-all-gpu' sweeps the catalog. An explicit list wins in the harness, so check-all-gpu would be silently ignored. Clear the instances field, or untick check-all-gpu." - exit 1 - fi - if [[ -z "${instances_trimmed}" && "${CHECK_ALL,,}" != "true" ]]; then - echo "::error::nothing to test: set 'instances', or tick 'check-all-gpu'." - exit 1 - fi - if [[ -n "${CUDA}" && "${CUDA,,}" != "all" ]]; then - normalized=$(printf '%s' "${CUDA}" | tr -d '[:space:]') - [[ "${normalized}" =~ ^[0-9]+\.[0-9]+(,[0-9]+\.[0-9]+)*$ ]] \ - || { echo "::error::cuda-versions must be 'all' or X.Y[,X.Y...] (e.g. '12.8, 13.0'), got '${CUDA}'"; exit 1; } - fi - - # TEMP-only: the mutual exclusion is enforced twice — by the Validate - # step above and by the generator the composite action calls. Assert - # the generator half here, since no run can exercise both branches. - - name: Assert instances + check-all-gpu is rejected - shell: bash - run: | - set -uo pipefail - out=$(python3 .github/scripts/generate_test_manifest.py \ - --profile gpu --refs '["runpod/pytorch:1.2.0-rc.162-cu1281-torch2121-ubuntu2204"]' \ - --instance "RTX A4000" --check-all-gpu \ - --output "${RUNNER_TEMP}/should-not-exist.yaml" 2>&1) && rc=0 || rc=$? - echo "${out}" - if [[ "${rc}" -eq 0 ]]; then - echo "::error::generator accepted --instance together with --check-all-gpu" - exit 1 - fi - echo "OK: generator exited ${rc}" - - - name: Summarize run parameters - shell: bash - env: - IMAGES: ${{ steps.prep.outputs.images }} - IMAGE_COUNT: ${{ steps.prep.outputs.image-count }} - PORTS: ${{ steps.prep.outputs.ports }} - EXCLUDE: ${{ steps.prep.outputs.exclude }} - INSTANCES: ${{ steps.prep.outputs.instances }} - CUDA: ${{ env.IN_CUDA_VERSIONS }} - CLOUD: ${{ env.IN_CLOUD }} - CHECK_ALL: ${{ env.IN_CHECK_ALL_GPU }} - MAX_PARALLEL: ${{ env.MAX_PARALLEL }} - JUPYTER: ${{ env.IN_TEST_JUPYTER }} - run: | - set -euo pipefail - ports_1l=$(printf '%s' "${PORTS}" | paste -sd, - | sed 's/,/, /g') - exclude_1l=$(printf '%s' "${EXCLUDE}" | paste -sd, - | sed 's/,/, /g') - instances_1l=$(printf '%s' "${INSTANCES}" | paste -sd, - | sed 's/,/, /g') - { - echo "### TEMP: gpu-compat via explicit instances" - echo - echo "| Setting | Value |" - echo "| --- | --- |" - echo "| Images (${IMAGE_COUNT}) | \`${IMAGES//, /\`, \`}\` |" - echo "| GPUs | ${instances_1l:-whole catalog (check-all-gpu=${CHECK_ALL})} |" - echo "| Cloud tier | ${CLOUD} |" - echo "| CUDA axis | ${CUDA:-none (floor from tag)} |" - echo "| Ports | ${ports_1l:-none} |" - echo "| Excluded GPUs | ${exclude_1l:-none} |" - echo "| Jupyter | ${JUPYTER} |" - echo "| Parallel pods | ${MAX_PARALLEL} |" - } >> "$GITHUB_STEP_SUMMARY" - - - name: Run compatibility matrix - uses: ./.github/actions/smoke-test - with: - image-refs: ${{ steps.prep.outputs.image-refs }} - profile: gpu - runpod-api-key: ${{ secrets.TESTING_RUNPOD_API_KEY }} - ssh-private-key: ${{ secrets.TESTING_RUNPOD_SSH_PRIVATE_KEY }} - - check-all-gpu: ${{ env.IN_CHECK_ALL_GPU }} - instances: ${{ steps.prep.outputs.instances }} - exclude-instances: ${{ steps.prep.outputs.exclude }} - min-vram-gb: "0" - cloud-type: ${{ env.IN_CLOUD }} - cuda-versions: ${{ env.IN_CUDA_VERSIONS }} - - test-ports: ${{ steps.prep.outputs.ports }} - test-jupyter: ${{ env.IN_TEST_JUPYTER }} - test-comfyui: ${{ env.IN_TEST_COMFYUI }} - test-comfyui-functional: ${{ env.IN_TEST_COMFYUI_FUNCTIONAL }} - - upload-results-json: "true" - results-artifact-name: zz-temp-instances-${{ github.run_id }} - - on-skip: pass - create-timeout: "1200" - max-parallel: ${{ env.MAX_PARALLEL }} From 6069bcf92a593f897e56065e8c6743c91d72eedc Mon Sep 17 00:00:00 2001 From: chmokachka Date: Wed, 2 Sep 2026 11:40:52 +0300 Subject: [PATCH 24/33] feat: hadolint --- .github/workflows/hadolint-pr.yml | 1 + .github/workflows/hadolint-push.yml | 1 + 2 files changed, 2 insertions(+) diff --git a/.github/workflows/hadolint-pr.yml b/.github/workflows/hadolint-pr.yml index 0a844d39..aa7f2442 100644 --- a/.github/workflows/hadolint-pr.yml +++ b/.github/workflows/hadolint-pr.yml @@ -24,6 +24,7 @@ jobs: ignore: "DL3008" - dockerfile: official-templates/pytorch-cluster/Dockerfile ignore: "DL3008" + - dockerfile: official-templates/comfyui/Dockerfile steps: - name: Checkout uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 diff --git a/.github/workflows/hadolint-push.yml b/.github/workflows/hadolint-push.yml index d92e97db..614ce802 100644 --- a/.github/workflows/hadolint-push.yml +++ b/.github/workflows/hadolint-push.yml @@ -27,6 +27,7 @@ jobs: ignore: "DL3008" - dockerfile: official-templates/pytorch-cluster/Dockerfile ignore: "DL3008" + - dockerfile: official-template/comfyui/Dockerfile steps: - name: Checkout uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 From eeedcf53c72a77a02789ad70450177181593130a Mon Sep 17 00:00:00 2001 From: chmokachka Date: Wed, 2 Sep 2026 11:54:51 +0300 Subject: [PATCH 25/33] fix: hadolint findings --- .github/workflows/hadolint-pr.yml | 1 + .github/workflows/hadolint-push.yml | 3 +- official-templates/comfyui/Dockerfile | 43 ++++++++++++++++----------- 3 files changed, 28 insertions(+), 19 deletions(-) diff --git a/.github/workflows/hadolint-pr.yml b/.github/workflows/hadolint-pr.yml index aa7f2442..5509608f 100644 --- a/.github/workflows/hadolint-pr.yml +++ b/.github/workflows/hadolint-pr.yml @@ -25,6 +25,7 @@ jobs: - dockerfile: official-templates/pytorch-cluster/Dockerfile ignore: "DL3008" - dockerfile: official-templates/comfyui/Dockerfile + ignore: "DL3008" steps: - name: Checkout uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 diff --git a/.github/workflows/hadolint-push.yml b/.github/workflows/hadolint-push.yml index 614ce802..225aad12 100644 --- a/.github/workflows/hadolint-push.yml +++ b/.github/workflows/hadolint-push.yml @@ -27,7 +27,8 @@ jobs: ignore: "DL3008" - dockerfile: official-templates/pytorch-cluster/Dockerfile ignore: "DL3008" - - dockerfile: official-template/comfyui/Dockerfile + - dockerfile: official-templates/comfyui/Dockerfile + ignore: "DL3008" steps: - name: Checkout uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 diff --git a/official-templates/comfyui/Dockerfile b/official-templates/comfyui/Dockerfile index 3e108e6d..79855226 100644 --- a/official-templates/comfyui/Dockerfile +++ b/official-templates/comfyui/Dockerfile @@ -30,7 +30,7 @@ RUN apt-get update && \ python3.12-venv \ python3.12-dev \ build-essential \ - && wget https://developer.download.nvidia.com/compute/cuda/repos/ubuntu2404/x86_64/cuda-keyring_1.1-1_all.deb \ + && curl -fSL https://developer.download.nvidia.com/compute/cuda/repos/ubuntu2404/x86_64/cuda-keyring_1.1-1_all.deb -o cuda-keyring_1.1-1_all.deb \ && dpkg -i cuda-keyring_1.1-1_all.deb \ && apt-get update \ && apt-get install -y --no-install-recommends cuda-minimal-build-${CUDA_VERSION_DASH} libcusparse-dev-${CUDA_VERSION_DASH} \ @@ -66,21 +66,27 @@ RUN curl -fSL "https://github.com/ltdrdata/ComfyUI-Manager/archive/${MANAGER_SHA # Init git repos with upstream remotes so ComfyUI-Manager can detect versions # and users can update via Manager at their own risk -RUN cd /tmp/build/ComfyUI && \ - git init && git add -A && git -c user.name=- -c user.email=- commit -q -m "ComfyUI ${COMFYUI_VERSION}" && git tag "${COMFYUI_VERSION}" && \ +WORKDIR /tmp/build/ComfyUI +RUN git init && git add -A && \ + git -c user.name=- -c user.email=- commit -q -m "ComfyUI ${COMFYUI_VERSION}" && \ + git tag "${COMFYUI_VERSION}" && \ git remote add origin https://github.com/comfyanonymous/ComfyUI.git && \ - cd /tmp/build/ComfyUI/custom_nodes/ComfyUI-Manager && \ - git init && git add -A && git -c user.name=- -c user.email=- commit -q -m "ComfyUI-Manager ${MANAGER_SHA}" && \ - git remote add origin https://github.com/ltdrdata/ComfyUI-Manager.git && \ - cd /tmp/build/ComfyUI/custom_nodes/ComfyUI-KJNodes && \ - git init && git add -A && git -c user.name=- -c user.email=- commit -q -m "ComfyUI-KJNodes ${KJNODES_SHA}" && \ - git remote add origin https://github.com/kijai/ComfyUI-KJNodes.git && \ - cd /tmp/build/ComfyUI/custom_nodes/Civicomfy && \ - git init && git add -A && git -c user.name=- -c user.email=- commit -q -m "Civicomfy ${CIVICOMFY_SHA}" && \ - git remote add origin https://github.com/MoonGoblinDev/Civicomfy.git && \ - cd /tmp/build/ComfyUI/custom_nodes/ComfyUI-RunpodDirect && \ - git init && git add -A && git -c user.name=- -c user.email=- commit -q -m "ComfyUI-RunpodDirect ${RUNPODDIRECT_SHA}" && \ - git remote add origin https://github.com/MadiatorLabs/ComfyUI-RunpodDirect.git + git -C custom_nodes/ComfyUI-Manager init && \ + git -C custom_nodes/ComfyUI-Manager add -A && \ + git -C custom_nodes/ComfyUI-Manager -c user.name=- -c user.email=- commit -q -m "ComfyUI-Manager ${MANAGER_SHA}" && \ + git -C custom_nodes/ComfyUI-Manager remote add origin https://github.com/ltdrdata/ComfyUI-Manager.git && \ + git -C custom_nodes/ComfyUI-KJNodes init && \ + git -C custom_nodes/ComfyUI-KJNodes add -A && \ + git -C custom_nodes/ComfyUI-KJNodes -c user.name=- -c user.email=- commit -q -m "ComfyUI-KJNodes ${KJNODES_SHA}" && \ + git -C custom_nodes/ComfyUI-KJNodes remote add origin https://github.com/kijai/ComfyUI-KJNodes.git && \ + git -C custom_nodes/Civicomfy init && \ + git -C custom_nodes/Civicomfy add -A && \ + git -C custom_nodes/Civicomfy -c user.name=- -c user.email=- commit -q -m "Civicomfy ${CIVICOMFY_SHA}" && \ + git -C custom_nodes/Civicomfy remote add origin https://github.com/MoonGoblinDev/Civicomfy.git && \ + git -C custom_nodes/ComfyUI-RunpodDirect init && \ + git -C custom_nodes/ComfyUI-RunpodDirect add -A && \ + git -C custom_nodes/ComfyUI-RunpodDirect -c user.name=- -c user.email=- commit -q -m "ComfyUI-RunpodDirect ${RUNPODDIRECT_SHA}" && \ + git -C custom_nodes/ComfyUI-RunpodDirect remote add origin https://github.com/MadiatorLabs/ComfyUI-RunpodDirect.git # Generate lock file from all requirements (including torch pins), then install with hash verification WORKDIR /tmp/build @@ -177,7 +183,7 @@ RUN apt-get update && \ openssl \ ffmpeg \ rsync \ - && wget https://developer.download.nvidia.com/compute/cuda/repos/ubuntu2404/x86_64/cuda-keyring_1.1-1_all.deb \ + && curl -fSL https://developer.download.nvidia.com/compute/cuda/repos/ubuntu2404/x86_64/cuda-keyring_1.1-1_all.deb -o cuda-keyring_1.1-1_all.deb \ && dpkg -i cuda-keyring_1.1-1_all.deb \ && apt-get update \ && apt-get install -y --no-install-recommends cuda-minimal-build-${CUDA_VERSION_DASH} \ @@ -205,9 +211,10 @@ RUN pip uninstall -y uv 2>/dev/null || true && \ # Install FileBrowser (pinned version with checksum) RUN curl -fSL "https://github.com/filebrowser/filebrowser/releases/download/${FILEBROWSER_VERSION}/linux-amd64-filebrowser.tar.gz" -o /tmp/fb.tar.gz && \ - echo "${FILEBROWSER_SHA256} /tmp/fb.tar.gz" | sha256sum -c - && \ + printf '%s /tmp/fb.tar.gz\n' "${FILEBROWSER_SHA256}" > /tmp/fb.sha256 && \ + sha256sum -c /tmp/fb.sha256 && \ tar xzf /tmp/fb.tar.gz -C /usr/local/bin filebrowser && \ - rm /tmp/fb.tar.gz + rm /tmp/fb.tar.gz /tmp/fb.sha256 # Set CUDA environment variables ENV PATH=/usr/local/cuda/bin:${PATH} From dbde59950c20ca86792e9009dd6e94c738c13348 Mon Sep 17 00:00:00 2001 From: mariiachekmasova-runpod Date: Mon, 7 Sep 2026 12:03:15 +0300 Subject: [PATCH 26/33] feat: TEM-89 pycache --- official-templates/comfyui/Dockerfile | 14 ++++++++ official-templates/comfyui/scripts/start.sh | 40 +++++++++++++++------ 2 files changed, 43 insertions(+), 11 deletions(-) diff --git a/official-templates/comfyui/Dockerfile b/official-templates/comfyui/Dockerfile index 79855226..91915d69 100644 --- a/official-templates/comfyui/Dockerfile +++ b/official-templates/comfyui/Dockerfile @@ -64,6 +64,13 @@ RUN curl -fSL "https://github.com/ltdrdata/ComfyUI-Manager/archive/${MANAGER_SHA curl -fSL "https://github.com/MadiatorLabs/ComfyUI-RunpodDirect/archive/${RUNPODDIRECT_SHA}.tar.gz" -o runpoddirect.tar.gz && \ mkdir -p ComfyUI-RunpodDirect && tar xzf runpoddirect.tar.gz --strip-components=1 -C ComfyUI-RunpodDirect && rm runpoddirect.tar.gz +# Manager's get_pip_cmd() probes `python -m pip --version` with a 5s timeout and +# treats a timeout as "pip is missing" — on a slow network volume a working pip +# reads as absent. Patched before the git commit below so the tree stays clean. +# The count assertion fails the build if the upstream pattern moves. +RUN test "$(grep -c 'timeout=5)' ComfyUI-Manager/glob/manager_util.py)" = 2 && \ + sed -i 's/timeout=5)/timeout=60)/g' ComfyUI-Manager/glob/manager_util.py + # Init git repos with upstream remotes so ComfyUI-Manager can detect versions # and users can update via Manager at their own risk WORKDIR /tmp/build/ComfyUI @@ -140,9 +147,13 @@ ENV DEBIAN_FRONTEND=noninteractive ENV PYTHONUNBUFFERED=1 ENV IMAGEIO_FFMPEG_EXE=/usr/bin/ffmpeg ENV FILEBROWSER_CONFIG=/workspace/runpod-slim/.filebrowser.json +# Keep bytecode off the /workspace volume: when the volume is full, writing +# .pyc fails silently and every start recompiles pip from scratch. +ENV PYTHONPYCACHEPREFIX=/var/cache/comfyui-pyc # ---- CUDA variant (re-declared for runtime stage) ---- ARG CUDA_VERSION_DASH=12-8 +ARG TORCH_INDEX_SUFFIX=cu128 ARG TORCH_VERSION ARG TORCHVISION_VERSION ARG TORCHAUDIO_VERSION @@ -152,6 +163,9 @@ ARG FILEBROWSER_VERSION ARG FILEBROWSER_SHA256 # Keep runtime pip installs aligned with the baked CUDA-specific PyTorch stack. +# The pins carry a local version (+cu128), which PyPI does not serve — without +# this index every runtime install that touches torch fails to resolve. +ENV PIP_EXTRA_INDEX_URL="https://download.pytorch.org/whl/${TORCH_INDEX_SUFFIX}" RUN printf "torch==%s\ntorchvision==%s\ntorchaudio==%s\n" \ "$TORCH_VERSION" "$TORCHVISION_VERSION" "$TORCHAUDIO_VERSION" \ > /opt/comfyui-runtime-constraints.txt diff --git a/official-templates/comfyui/scripts/start.sh b/official-templates/comfyui/scripts/start.sh index fb87c0a3..e14a20f2 100755 --- a/official-templates/comfyui/scripts/start.sh +++ b/official-templates/comfyui/scripts/start.sh @@ -170,6 +170,21 @@ upgrade_comfyui_if_needed() { echo "ComfyUI workspace upgraded successfully" } +# The venv has no pip of its own, so a bare `pip` would resolve to +# /usr/local/bin/pip and install against the base interpreter. Custom-node +# install scripts do call it that way. Existing venvs keep their real pip. +create_pip_shim() { + if [ -e "$VENV_DIR/bin/pip" ]; then + return + fi + if printf '#!/bin/sh\nexec "%s/bin/python" -m pip "$@"\n' "$VENV_DIR" \ + > "$VENV_DIR/bin/pip" 2>/dev/null; then + chmod +x "$VENV_DIR/bin/pip" + else + echo "WARNING: could not write $VENV_DIR/bin/pip — is the volume full?" + fi +} + log_cuda_venv_diagnostics() { local expected_build local_packages status expected_build=$(sed -n 's/^torch==.*+\(cu[0-9][0-9]*\).*$/\1/p' \ @@ -303,11 +318,10 @@ if [ -d "$OLD_VENV_DIR" ] && [ ! -d "$VENV_DIR" ]; then echo "=============================================" mv "$OLD_VENV_DIR" "${OLD_VENV_DIR}.bak" cd "$COMFYUI_DIR" - python3.12 -m venv --system-site-packages "$VENV_DIR" + python3.12 -m venv --system-site-packages --without-pip "$VENV_DIR" # The venv is created at runtime, so there is nothing for shellcheck to follow. # shellcheck source=/dev/null source "$VENV_DIR/bin/activate" - python -m ensurepip # Skip nodes baked into the image — their deps are in system site-packages CURRENT=0 INSTALLED=0 @@ -319,12 +333,12 @@ if [ -d "$OLD_VENV_DIR" ] && [ ! -d "$VENV_DIR" ]; then esac CURRENT=$((CURRENT + 1)) echo "[$CURRENT] $NODE_NAME" - pip install -r "$req" 2>&1 | grep -E "^(Successfully|ERROR)" || true + python -m pip install -r "$req" 2>&1 | grep -E "^(Successfully|ERROR)" || true INSTALLED=$((INSTALLED + 1)) fi done echo "Ensuring ComfyUI requirements are present..." - pip install -r "$COMFYUI_DIR/requirements.txt" 2>&1 | grep -E "^(Successfully|ERROR)" || true + python -m pip install -r "$COMFYUI_DIR/requirements.txt" 2>&1 | grep -E "^(Successfully|ERROR)" || true echo "Migration complete — $INSTALLED user nodes processed (${NODE_COUNT} total, baked nodes skipped)" echo "Old venv backed up at ${OLD_VENV_DIR}.bak — delete it to free space:" echo " rm -rf ${OLD_VENV_DIR}.bak" @@ -343,13 +357,14 @@ if [ ! -d "$COMFYUI_DIR" ] || [ ! -d "$VENV_DIR" ]; then # Create venv with access to system packages (torch, numpy, etc. pre-installed in image) if [ ! -d "$VENV_DIR" ]; then cd "$COMFYUI_DIR" - python3.12 -m venv --system-site-packages "$VENV_DIR" + # --without-pip: pip stays in the image (local disk, bytecode compiled + # at build) instead of on the network volume, where importing it can + # exceed ComfyUI-Manager's probe timeout. --system-site-packages keeps + # it importable, and installs still land in this venv via sys.prefix. + python3.12 -m venv --system-site-packages --without-pip "$VENV_DIR" # shellcheck source=/dev/null source "$VENV_DIR/bin/activate" - # Ensure pip is available in the venv (needed for ComfyUI-Manager) - python -m ensurepip - echo "Base packages (torch, numpy, etc.) available from system site-packages" echo "ComfyUI ready — all dependencies pre-installed in image" fi @@ -360,9 +375,12 @@ else echo "Using existing ComfyUI installation" fi -# Warm up pip so ComfyUI-Manager's 5s timeout check doesn't fail on cold start. -# Log wall time — Manager fails if `python -m pip --version` takes >5s. -echo "Warming up pip (Manager timeout is 5s)..." +create_pip_shim + +# Warm up pip before Manager probes it: this compiles pip into +# PYTHONPYCACHEPREFIX on container-local disk, so Manager's probe reads warm +# bytecode. Log wall time — the Dockerfile raises Manager's timeout to 60s. +echo "Warming up pip (Manager timeout is 60s)..." time python -m pip --version log_cuda_venv_diagnostics From 03d886104b32414d9d67ffa0d099befd3fed6890 Mon Sep 17 00:00:00 2001 From: mariiachekmasova-runpod Date: Mon, 7 Sep 2026 17:42:14 +0300 Subject: [PATCH 27/33] feat: pip timeout --- official-templates/comfyui/Dockerfile | 2 +- official-templates/comfyui/scripts/start.sh | 5 ++++- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/official-templates/comfyui/Dockerfile b/official-templates/comfyui/Dockerfile index 91915d69..5d770870 100644 --- a/official-templates/comfyui/Dockerfile +++ b/official-templates/comfyui/Dockerfile @@ -69,7 +69,7 @@ RUN curl -fSL "https://github.com/ltdrdata/ComfyUI-Manager/archive/${MANAGER_SHA # reads as absent. Patched before the git commit below so the tree stays clean. # The count assertion fails the build if the upstream pattern moves. RUN test "$(grep -c 'timeout=5)' ComfyUI-Manager/glob/manager_util.py)" = 2 && \ - sed -i 's/timeout=5)/timeout=60)/g' ComfyUI-Manager/glob/manager_util.py + sed -i 's/timeout=5)/timeout=30)/g' ComfyUI-Manager/glob/manager_util.py # Init git repos with upstream remotes so ComfyUI-Manager can detect versions # and users can update via Manager at their own risk diff --git a/official-templates/comfyui/scripts/start.sh b/official-templates/comfyui/scripts/start.sh index e14a20f2..a8e8c7c8 100755 --- a/official-templates/comfyui/scripts/start.sh +++ b/official-templates/comfyui/scripts/start.sh @@ -60,7 +60,10 @@ export_env_vars() { : > "$SSH_ENV_DIR" # Export to multiple locations for maximum compatibility - printenv | grep -E '^RUNPOD_|^PATH=|^_=|^CUDA|^LD_LIBRARY_PATH|^PYTHONPATH|^PIP_CONSTRAINT=' | while read -r line; do + # PIP_EXTRA_INDEX_URL travels with PIP_CONSTRAINT: the constraint pins a + # +cuXXX build that only the PyTorch index serves, so one without the other + # makes every install in an SSH session unresolvable. + printenv | grep -E '^RUNPOD_|^PATH=|^_=|^CUDA|^LD_LIBRARY_PATH|^PYTHONPATH|^PIP_CONSTRAINT=|^PIP_EXTRA_INDEX_URL=|^PYTHONPYCACHEPREFIX=' | while read -r line; do # Get variable name and value name=$(echo "$line" | cut -d= -f1) value=$(echo "$line" | cut -d= -f2-) From fcf1fc806dbe6b70d126f1a46dfb97a00ae8d88e Mon Sep 17 00:00:00 2001 From: mariiachekmasova-runpod Date: Mon, 7 Sep 2026 18:47:43 +0300 Subject: [PATCH 28/33] feat: removed PYTHONPYCACHEPREFIX --- official-templates/comfyui/Dockerfile | 3 --- official-templates/comfyui/scripts/start.sh | 7 +++---- 2 files changed, 3 insertions(+), 7 deletions(-) diff --git a/official-templates/comfyui/Dockerfile b/official-templates/comfyui/Dockerfile index 5d770870..2b56f8d2 100644 --- a/official-templates/comfyui/Dockerfile +++ b/official-templates/comfyui/Dockerfile @@ -147,9 +147,6 @@ ENV DEBIAN_FRONTEND=noninteractive ENV PYTHONUNBUFFERED=1 ENV IMAGEIO_FFMPEG_EXE=/usr/bin/ffmpeg ENV FILEBROWSER_CONFIG=/workspace/runpod-slim/.filebrowser.json -# Keep bytecode off the /workspace volume: when the volume is full, writing -# .pyc fails silently and every start recompiles pip from scratch. -ENV PYTHONPYCACHEPREFIX=/var/cache/comfyui-pyc # ---- CUDA variant (re-declared for runtime stage) ---- ARG CUDA_VERSION_DASH=12-8 diff --git a/official-templates/comfyui/scripts/start.sh b/official-templates/comfyui/scripts/start.sh index a8e8c7c8..9552785f 100755 --- a/official-templates/comfyui/scripts/start.sh +++ b/official-templates/comfyui/scripts/start.sh @@ -63,7 +63,7 @@ export_env_vars() { # PIP_EXTRA_INDEX_URL travels with PIP_CONSTRAINT: the constraint pins a # +cuXXX build that only the PyTorch index serves, so one without the other # makes every install in an SSH session unresolvable. - printenv | grep -E '^RUNPOD_|^PATH=|^_=|^CUDA|^LD_LIBRARY_PATH|^PYTHONPATH|^PIP_CONSTRAINT=|^PIP_EXTRA_INDEX_URL=|^PYTHONPYCACHEPREFIX=' | while read -r line; do + printenv | grep -E '^RUNPOD_|^PATH=|^_=|^CUDA|^LD_LIBRARY_PATH|^PYTHONPATH|^PIP_CONSTRAINT=|^PIP_EXTRA_INDEX_URL=' | while read -r line; do # Get variable name and value name=$(echo "$line" | cut -d= -f1) value=$(echo "$line" | cut -d= -f2-) @@ -380,9 +380,8 @@ fi create_pip_shim -# Warm up pip before Manager probes it: this compiles pip into -# PYTHONPYCACHEPREFIX on container-local disk, so Manager's probe reads warm -# bytecode. Log wall time — the Dockerfile raises Manager's timeout to 60s. +# Warm up pip before Manager probes it. Log wall time — the Dockerfile raises +# Manager's timeout to 60s. echo "Warming up pip (Manager timeout is 60s)..." time python -m pip --version From 3bef5fea289f2e40d57063a69c91363752a3262c Mon Sep 17 00:00:00 2001 From: mariiachekmasova-runpod Date: Tue, 8 Sep 2026 15:08:43 +0300 Subject: [PATCH 29/33] fix: pip --- official-templates/comfyui/scripts/start.sh | 28 +++++++++++++++++---- 1 file changed, 23 insertions(+), 5 deletions(-) diff --git a/official-templates/comfyui/scripts/start.sh b/official-templates/comfyui/scripts/start.sh index 9552785f..33f143d7 100755 --- a/official-templates/comfyui/scripts/start.sh +++ b/official-templates/comfyui/scripts/start.sh @@ -56,6 +56,7 @@ export_env_vars() { # Clear files : > "$ENV_FILE" : > "$PAM_ENV_FILE" + : > /etc/rp_environment mkdir -p /root/.ssh : > "$SSH_ENV_DIR" @@ -319,7 +320,16 @@ if [ -d "$OLD_VENV_DIR" ] && [ ! -d "$VENV_DIR" ]; then echo " Reinstalling deps for $NODE_COUNT custom nodes" echo " This may take several minutes" echo "=============================================" - mv "$OLD_VENV_DIR" "${OLD_VENV_DIR}.bak" + # Timestamped, and failure must not abort the boot: with a plain `.bak` + # target left over from an earlier migration, `mv` moves the venv *inside* + # it, fails under `set -e`, and the pod restarts in a loop. + VENV_BACKUP="${OLD_VENV_DIR}.bak.$(date +%Y%m%d%H%M%S)" + if mv "$OLD_VENV_DIR" "$VENV_BACKUP"; then + BACKED_UP=1 + else + BACKED_UP=0 + echo "WARNING: could not move $OLD_VENV_DIR aside; continuing with a fresh venv" + fi cd "$COMFYUI_DIR" python3.12 -m venv --system-site-packages --without-pip "$VENV_DIR" # The venv is created at runtime, so there is nothing for shellcheck to follow. @@ -343,8 +353,10 @@ if [ -d "$OLD_VENV_DIR" ] && [ ! -d "$VENV_DIR" ]; then echo "Ensuring ComfyUI requirements are present..." python -m pip install -r "$COMFYUI_DIR/requirements.txt" 2>&1 | grep -E "^(Successfully|ERROR)" || true echo "Migration complete — $INSTALLED user nodes processed (${NODE_COUNT} total, baked nodes skipped)" - echo "Old venv backed up at ${OLD_VENV_DIR}.bak — delete it to free space:" - echo " rm -rf ${OLD_VENV_DIR}.bak" + if [ "$BACKED_UP" = "1" ]; then + echo "Old venv backed up at $VENV_BACKUP — delete it to free space:" + echo " rm -rf $VENV_BACKUP" + fi fi # Setup ComfyUI if needed @@ -380,9 +392,15 @@ fi create_pip_shim +# Interactive sessions are started by sshd, not by this script, and the PATH +# copied into the login files above was captured before the venv existed — +# `python` was then missing entirely and `pip` resolved to the base interpreter. +printf 'if [ -f "%s/bin/activate" ]; then . "%s/bin/activate"; fi\n' \ + "$VENV_DIR" "$VENV_DIR" >> /etc/rp_environment + # Warm up pip before Manager probes it. Log wall time — the Dockerfile raises -# Manager's timeout to 60s. -echo "Warming up pip (Manager timeout is 60s)..." +# Manager's timeout to 30s. +echo "Warming up pip (Manager timeout is 30s)..." time python -m pip --version log_cuda_venv_diagnostics From f0a702430d3bdf6dcedfc4a2df6713e9bd32e96e Mon Sep 17 00:00:00 2001 From: mariiachekmasova-runpod Date: Tue, 8 Sep 2026 19:13:53 +0300 Subject: [PATCH 30/33] fix: JUPYTER_PASSWORD --- official-templates/comfyui/scripts/start.sh | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/official-templates/comfyui/scripts/start.sh b/official-templates/comfyui/scripts/start.sh index 33f143d7..93517fc4 100755 --- a/official-templates/comfyui/scripts/start.sh +++ b/official-templates/comfyui/scripts/start.sh @@ -94,6 +94,12 @@ export_env_vars() { # Start Jupyter Lab server for remote access start_jupyter() { mkdir -p /workspace + + if [ -z "${JUPYTER_PASSWORD:-}" ]; then + JUPYTER_PASSWORD=$(openssl rand -hex 16) + echo "JUPYTER_PASSWORD was not set; generated one for this pod: ${JUPYTER_PASSWORD}" + fi + echo "Starting Jupyter Lab on port 8888..." nohup jupyter lab \ --allow-root \ @@ -104,7 +110,7 @@ start_jupyter() { --FileContentsManager.preferred_dir=/workspace \ --ServerApp.root_dir=/workspace \ --ServerApp.terminado_settings='{"shell_command":["/bin/bash"]}' \ - --IdentityProvider.token="${JUPYTER_PASSWORD:-}" \ + --IdentityProvider.token="${JUPYTER_PASSWORD}" \ --ServerApp.allow_origin=* &> /jupyter.log & echo "Jupyter Lab started" } From 143ca0460478e397ceccfaa36848ce787c66d263 Mon Sep 17 00:00:00 2001 From: mariiachekmasova-runpod Date: Wed, 9 Sep 2026 16:25:15 +0300 Subject: [PATCH 31/33] fix: TEM-120 HF_TOKEN --- official-templates/comfyui/docker-bake.hcl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/official-templates/comfyui/docker-bake.hcl b/official-templates/comfyui/docker-bake.hcl index 5301ddac..200294e0 100644 --- a/official-templates/comfyui/docker-bake.hcl +++ b/official-templates/comfyui/docker-bake.hcl @@ -12,7 +12,7 @@ variable "CIVICOMFY_SHA" { default = "555e984bbcb0" } variable "RUNPODDIRECT_SHA" { - default = "809065c9d2f3" + default = "9e32b1a09577" } variable "FILEBROWSER_VERSION" { default = "v2.59.0" From fd88473a4deecabfd6c3f6f0ed05a0a84101af35 Mon Sep 17 00:00:00 2001 From: mariiachekmasova-runpod Date: Tue, 15 Sep 2026 14:00:32 +0300 Subject: [PATCH 32/33] feat: registry-auth-name --- .github/actions/smoke-test/action.yml | 15 +++++ tests/README.md | 29 ++++++--- tests/runpod_smoke/config.py | 15 ++--- tests/runpod_smoke/pod.py | 17 +++-- tests/test_images.py | 52 ++++++++++++---- tests/unit/test_registry_auth.py | 90 +++++++++++++++++++++++++++ 6 files changed, 179 insertions(+), 39 deletions(-) create mode 100644 tests/unit/test_registry_auth.py diff --git a/.github/actions/smoke-test/action.yml b/.github/actions/smoke-test/action.yml index e9eae929..dd1b7a39 100644 --- a/.github/actions/smoke-test/action.yml +++ b/.github/actions/smoke-test/action.yml @@ -223,6 +223,18 @@ inputs: flips the safer default off. required: false default: "fail" + registry-auth-name: + description: | + Display name of the registry credential in the RunPod account + (the `name` shown by `GET /v2/registries`) to attach to every pod. + + Empty (default) pulls anonymously — fine for our public images, + but the datacenter IP pool shares Docker Hub's rate limit. A name + that the account doesn't have is fatal: the run stops before any + pod is created rather than pulling with credentials nobody asked + for. + required: false + default: "" runs: using: composite @@ -423,6 +435,9 @@ runs: # deadline for slow pulls (mainly multi-GB ROCm base images). CREATE_TIMEOUT: ${{ inputs.create-timeout }} SAVE_COMFYUI_IMAGES: ${{ inputs.save-comfyui-images }} + # Empty → anonymous pulls. Non-empty and unknown to the account + # → test_images.py exits before creating pods. + REGISTRY_AUTH_NAME: ${{ inputs.registry-auth-name }} # test_images.py appends the matrix to $GITHUB_STEP_SUMMARY (always) # and writes this JSON, so results are readable without log access. SMOKE_RESULTS_JSON: ${{ runner.temp }}/smoke-results/results.json diff --git a/tests/README.md b/tests/README.md index 54177fd4..b41e3ac8 100644 --- a/tests/README.md +++ b/tests/README.md @@ -59,13 +59,18 @@ tests/ group/world-readable keys, and the SSH probe will fail every pod with no obvious reason. -4. **(Recommended)** A Docker Hub registry credential on the account. +4. **(Recommended)** A Docker Hub registry credential on the account, + named with `REGISTRY_AUTH_NAME` (or pinned by `REGISTRY_AUTH_ID`). RunPod datacenters share an anonymous Hub IP pool that hits the `toomanyrequests` rate limit fast — without auth, parallel runs in particular produce a wave of "image pull backoff" failures that look - like image bugs but aren't. The script auto-discovers the first entry - from `GET /v2/registries`; pin a specific one with `REGISTRY_AUTH_ID` - or `REGISTRY_AUTH_NAME`. + like image bugs but aren't. + + Nothing is picked implicitly. Unset means anonymous pulls; a name the + account doesn't have stops the run before the first pod. Handing a + pod the wrong login is worse than handing it none — Docker Hub + answers `unauthorized: incorrect username or password` and never + falls back to an anonymous pull, so even a public image fails. ## Quick start @@ -88,7 +93,8 @@ Run it: You should see, in order: 1. `loaded N GPU types from GET /v2/catalog/gpus` — startup catalog query -2. `using registry auth: …` — Docker Hub auth resolved (or a warning) +2. `using registry auth: …` — the named credential resolved (or `no + registry auth requested` when none was asked for) 3. `==================== running 1 job(s) with MAX_PARALLEL=1 ===` 4. `attempt: CPU pod …` → `pod p-xxx created, waiting for RUNNING` 5. `t+Ns endpoint=root@…:NNNN ssh_probe=OK` — pod is up @@ -284,7 +290,7 @@ DWELL_SEC=0 ./test_images.py images.yaml base_cpu # Use a non-default SSH key RUNPOD_SSH_KEY=~/.ssh/my_runpod_key ./test_images.py images.yaml -# Pin to a specific registry auth (avoid auto-pick when you have several) +# Attach a registry credential by name (unset = anonymous pulls) REGISTRY_AUTH_NAME='dockerhub-prod' ./test_images.py images.yaml # …or by id REGISTRY_AUTH_ID='clxxxxxxxxxx' ./test_images.py images.yaml @@ -586,8 +592,8 @@ pytorch: | `CPU_VCPU_COUNT` | `4` | vCPUs requested for CPU pods. Must be a power of two inside the chosen flavor's `vcpu.min..max`. | | `CPU_FLAVOR_ID` | _(empty)_ | Pin a CPU flavor (e.g. `cpu3c`) instead of auto-picking the cheapest fitting one from `GET /v2/catalog/cpus`. | | `SMOKE_RESULTS_JSON` | _(empty)_ | Path to write the machine-readable result report to. Empty = don't write it. The markdown step summary is written regardless. | -| `REGISTRY_AUTH_ID` | _(empty)_ | Explicit Docker Hub registry auth id to pass as `--registry-auth-id`. Overrides auto-discovery. | -| `REGISTRY_AUTH_NAME` | _(empty)_ | Display name to look up via `GET /v2/registries` when `REGISTRY_AUTH_ID` is not set. Falls back to the first entry. | +| `REGISTRY_AUTH_ID` | _(empty)_ | Registry credential id to attach to every pod. Skips the name lookup. | +| `REGISTRY_AUTH_NAME` | _(empty)_ | Display name to look up via `GET /v2/registries`. Empty = anonymous pulls; a name the account doesn't have exits 1 before any pod is created. | | `DWELL_SEC` | `60` | Extra seconds to wait after SSH becomes reachable, then re-probe SSH to catch containers that boot, accept SSH, then crash. Set 0 to skip the re-probe. | | `CREATE_TIMEOUT` | `600` | Max seconds to wait for SSH to become reachable. Raise for ROCm workflows (`create-timeout: "1200"` on the action) — the official `rocm/pytorch:*` base images are 30-50GB and routinely take 8-15 minutes to pull. | | `POLL_INTERVAL` | `10` | Poll cadence for SSH probes. | @@ -683,7 +689,9 @@ wraps everything in this script needs for a clean CI run: fails the generator instead of silently ignoring one. 4. Invokes `python3 tests/test_images.py ` with `MAX_PARALLEL=`, `CLOUD_TYPE=`, - `ON_SKIP=` and `CREATE_TIMEOUT=`. A failed + `ON_SKIP=`, `CREATE_TIMEOUT=` and + `REGISTRY_AUTH_NAME=` (empty by default, i.e. + anonymous pulls). A failed image makes the smoke-test action fail, which prevents a release from being created. @@ -723,7 +731,8 @@ fields. | `no RunPod API key — set RUNPOD_API_KEY…` | key missing from env and `~/.runpod/config.toml` | `export RUNPOD_API_KEY=` | | `RunPod API rejected the key (HTTP 401…)` | key expired or lacks pod-management permission | regenerate at | | `warn: no GPU catalog` | `GET /v2/catalog/gpus` failed — usually a bad/absent key | fix the key; budget and `check_all_gpu` selection are disabled without it | -| `warn: no registry auth configured` | no Docker Hub credential on the account | add one in the RunPod console (paid Hub account strongly recommended for parallel runs) | +| `registry auth '' not found in this RunPod account` | `REGISTRY_AUTH_NAME` doesn't match any credential — typo, or a different account than the one that has it | the error lists the names the account does have; fix the name or add the credential in the RunPod console | +| `unauthorized: incorrect username or password` in the pod's system log | the attached credential's Docker Hub login is stale (common after switching accounts) | replace the token on that credential in the RunPod console, or drop `REGISTRY_AUTH_NAME` to pull our public images anonymously | | every pod SKIPs with an SSH failure | private key not mode `600`, or its public half isn't registered | `chmod 600 `; verify the fingerprint appears in `GET /v2/account/ssh-keys` | | `no ssh endpoint assigned yet` for the whole timeout | the pod never got a machine, so neither `ssh.direct` nor `ssh.proxy` exists | genuine provisioning failure — retry, or check the pod in the console. A missing *direct* port alone no longer causes this: the proxy is used instead | | `cuda_versions is set but none of the N candidate GPUs reports any CUDA version in the SECURE cloud` | CUDA axis on a ROCm/AMD sweep, or every candidate lives in the other cloud tier | drop `cuda_versions` for ROCm — the axis is NVIDIA-only; otherwise rerun with the other `CLOUD_TYPE` | diff --git a/tests/runpod_smoke/config.py b/tests/runpod_smoke/config.py index ad6c0fd0..bedab5b1 100644 --- a/tests/runpod_smoke/config.py +++ b/tests/runpod_smoke/config.py @@ -95,15 +95,16 @@ def _coerce_on_skip(raw: str) -> str: STALL_HINT_AFTER = int(os.environ.get("STALL_HINT_AFTER", "180")) -# Docker Hub authenticated pulls — without this, RunPod datacenters share -# an anonymous IP pool that hits Docker Hub's `toomanyrequests` rate limit -# fast. Either set REGISTRY_AUTH_ID explicitly, or REGISTRY_AUTH_NAME to -# pick by display name, or the script auto-picks the first entry from -# `GET /v2/registries`. +# Docker Hub authenticated pulls. Unset means anonymous pulls: fine for +# public images, but RunPod datacenters share an IP pool that hits Docker +# Hub's `toomanyrequests` rate limit fast. Name the credential with +# REGISTRY_AUTH_NAME (looked up in `GET /v2/registries`) or pass its +# REGISTRY_AUTH_ID directly. A name that doesn't resolve is fatal — +# nothing is ever picked implicitly. # -# REGISTRY_AUTH_ID is reassigned by main() after auto-discovery — access +# REGISTRY_AUTH_ID is reassigned by main() after the name lookup — access # it via `config.REGISTRY_AUTH_ID` (not a bare `from config import`) to -# pick up the post-discovery value. +# pick up the resolved value. REGISTRY_AUTH_ID = os.environ.get("REGISTRY_AUTH_ID", "") REGISTRY_AUTH_NAME = os.environ.get("REGISTRY_AUTH_NAME", "") diff --git a/tests/runpod_smoke/pod.py b/tests/runpod_smoke/pod.py index eb838aa1..3ba34538 100644 --- a/tests/runpod_smoke/pod.py +++ b/tests/runpod_smoke/pod.py @@ -108,19 +108,18 @@ def _signal_handler(signum: int, _frame) -> None: # --------------------------------------------------------------------------- -def discover_registry_auth(prefer_name: str = "") -> Optional[str]: - """Find a registry credential id from `GET /v2/registries`.""" +def list_registries() -> Optional[list[dict]]: + """Registry credentials on the account, or None if the call failed. + + Resolution by name is the caller's job: a credential is never picked + implicitly, because the wrong Docker Hub login turns a public pull + into `unauthorized: incorrect username or password` inside the pod. + """ status, data = api.request_with_retries("GET", "/registries", timeout=30) if not (200 <= status < 300) or not isinstance(data, dict): return None registries = data.get("registries") - if not isinstance(registries, list) or not registries: - return None - if prefer_name: - for item in registries: - if (item.get("name") or "").lower() == prefer_name.lower(): - return item.get("id") - return registries[0].get("id") + return registries if isinstance(registries, list) else None # --------------------------------------------------------------------------- diff --git a/tests/test_images.py b/tests/test_images.py index 54a765db..d917afa2 100755 --- a/tests/test_images.py +++ b/tests/test_images.py @@ -45,7 +45,7 @@ _normalize_cuda_version, parse_manifest, ) -from runpod_smoke.pod import discover_registry_auth +from runpod_smoke.pod import list_registries from runpod_smoke.runner import test_image @@ -81,8 +81,8 @@ def _check_prereqs(manifest_path: Path) -> Optional[int]: """Return None on success, or an exit-code int on failure. Verifies: 1. the manifest file actually exists 2. the REST API v2 accepts our key - Anything else (GPU catalog, registry auth) is best-effort — the script - degrades gracefully if those are missing.""" + The GPU catalog is best-effort — the script degrades gracefully + without it. Registry auth is checked separately, in main().""" if not manifest_path.is_file(): log(f"Images manifest not found: {manifest_path}") return 1 @@ -119,18 +119,42 @@ def _init_gpu_catalog() -> None: ) -def _init_registry_auth() -> None: - if not config.REGISTRY_AUTH_ID: - config.REGISTRY_AUTH_ID = ( - discover_registry_auth(config.REGISTRY_AUTH_NAME) or "" - ) +def _init_registry_auth() -> Optional[int]: + """Resolve the credential to attach to every pod. Exit code on failure. + + Asking for a credential that the account doesn't have is fatal: we + would otherwise fall back to a pull the caller didn't ask for and + only find out from a rate-limit or auth error inside the pod. + """ if config.REGISTRY_AUTH_ID: - log(f"using registry auth: {config.REGISTRY_AUTH_ID}") - else: + log(f"using registry auth: {config.REGISTRY_AUTH_ID} (REGISTRY_AUTH_ID)") + return None + if not config.REGISTRY_AUTH_NAME: log( - "warn: no registry auth configured — Docker Hub pulls will be " - "anonymous and likely hit the toomanyrequests rate limit" + "no registry auth requested — pulls will be anonymous, which " + "works for public images but shares Docker Hub's rate limit" ) + return None + + registries = list_registries() + if registries is None: + print("::error::GET /v2/registries failed — cannot resolve " + f"registry auth '{config.REGISTRY_AUTH_NAME}'") + return 1 + for item in registries: + if (item.get("name") or "").lower() == config.REGISTRY_AUTH_NAME.lower(): + config.REGISTRY_AUTH_ID = item.get("id") or "" + break + if not config.REGISTRY_AUTH_ID: + have = ", ".join(sorted(r.get("name") or "?" for r in registries)) + print(f"::error::registry auth '{config.REGISTRY_AUTH_NAME}' not found " + f"in this RunPod account (has: {have or 'none'})") + return 1 + log( + f"using registry auth: {config.REGISTRY_AUTH_ID} " + f"('{config.REGISTRY_AUTH_NAME}')" + ) + return None # --------------------------------------------------------------------------- @@ -877,7 +901,9 @@ def main() -> int: if rc is not None: return rc - _init_registry_auth() + rc = _init_registry_auth() + if rc is not None: + return rc manifest = parse_manifest(manifest_path) try: diff --git a/tests/unit/test_registry_auth.py b/tests/unit/test_registry_auth.py new file mode 100644 index 00000000..a0da2c2e --- /dev/null +++ b/tests/unit/test_registry_auth.py @@ -0,0 +1,90 @@ +"""A credential must never be attached unless it was asked for by name. + +Auto-picking `registries[0]` made the run depend on whatever the RunPod +account happened to list first. After an account switch that entry held a +stale Docker Hub login, and every pod died in ~12s with `unauthorized: +incorrect username or password` — on a public image that needs no login +at all, because Docker Hub rejects bad credentials instead of falling +back to an anonymous pull. +""" + +import unittest +from unittest import mock + +import test_images as T +from runpod_smoke import config + + +REGISTRIES = [ + {"id": "clstale0000", "name": "dockerhub-old"}, + {"id": "clgood11111", "name": "dockerhub-ci"}, +] + + +class RegistryAuthResolution(unittest.TestCase): + def setUp(self) -> None: + self.addCleanup( + setattr, config, "REGISTRY_AUTH_ID", config.REGISTRY_AUTH_ID + ) + self.addCleanup( + setattr, config, "REGISTRY_AUTH_NAME", config.REGISTRY_AUTH_NAME + ) + config.REGISTRY_AUTH_ID = "" + config.REGISTRY_AUTH_NAME = "" + + def test_no_name_pulls_anonymously_without_listing(self): + with mock.patch.object(T, "list_registries") as listed: + self.assertIsNone(T._init_registry_auth()) + self.assertEqual(config.REGISTRY_AUTH_ID, "") + listed.assert_not_called() + + def test_name_resolves_to_its_own_id(self): + config.REGISTRY_AUTH_NAME = "dockerhub-ci" + with mock.patch.object(T, "list_registries", return_value=REGISTRIES): + self.assertIsNone(T._init_registry_auth()) + self.assertEqual(config.REGISTRY_AUTH_ID, "clgood11111") + + def test_name_match_ignores_case(self): + config.REGISTRY_AUTH_NAME = "DockerHub-CI" + with mock.patch.object(T, "list_registries", return_value=REGISTRIES): + T._init_registry_auth() + self.assertEqual(config.REGISTRY_AUTH_ID, "clgood11111") + + def test_unknown_name_is_fatal_and_attaches_nothing(self): + config.REGISTRY_AUTH_NAME = "dockerhub-typo" + with mock.patch.object(T, "list_registries", return_value=REGISTRIES): + self.assertEqual(T._init_registry_auth(), 1) + self.assertEqual(config.REGISTRY_AUTH_ID, "") + + def test_empty_account_is_fatal_when_a_name_was_asked_for(self): + config.REGISTRY_AUTH_NAME = "dockerhub-ci" + with mock.patch.object(T, "list_registries", return_value=[]): + self.assertEqual(T._init_registry_auth(), 1) + + def test_failed_listing_is_fatal(self): + config.REGISTRY_AUTH_NAME = "dockerhub-ci" + with mock.patch.object(T, "list_registries", return_value=None): + self.assertEqual(T._init_registry_auth(), 1) + + def test_explicit_id_skips_the_lookup(self): + config.REGISTRY_AUTH_ID = "clpinned0000" + with mock.patch.object(T, "list_registries") as listed: + self.assertIsNone(T._init_registry_auth()) + self.assertEqual(config.REGISTRY_AUTH_ID, "clpinned0000") + listed.assert_not_called() + + +class MainStopsBeforeSpendingMoney(unittest.TestCase): + def test_unresolved_name_returns_before_any_pod_is_planned(self): + with mock.patch.object(T, "_parse_args", return_value=("images", None)), \ + mock.patch.object(T, "_check_prereqs", return_value=None), \ + mock.patch.object(T, "_init_registry_auth", return_value=1), \ + mock.patch.object(T, "parse_manifest") as parsed, \ + mock.patch.object(T, "_run_jobs") as ran: + self.assertEqual(T.main(), 1) + parsed.assert_not_called() + ran.assert_not_called() + + +if __name__ == "__main__": + unittest.main() From dcca866e3ca8ed978559e34832f80bd6f1496ddf Mon Sep 17 00:00:00 2001 From: mariiachekmasova-runpod Date: Tue, 15 Sep 2026 15:03:23 +0300 Subject: [PATCH 33/33] feat: What's Changed from Squash description --- .github/workflows/manual-release.yml | 12 ++++++++++++ .github/workflows/release.yml | 13 +++++++++++++ 2 files changed, 25 insertions(+) diff --git a/.github/workflows/manual-release.yml b/.github/workflows/manual-release.yml index 60999a6d..a0052956 100644 --- a/.github/workflows/manual-release.yml +++ b/.github/workflows/manual-release.yml @@ -204,12 +204,24 @@ jobs: exit 1 fi + # Same as release.yml: the squash commit we are tagging carries the body, + # and GitHub pre-pends it to its own generated notes. + - name: Release body from the squash commit + id: notes + run: | + set -euo pipefail + BODY="${RUNNER_TEMP}/release-body.md" + git log -1 --format=%b HEAD \ + | grep -viE '^(co-authored-by|signed-off-by):' > "${BODY}" || true + echo "path=${BODY}" >> "$GITHUB_OUTPUT" + - name: Create tag and GitHub Release uses: softprops/action-gh-release@3d0d9888cb7fd7b750713d6e236d1fcb99157228 with: tag_name: ${{ inputs.tag }} name: ${{ inputs.tag }} target_commitish: ${{ steps.source.outputs.head_sha }} + body_path: ${{ steps.notes.outputs.path }} generate_release_notes: true env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 090c08d8..c8d4a5f9 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -170,12 +170,25 @@ jobs: fetch-depth: 0 fetch-tags: true + # A releasable push to main is one squash commit, so its extended + # description is the release body. GitHub pre-pends it to the notes it + # generates, keeping the PR line and the Full Changelog link. + - name: Release body from the squash commit + id: notes + run: | + set -euo pipefail + BODY="${RUNNER_TEMP}/release-body.md" + git log -1 --format=%b HEAD \ + | grep -viE '^(co-authored-by|signed-off-by):' > "${BODY}" || true + echo "path=${BODY}" >> "$GITHUB_OUTPUT" + - name: Create tag and GitHub Release uses: softprops/action-gh-release@3d0d9888cb7fd7b750713d6e236d1fcb99157228 with: tag_name: ${{ needs.version.outputs.tag }} name: ${{ needs.version.outputs.tag }} target_commitish: ${{ github.sha }} + body_path: ${{ steps.notes.outputs.path }} generate_release_notes: true env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}