fix: TEM-89 ComfyUI-Manager pip failures (probe timeout, venv pip, torch index)

.github/actions/smoke-test/action.yml

@@ -223,6 +223,18 @@ inputs:
       flips the safer default off.
     required: false
     default: "fail"
+  registry-auth-name:
+    description: |
+      Display name of the registry credential in the RunPod account
+      (the `name` shown by `GET /v2/registries`) to attach to every pod.
+
+      Empty (default) pulls anonymously — fine for our public images,
+      but the datacenter IP pool shares Docker Hub's rate limit. A name
+      that the account doesn't have is fatal: the run stops before any
+      pod is created rather than pulling with credentials nobody asked
+      for.
+    required: false
+    default: ""
 
 runs:
   using: composite
@@ -423,6 +435,9 @@ runs:
         # deadline for slow pulls (mainly multi-GB ROCm base images).
         CREATE_TIMEOUT: ${{ inputs.create-timeout }}
         SAVE_COMFYUI_IMAGES: ${{ inputs.save-comfyui-images }}
+        # Empty → anonymous pulls. Non-empty and unknown to the account
+        # → test_images.py exits before creating pods.
+        REGISTRY_AUTH_NAME: ${{ inputs.registry-auth-name }}
         # test_images.py appends the matrix to $GITHUB_STEP_SUMMARY (always)
         # and writes this JSON, so results are readable without log access.
         SMOKE_RESULTS_JSON: ${{ runner.temp }}/smoke-results/results.json

.github/workflows/manual-release.yml

@@ -204,12 +204,24 @@ jobs:
             exit 1
           fi
 
+      # Same as release.yml: the squash commit we are tagging carries the body,
+      # and GitHub pre-pends it to its own generated notes.
+      - name: Release body from the squash commit
+        id: notes
+        run: |
+          set -euo pipefail
+          BODY="${RUNNER_TEMP}/release-body.md"
+          git log -1 --format=%b HEAD \
+            | grep -viE '^(co-authored-by|signed-off-by):' > "${BODY}" || true
+          echo "path=${BODY}" >> "$GITHUB_OUTPUT"
+
       - name: Create tag and GitHub Release
         uses: softprops/action-gh-release@3d0d9888cb7fd7b750713d6e236d1fcb99157228
         with:
           tag_name: ${{ inputs.tag }}
           name: ${{ inputs.tag }}
           target_commitish: ${{ steps.source.outputs.head_sha }}
+          body_path: ${{ steps.notes.outputs.path }}
           generate_release_notes: true
         env:
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}

.github/workflows/release.yml

@@ -170,12 +170,25 @@ jobs:
           fetch-depth: 0
           fetch-tags: true
 
+      # A releasable push to main is one squash commit, so its extended
+      # description is the release body. GitHub pre-pends it to the notes it
+      # generates, keeping the PR line and the Full Changelog link.
+      - name: Release body from the squash commit
+        id: notes
+        run: |
+          set -euo pipefail
+          BODY="${RUNNER_TEMP}/release-body.md"
+          git log -1 --format=%b HEAD \
+            | grep -viE '^(co-authored-by|signed-off-by):' > "${BODY}" || true
+          echo "path=${BODY}" >> "$GITHUB_OUTPUT"
+
       - name: Create tag and GitHub Release
         uses: softprops/action-gh-release@3d0d9888cb7fd7b750713d6e236d1fcb99157228
         with:
           tag_name: ${{ needs.version.outputs.tag }}
           name: ${{ needs.version.outputs.tag }}
           target_commitish: ${{ github.sha }}
+          body_path: ${{ steps.notes.outputs.path }}
           generate_release_notes: true
         env:
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}

official-templates/comfyui/Dockerfile

@@ -64,6 +64,13 @@ RUN curl -fSL "https://github.com/ltdrdata/ComfyUI-Manager/archive/${MANAGER_SHA
     curl -fSL "https://github.com/MadiatorLabs/ComfyUI-RunpodDirect/archive/${RUNPODDIRECT_SHA}.tar.gz" -o runpoddirect.tar.gz && \
     mkdir -p ComfyUI-RunpodDirect && tar xzf runpoddirect.tar.gz --strip-components=1 -C ComfyUI-RunpodDirect && rm runpoddirect.tar.gz
 
+# Manager's get_pip_cmd() probes `python -m pip --version` with a 5s timeout and
+# treats a timeout as "pip is missing" — on a slow network volume a working pip
+# reads as absent. Patched before the git commit below so the tree stays clean.
+# The count assertion fails the build if the upstream pattern moves.
+RUN test "$(grep -c 'timeout=5)' ComfyUI-Manager/glob/manager_util.py)" = 2 && \
+    sed -i 's/timeout=5)/timeout=30)/g' ComfyUI-Manager/glob/manager_util.py
+
 # Init git repos with upstream remotes so ComfyUI-Manager can detect versions
 # and users can update via Manager at their own risk
 WORKDIR /tmp/build/ComfyUI
@@ -143,6 +150,7 @@ ENV FILEBROWSER_CONFIG=/workspace/runpod-slim/.filebrowser.json
 
 # ---- CUDA variant (re-declared for runtime stage) ----
 ARG CUDA_VERSION_DASH=12-8
+ARG TORCH_INDEX_SUFFIX=cu128
 ARG TORCH_VERSION
 ARG TORCHVISION_VERSION
 ARG TORCHAUDIO_VERSION
@@ -152,6 +160,9 @@ ARG FILEBROWSER_VERSION
 ARG FILEBROWSER_SHA256
 
 # Keep runtime pip installs aligned with the baked CUDA-specific PyTorch stack.
+# The pins carry a local version (+cu128), which PyPI does not serve — without
+# this index every runtime install that touches torch fails to resolve.
+ENV PIP_EXTRA_INDEX_URL="https://download.pytorch.org/whl/${TORCH_INDEX_SUFFIX}"
 RUN printf "torch==%s\ntorchvision==%s\ntorchaudio==%s\n" \
     "$TORCH_VERSION" "$TORCHVISION_VERSION" "$TORCHAUDIO_VERSION" \
     > /opt/comfyui-runtime-constraints.txt

official-templates/comfyui/docker-bake.hcl

@@ -12,7 +12,7 @@ variable "CIVICOMFY_SHA" {
   default = "555e984bbcb0"
 }
 variable "RUNPODDIRECT_SHA" {
-  default = "809065c9d2f3"
+  default = "9e32b1a09577"
 }
 variable "FILEBROWSER_VERSION" {
   default = "v2.59.0"

official-templates/comfyui/scripts/start.sh

@@ -56,11 +56,15 @@ export_env_vars() {
     # Clear files
     : > "$ENV_FILE"
     : > "$PAM_ENV_FILE"
+    : > /etc/rp_environment
     mkdir -p /root/.ssh
     : > "$SSH_ENV_DIR"
 
     # Export to multiple locations for maximum compatibility
-    printenv | grep -E '^RUNPOD_|^PATH=|^_=|^CUDA|^LD_LIBRARY_PATH|^PYTHONPATH|^PIP_CONSTRAINT=' | while read -r line; do
+    # PIP_EXTRA_INDEX_URL travels with PIP_CONSTRAINT: the constraint pins a
+    # +cuXXX build that only the PyTorch index serves, so one without the other
+    # makes every install in an SSH session unresolvable.
+    printenv | grep -E '^RUNPOD_|^PATH=|^_=|^CUDA|^LD_LIBRARY_PATH|^PYTHONPATH|^PIP_CONSTRAINT=|^PIP_EXTRA_INDEX_URL=' | while read -r line; do
         # Get variable name and value
         name=$(echo "$line" | cut -d= -f1)
         value=$(echo "$line" | cut -d= -f2-)
@@ -90,6 +94,12 @@ export_env_vars() {
 # Start Jupyter Lab server for remote access
 start_jupyter() {
     mkdir -p /workspace
+
+    if [ -z "${JUPYTER_PASSWORD:-}" ]; then
+        JUPYTER_PASSWORD=$(openssl rand -hex 16)
+        echo "JUPYTER_PASSWORD was not set; generated one for this pod: ${JUPYTER_PASSWORD}"
+    fi
+
     echo "Starting Jupyter Lab on port 8888..."
     nohup jupyter lab \
         --allow-root \
@@ -100,7 +110,7 @@ start_jupyter() {
         --FileContentsManager.preferred_dir=/workspace \
         --ServerApp.root_dir=/workspace \
         --ServerApp.terminado_settings='{"shell_command":["/bin/bash"]}' \
-        --IdentityProvider.token="${JUPYTER_PASSWORD:-}" \
+        --IdentityProvider.token="${JUPYTER_PASSWORD}" \
         --ServerApp.allow_origin=* &> /jupyter.log &
     echo "Jupyter Lab started"
 }
@@ -170,6 +180,21 @@ upgrade_comfyui_if_needed() {
     echo "ComfyUI workspace upgraded successfully"
 }
 
+# The venv has no pip of its own, so a bare `pip` would resolve to
+# /usr/local/bin/pip and install against the base interpreter. Custom-node
+# install scripts do call it that way. Existing venvs keep their real pip.
+create_pip_shim() {
+    if [ -e "$VENV_DIR/bin/pip" ]; then
+        return
+    fi
+    if printf '#!/bin/sh\nexec "%s/bin/python" -m pip "$@"\n' "$VENV_DIR" \
+        > "$VENV_DIR/bin/pip" 2>/dev/null; then
+        chmod +x "$VENV_DIR/bin/pip"
+    else
+        echo "WARNING: could not write $VENV_DIR/bin/pip — is the volume full?"
+    fi
+}
+
 log_cuda_venv_diagnostics() {
     local expected_build local_packages status
     expected_build=$(sed -n 's/^torch==.*+\(cu[0-9][0-9]*\).*$/\1/p' \
@@ -301,13 +326,21 @@ if [ -d "$OLD_VENV_DIR" ] && [ ! -d "$VENV_DIR" ]; then
     echo "  Reinstalling deps for $NODE_COUNT custom nodes"
     echo "  This may take several minutes"
     echo "============================================="
-    mv "$OLD_VENV_DIR" "${OLD_VENV_DIR}.bak"
+    # Timestamped, and failure must not abort the boot: with a plain `.bak`
+    # target left over from an earlier migration, `mv` moves the venv *inside*
+    # it, fails under `set -e`, and the pod restarts in a loop.
+    VENV_BACKUP="${OLD_VENV_DIR}.bak.$(date +%Y%m%d%H%M%S)"
+    if mv "$OLD_VENV_DIR" "$VENV_BACKUP"; then
+        BACKED_UP=1
+    else
+        BACKED_UP=0
+        echo "WARNING: could not move $OLD_VENV_DIR aside; continuing with a fresh venv"
+    fi
     cd "$COMFYUI_DIR"
-    python3.12 -m venv --system-site-packages "$VENV_DIR"
+    python3.12 -m venv --system-site-packages --without-pip "$VENV_DIR"
     # The venv is created at runtime, so there is nothing for shellcheck to follow.
     # shellcheck source=/dev/null
     source "$VENV_DIR/bin/activate"
-    python -m ensurepip
     # Skip nodes baked into the image — their deps are in system site-packages
     CURRENT=0
     INSTALLED=0
@@ -319,15 +352,17 @@ if [ -d "$OLD_VENV_DIR" ] && [ ! -d "$VENV_DIR" ]; then
             esac
             CURRENT=$((CURRENT + 1))
             echo "[$CURRENT] $NODE_NAME"
-            pip install -r "$req" 2>&1 | grep -E "^(Successfully|ERROR)" || true
+            python -m pip install -r "$req" 2>&1 | grep -E "^(Successfully|ERROR)" || true
             INSTALLED=$((INSTALLED + 1))
         fi
     done
     echo "Ensuring ComfyUI requirements are present..."
-    pip install -r "$COMFYUI_DIR/requirements.txt" 2>&1 | grep -E "^(Successfully|ERROR)" || true
+    python -m pip install -r "$COMFYUI_DIR/requirements.txt" 2>&1 | grep -E "^(Successfully|ERROR)" || true
     echo "Migration complete — $INSTALLED user nodes processed (${NODE_COUNT} total, baked nodes skipped)"
-    echo "Old venv backed up at ${OLD_VENV_DIR}.bak — delete it to free space:"
-    echo "  rm -rf ${OLD_VENV_DIR}.bak"
+    if [ "$BACKED_UP" = "1" ]; then
+        echo "Old venv backed up at $VENV_BACKUP — delete it to free space:"
+        echo "  rm -rf $VENV_BACKUP"
+    fi
 fi
 
 # Setup ComfyUI if needed
@@ -343,13 +378,14 @@ if [ ! -d "$COMFYUI_DIR" ] || [ ! -d "$VENV_DIR" ]; then
     # Create venv with access to system packages (torch, numpy, etc. pre-installed in image)
     if [ ! -d "$VENV_DIR" ]; then
         cd "$COMFYUI_DIR"
-        python3.12 -m venv --system-site-packages "$VENV_DIR"
+        # --without-pip: pip stays in the image (local disk, bytecode compiled
+        # at build) instead of on the network volume, where importing it can
+        # exceed ComfyUI-Manager's probe timeout. --system-site-packages keeps
+        # it importable, and installs still land in this venv via sys.prefix.
+        python3.12 -m venv --system-site-packages --without-pip "$VENV_DIR"
         # shellcheck source=/dev/null
         source "$VENV_DIR/bin/activate"
 
-        # Ensure pip is available in the venv (needed for ComfyUI-Manager)
-        python -m ensurepip
-
         echo "Base packages (torch, numpy, etc.) available from system site-packages"
         echo "ComfyUI ready — all dependencies pre-installed in image"
     fi
@@ -360,9 +396,17 @@ else
     echo "Using existing ComfyUI installation"
 fi
 
-# Warm up pip so ComfyUI-Manager's 5s timeout check doesn't fail on cold start.
-# Log wall time — Manager fails if `python -m pip --version` takes >5s.
-echo "Warming up pip (Manager timeout is 5s)..."
+create_pip_shim
+
+# Interactive sessions are started by sshd, not by this script, and the PATH
+# copied into the login files above was captured before the venv existed —
+# `python` was then missing entirely and `pip` resolved to the base interpreter.
+printf 'if [ -f "%s/bin/activate" ]; then . "%s/bin/activate"; fi\n' \
+    "$VENV_DIR" "$VENV_DIR" >> /etc/rp_environment
+
+# Warm up pip before Manager probes it. Log wall time — the Dockerfile raises
+# Manager's timeout to 30s.
+echo "Warming up pip (Manager timeout is 30s)..."
 time python -m pip --version
 
 log_cuda_venv_diagnostics

tests/README.md

@@ -59,13 +59,18 @@ tests/
    group/world-readable keys, and the SSH probe will fail every pod
    with no obvious reason.
 
-4. **(Recommended)** A Docker Hub registry credential on the account.
+4. **(Recommended)** A Docker Hub registry credential on the account,
+   named with `REGISTRY_AUTH_NAME` (or pinned by `REGISTRY_AUTH_ID`).
    RunPod datacenters share an anonymous Hub IP pool that hits the
    `toomanyrequests` rate limit fast — without auth, parallel runs in
    particular produce a wave of "image pull backoff" failures that look
-   like image bugs but aren't. The script auto-discovers the first entry
-   from `GET /v2/registries`; pin a specific one with `REGISTRY_AUTH_ID`
-   or `REGISTRY_AUTH_NAME`.
+   like image bugs but aren't.
+
+   Nothing is picked implicitly. Unset means anonymous pulls; a name the
+   account doesn't have stops the run before the first pod. Handing a
+   pod the wrong login is worse than handing it none — Docker Hub
+   answers `unauthorized: incorrect username or password` and never
+   falls back to an anonymous pull, so even a public image fails.
 
 
 ## Quick start
@@ -88,7 +93,8 @@ Run it:
 You should see, in order:
 
 1. `loaded N GPU types from GET /v2/catalog/gpus` — startup catalog query
-2. `using registry auth: …` — Docker Hub auth resolved (or a warning)
+2. `using registry auth: …` — the named credential resolved (or `no
+   registry auth requested` when none was asked for)
 3. `==================== running 1 job(s) with MAX_PARALLEL=1 ===`
 4. `attempt: CPU pod …` → `pod p-xxx created, waiting for RUNNING`
 5. `t+Ns endpoint=root@…:NNNN ssh_probe=OK` — pod is up
@@ -284,7 +290,7 @@ DWELL_SEC=0 ./test_images.py images.yaml base_cpu
 # Use a non-default SSH key
 RUNPOD_SSH_KEY=~/.ssh/my_runpod_key ./test_images.py images.yaml
 
-# Pin to a specific registry auth (avoid auto-pick when you have several)
+# Attach a registry credential by name (unset = anonymous pulls)
 REGISTRY_AUTH_NAME='dockerhub-prod' ./test_images.py images.yaml
 # …or by id
 REGISTRY_AUTH_ID='clxxxxxxxxxx' ./test_images.py images.yaml
@@ -586,8 +592,8 @@ pytorch:
 | `CPU_VCPU_COUNT` | `4` | vCPUs requested for CPU pods. Must be a power of two inside the chosen flavor's `vcpu.min..max`. |
 | `CPU_FLAVOR_ID` | _(empty)_ | Pin a CPU flavor (e.g. `cpu3c`) instead of auto-picking the cheapest fitting one from `GET /v2/catalog/cpus`. |
 | `SMOKE_RESULTS_JSON` | _(empty)_ | Path to write the machine-readable result report to. Empty = don't write it. The markdown step summary is written regardless. |
-| `REGISTRY_AUTH_ID` | _(empty)_ | Explicit Docker Hub registry auth id to pass as `--registry-auth-id`. Overrides auto-discovery. |
-| `REGISTRY_AUTH_NAME` | _(empty)_ | Display name to look up via `GET /v2/registries` when `REGISTRY_AUTH_ID` is not set. Falls back to the first entry. |
+| `REGISTRY_AUTH_ID` | _(empty)_ | Registry credential id to attach to every pod. Skips the name lookup. |
+| `REGISTRY_AUTH_NAME` | _(empty)_ | Display name to look up via `GET /v2/registries`. Empty = anonymous pulls; a name the account doesn't have exits 1 before any pod is created. |
 | `DWELL_SEC` | `60` | Extra seconds to wait after SSH becomes reachable, then re-probe SSH to catch containers that boot, accept SSH, then crash. Set 0 to skip the re-probe. |
 | `CREATE_TIMEOUT` | `600` | Max seconds to wait for SSH to become reachable. Raise for ROCm workflows (`create-timeout: "1200"` on the action) — the official `rocm/pytorch:*` base images are 30-50GB and routinely take 8-15 minutes to pull. |
 | `POLL_INTERVAL` | `10` | Poll cadence for SSH probes. |
@@ -683,7 +689,9 @@ wraps everything in this script needs for a clean CI run:
    fails the generator instead of silently ignoring one.
 4. Invokes `python3 tests/test_images.py <generated-manifest>` with
    `MAX_PARALLEL=<max-parallel>`, `CLOUD_TYPE=<cloud-type>`,
-   `ON_SKIP=<on-skip>` and `CREATE_TIMEOUT=<create-timeout>`. A failed
+   `ON_SKIP=<on-skip>`, `CREATE_TIMEOUT=<create-timeout>` and
+   `REGISTRY_AUTH_NAME=<registry-auth-name>` (empty by default, i.e.
+   anonymous pulls). A failed
    image makes the smoke-test action fail, which prevents a release from
    being created.
 
@@ -723,7 +731,8 @@ fields.
 | `no RunPod API key — set RUNPOD_API_KEY…` | key missing from env and `~/.runpod/config.toml` | `export RUNPOD_API_KEY=<KEY>` |
 | `RunPod API rejected the key (HTTP 401…)` | key expired or lacks pod-management permission | regenerate at <https://www.runpod.io/console/user/settings> |
 | `warn: no GPU catalog` | `GET /v2/catalog/gpus` failed — usually a bad/absent key | fix the key; budget and `check_all_gpu` selection are disabled without it |
-| `warn: no registry auth configured` | no Docker Hub credential on the account | add one in the RunPod console (paid Hub account strongly recommended for parallel runs) |
+| `registry auth '<name>' not found in this RunPod account` | `REGISTRY_AUTH_NAME` doesn't match any credential — typo, or a different account than the one that has it | the error lists the names the account does have; fix the name or add the credential in the RunPod console |
+| `unauthorized: incorrect username or password` in the pod's system log | the attached credential's Docker Hub login is stale (common after switching accounts) | replace the token on that credential in the RunPod console, or drop `REGISTRY_AUTH_NAME` to pull our public images anonymously |
 | every pod SKIPs with an SSH failure | private key not mode `600`, or its public half isn't registered | `chmod 600 <key>`; verify the fingerprint appears in `GET /v2/account/ssh-keys` |
 | `no ssh endpoint assigned yet` for the whole timeout | the pod never got a machine, so neither `ssh.direct` nor `ssh.proxy` exists | genuine provisioning failure — retry, or check the pod in the console. A missing *direct* port alone no longer causes this: the proxy is used instead |
 | `cuda_versions is set but none of the N candidate GPUs reports any CUDA version in the SECURE cloud` | CUDA axis on a ROCm/AMD sweep, or every candidate lives in the other cloud tier | drop `cuda_versions` for ROCm — the axis is NVIDIA-only; otherwise rerun with the other `CLOUD_TYPE` |

tests/runpod_smoke/config.py

@@ -95,15 +95,16 @@ def _coerce_on_skip(raw: str) -> str:
 STALL_HINT_AFTER = int(os.environ.get("STALL_HINT_AFTER", "180"))
 
 
-# Docker Hub authenticated pulls — without this, RunPod datacenters share
-# an anonymous IP pool that hits Docker Hub's `toomanyrequests` rate limit
-# fast. Either set REGISTRY_AUTH_ID explicitly, or REGISTRY_AUTH_NAME to
-# pick by display name, or the script auto-picks the first entry from
-# `GET /v2/registries`.
+# Docker Hub authenticated pulls. Unset means anonymous pulls: fine for
+# public images, but RunPod datacenters share an IP pool that hits Docker
+# Hub's `toomanyrequests` rate limit fast. Name the credential with
+# REGISTRY_AUTH_NAME (looked up in `GET /v2/registries`) or pass its
+# REGISTRY_AUTH_ID directly. A name that doesn't resolve is fatal —
+# nothing is ever picked implicitly.
 #
-# REGISTRY_AUTH_ID is reassigned by main() after auto-discovery — access
+# REGISTRY_AUTH_ID is reassigned by main() after the name lookup — access
 # it via `config.REGISTRY_AUTH_ID` (not a bare `from config import`) to
-# pick up the post-discovery value.
+# pick up the resolved value.
 REGISTRY_AUTH_ID = os.environ.get("REGISTRY_AUTH_ID", "")
 REGISTRY_AUTH_NAME = os.environ.get("REGISTRY_AUTH_NAME", "")