Restore DEV disk space with guarded API image retention

.github/workflows/dev-image-retention.yml

@@ -0,0 +1,59 @@
+name: DEV API image retention
+
+on:
+  push:
+    branches: [dev]
+    paths: [.github/workflows/dev-image-retention.yml, scripts/dev_image_retention.py]
+  workflow_dispatch:
+    inputs:
+      mode:
+        type: choice
+        options: [plan, apply]
+        default: plan
+        required: true
+      plan_sha256:
+        description: Exact digest printed by the read-only plan
+        type: string
+        required: false
+
+permissions:
+  contents: read
+
+concurrency:
+  group: api-deploy-dev
+  cancel-in-progress: false
+
+jobs:
+  retention:
+    if: github.ref == 'refs/heads/dev'
+    environment: dev
+    runs-on: ubuntu-latest
+    timeout-minutes: 10
+    steps:
+      - uses: actions/checkout@v4
+      - uses: actions/setup-node@v4
+        with:
+          node-version: '24'
+      - name: Install verified SSH transport
+        run: npm install --prefix "$RUNNER_TEMP/dev-migration-ssh" --ignore-scripts --no-package-lock --no-audit --no-fund ssh2@1.16.0
+      - name: Plan or apply exact unused DEV image removals
+        env:
+          NODE_PATH: ${{ runner.temp }}/dev-migration-ssh/node_modules
+          DEV_SERVER_HOST: ${{ secrets.DEV_SERVER_HOST }}
+          DEV_SERVER_USER: ${{ secrets.DEV_SERVER_USER }}
+          DEV_SERVER_PASSWORD: ${{ secrets.DEV_SERVER_PASSWORD }}
+          DEV_SSH_HOST_KEY_SHA256: ${{ vars.DEV_SSH_HOST_KEY_SHA256 }}
+          RETENTION_MODE: ${{ github.event_name == 'push' && 'plan' || inputs.mode }}
+          PLAN_SHA256: ${{ inputs.plan_sha256 }}
+        run: |
+          node <<'NODE'
+          const fs = require('node:fs');
+          const mode = process.env.RETENTION_MODE;
+          const digest = process.env.PLAN_SHA256 || 'none';
+          if (!['plan', 'apply'].includes(mode)) throw new Error('Invalid mode');
+          if (!/^(none|[0-9a-f]{64})$/.test(digest)) throw new Error('Invalid plan digest');
+          if (mode === 'apply' && !/^[0-9a-f]{64}$/.test(digest)) throw new Error('Inspect a plan first');
+          const source = fs.readFileSync('scripts/dev_image_retention.py', 'utf8');
+          process.env.DEV_DEPLOY_SCRIPT = `set -euo pipefail\npython3 - ${mode} ${digest} <<'DEV_IMAGE_RETENTION_PY'\n${source}\nDEV_IMAGE_RETENTION_PY\n`;
+          require('./scripts/dev_ci_ssh.cjs');
+          NODE

scripts/dev_image_retention.py

@@ -0,0 +1,221 @@
+"""Plan or remove old, reproducible DEV API images; never prune Docker globally."""
+
+import hashlib
+import json
+import os
+import re
+import subprocess
+import sys
+from datetime import datetime, timedelta, timezone
+
+TAG = re.compile(r"procollab-dev-api:([0-9a-f]{40})(?:-[0-9a-f]{12})?$")
+REPO = "/root/api"
+
+
+def command(*args):
+    return subprocess.check_output(args, text=True).strip()
+
+
+def require(condition, message):
+    if not condition:
+        raise RuntimeError(message)
+
+
+def output(message):
+    sys.stdout.write(message + "\n")
+    sys.stdout.flush()
+
+
+def digest(value):
+    return hashlib.sha256(json.dumps(value, sort_keys=True).encode()).hexdigest()
+
+
+def inventory():
+    result = {}
+    for image_id in set(command("docker", "image", "ls", "-aq", "--no-trunc").split()):
+        value = json.loads(
+            command(
+                "docker",
+                "image",
+                "inspect",
+                "--format",
+                '{"id":{{json .Id}},"tags":{{json .RepoTags}},"created":{{json .Created}}}',
+                image_id,
+            )
+        )
+        value["tags"] = sorted(value["tags"] or [])
+        result[image_id] = value
+    return result
+
+
+def containers():
+    result = []
+    for container_id in command("docker", "ps", "-aq", "--no-trunc").split():
+        result.append(
+            json.loads(
+                command(
+                    "docker",
+                    "inspect",
+                    "--format",
+                    '{"id":{{json .Id}},"image":{{json .Image}},"state":{{json .State.Status}},'
+                    '"started":{{json .State.StartedAt}},"restarts":{{json .RestartCount}}}',
+                    container_id,
+                )
+            )
+        )
+    return sorted(result, key=lambda value: value["id"])
+
+
+def volumes():
+    return sorted(
+        command(
+            "docker", "volume", "ls", "--format", "{{.Driver}} {{.Name}}"
+        ).splitlines()
+    )
+
+
+def created_at(image):
+    value = re.sub(r"(\.\d{6})\d+", r"\1", image["created"])
+    return datetime.fromisoformat(value.replace("Z", "+00:00"))
+
+
+def plan(images, container_rows, volume_rows, now, reachable):
+    used = {row["image"] for row in container_rows}
+    reproducible = []
+    for image in images.values():
+        tags = image["tags"]
+        matches = [TAG.fullmatch(tag) for tag in tags]
+        if tags and all(matches) and all(reachable(match[1]) for match in matches):
+            reproducible.append(image)
+    newest = {
+        image["id"]
+        for image in sorted(
+            reproducible, key=lambda row: (created_at(row), row["id"]), reverse=True
+        )[:3]
+    }
+    candidates = sorted(
+        (
+            image
+            for image in reproducible
+            if image["id"] not in used | newest
+            and created_at(image) < now - timedelta(hours=48)
+        ),
+        key=lambda image: image["id"],
+    )
+    proof = {
+        "images": images,
+        "containers": container_rows,
+        "volumes": volume_rows,
+        "candidates": candidates,
+    }
+    return {
+        "sha256": digest(proof),
+        "candidates": candidates,
+        "protected_count": len(images) - len(candidates),
+    }
+
+
+def apply(plan_value, expected, images_before, containers_before, volumes_before):
+    if plan_value["sha256"] != expected:
+        raise RuntimeError("Image inventory changed; inspect a new plan before cleanup")
+    remaining = dict(images_before)
+    for image in plan_value["candidates"]:
+        if (
+            containers() != containers_before
+            or volumes() != volumes_before
+            or inventory() != remaining
+        ):
+            raise RuntimeError("Docker state changed; no further images will be removed")
+        for tag in image["tags"]:
+            # Non-force removal preserves images referenced by any container.
+            # --no-prune also prevents removal of untagged parent images.
+            subprocess.run(["docker", "image", "rm", "--no-prune", tag], check=True)
+        del remaining[image["id"]]
+    if (
+        inventory() != remaining
+        or containers() != containers_before
+        or volumes() != volumes_before
+    ):
+        raise RuntimeError("Docker state differs from the reviewed removal plan")
+    output(
+        "PROTECTED_IMAGES_UNCHANGED=true\nCONTAINERS_UNCHANGED=true\nVOLUMES_UNCHANGED=true"
+    )
+
+
+def main():
+    mode, expected = sys.argv[1:]
+    require(mode in ("plan", "apply"), "Invalid operation")
+    require(
+        mode == "plan" or re.fullmatch(r"[0-9a-f]{64}", expected), "Inspect a plan first"
+    )
+    require(
+        os.geteuid() == 0 and os.path.realpath(REPO) == REPO,
+        "Unexpected host or repository",
+    )
+    command(
+        "git",
+        "-C",
+        REPO,
+        "merge-base",
+        "--is-ancestor",
+        "beef71645567128398d0afa5a2eb80dc6bc8dac7",
+        "HEAD",
+    )
+    command(
+        "grep",
+        "-Eq",
+        r"server_name[[:space:]]+dev\.procollab\.ru;",
+        "/etc/nginx/sites-enabled/dev.procollab.ru",
+    )
+    runtime = []
+    for service in ("web", "celerys"):
+        container_id = command(
+            "docker",
+            "ps",
+            "-q",
+            "--no-trunc",
+            "--filter",
+            f"label=com.docker.compose.project.working_dir={REPO}",
+            "--filter",
+            f"label=com.docker.compose.service={service}",
+        )
+        require(
+            re.fullmatch(r"[0-9a-f]{64}", container_id),
+            "Expected one running DEV service",
+        )
+        reference = command("docker", "inspect", "-f", "{{.Config.Image}}", container_id)
+        require(TAG.fullmatch(reference), "Unexpected DEV image reference")
+        runtime.append(command("docker", "inspect", "-f", "{{.Image}}", container_id))
+    require(len(set(runtime)) == 1, "DEV web and worker image mismatch")
+    images_before, containers_before, volumes_before = (
+        inventory(),
+        containers(),
+        volumes(),
+    )
+
+    def reachable(sha):
+        return (
+            subprocess.run(
+                ["git", "-C", REPO, "merge-base", "--is-ancestor", sha, "HEAD"],
+                stdout=subprocess.DEVNULL,
+                stderr=subprocess.DEVNULL,
+            ).returncode
+            == 0
+        )
+
+    value = plan(
+        images_before,
+        containers_before,
+        volumes_before,
+        datetime.now(timezone.utc),
+        reachable,
+    )
+    output("DEV_IMAGE_RETENTION_PLAN=" + json.dumps(value, sort_keys=True))
+    output(command("df", "-h", "/"))
+    if mode == "apply":
+        apply(value, expected, images_before, containers_before, volumes_before)
+        output(command("df", "-h", "/"))
+
+
+if __name__ == "__main__":
+    main()

tests/test_dev_image_retention.py

@@ -0,0 +1,112 @@
+import unittest
+from datetime import datetime, timedelta, timezone
+from unittest.mock import patch
+
+from scripts import dev_image_retention as service
+
+
+class DevImageRetentionTests(unittest.TestCase):
+    now = datetime(2026, 10, 25, tzinfo=timezone.utc)
+
+    def image(self, number, age):
+        return {
+            "id": f"sha256:{number:064x}",
+            "tags": [f"procollab-dev-api:{number:040x}"],
+            "created": (self.now - timedelta(days=age)).isoformat(),
+        }
+
+    def test_plan_keeps_all_container_images_newest_three_and_unknown_images(self):
+        rows = [self.image(number, number) for number in range(1, 7)]
+        foreign = self.image(7, 9)
+        foreign["tags"] = ["other-project:old"]
+        mixed = self.image(8, 9)
+        mixed["tags"].append("api-web:latest")
+        dangling = self.image(9, 9)
+        dangling["tags"] = []
+        unreachable = self.image(10, 9)
+        images = {
+            row["id"]: row for row in rows + [foreign, mixed, dangling, unreachable]
+        }
+        containers = [
+            {"id": "stopped-container", "image": rows[4]["id"], "state": "exited"}
+        ]
+        plan = service.plan(
+            images,
+            containers,
+            ["volume"],
+            self.now,
+            lambda sha: sha != f"{10:040x}",
+        )
+        self.assertEqual(
+            {row["id"] for row in plan["candidates"]}, {rows[3]["id"], rows[5]["id"]}
+        )
+        self.assertEqual(plan["protected_count"], 8)
+        # Time alone does not change the inventory digest while candidates stay identical.
+        self.assertEqual(
+            plan["sha256"],
+            service.plan(
+                images,
+                containers,
+                ["volume"],
+                self.now + timedelta(minutes=1),
+                lambda sha: sha != f"{10:040x}",
+            )["sha256"],
+        )
+
+    def test_young_images_are_not_removed_even_when_more_than_three_exist(self):
+        images = {
+            row["id"]: row
+            for row in [self.image(number, 0.1 * number) for number in range(1, 7)]
+        }
+        self.assertEqual(
+            service.plan(images, [], [], self.now, lambda sha: True)["candidates"], []
+        )
+
+    def test_nanosecond_docker_timestamp_is_supported(self):
+        self.assertEqual(
+            service.created_at({"created": "2026-10-25T00:00:00.123456789Z"}).microsecond,
+            123456,
+        )
+
+    @patch.object(service.subprocess, "run")
+    def test_wrong_review_digest_never_mutates_docker(self, run):
+        with self.assertRaises(RuntimeError):
+            service.apply({"sha256": "reviewed", "candidates": []}, "changed", {}, [], [])
+        run.assert_not_called()
+
+    @patch.object(service.subprocess, "run")
+    @patch.object(service, "containers", return_value=[{"image": "new-container"}])
+    def test_new_container_aborts_before_removing_any_tag(self, containers, run):
+        image = self.image(1, 9)
+        with self.assertRaises(RuntimeError):
+            service.apply(
+                {"sha256": "same", "candidates": [image]},
+                "same",
+                {image["id"]: image},
+                [],
+                [],
+            )
+        run.assert_not_called()
+
+    @patch.object(service, "containers", return_value=[])
+    @patch.object(service, "volumes", return_value=["volume"])
+    @patch.object(service, "inventory")
+    @patch.object(service.subprocess, "run")
+    def test_removes_only_reviewed_tags_without_force_or_parent_pruning(
+        self, run, inventory, volumes, containers
+    ):
+        image, protected = self.image(1, 9), self.image(2, 1)
+        image["tags"].append(image["tags"][0] + "-0123456789ab")
+        before = {row["id"]: row for row in (image, protected)}
+        inventory.side_effect = [before, {protected["id"]: protected}]
+        service.apply(
+            {"sha256": "same", "candidates": [image]}, "same", before, [], ["volume"]
+        )
+        self.assertEqual(run.call_count, 2)
+        for call, tag in zip(run.call_args_list, image["tags"]):
+            self.assertEqual(call.args, (["docker", "image", "rm", "--no-prune", tag],))
+            self.assertEqual(call.kwargs, {"check": True})
+
+
+if __name__ == "__main__":
+    unittest.main()