Refresh template bases and rebuild when image digests change

.github/workflows/on-pull-request.yml

@@ -158,6 +158,11 @@ jobs:
   api-tests:
     name: Run API Tests (docker provider)
     runs-on: ubuntu-latest
+    services:
+      registry:
+        image: registry:3
+        ports:
+          - 5000:5000
     # Black-box e2e against a real deployment. The docker provider provisions
     # local containers on the runner, so this needs no cloud credentials.
     env:
@@ -168,15 +173,17 @@ jobs:
       # The sandbox sends its HTTPS through a proxy on the Docker bridge; the API test only checks delivery.
       SECRETS_PROXY_URL: http://172.17.0.1:8880
       SECRETS_PROXY_CA_FILE: /secrets-proxy-ca.pem
-      DOCKER_DEFAULT_IMAGE: drukbox/sandbox:ci
+      DOCKER_DEFAULT_IMAGE: localhost:5000/drukbox/sandbox:ci
       UVICORN_HOST: 127.0.0.1
 
     steps:
       - name: Check out repository
         uses: actions/checkout@v6
 
       - name: Build sandbox image
-        run: docker build -t drukbox/sandbox:ci images/local/
+        run: |
+          docker build -t "$DOCKER_DEFAULT_IMAGE" images/local/
+          docker push "$DOCKER_DEFAULT_IMAGE"
 
       - name: Build drukbox image
         run: docker build -t drukbox:api-test .

alembic/versions/0008_template_base_image_ref.py

@@ -0,0 +1,35 @@
+"""Record the base image reference used for each template."""
+
+from collections.abc import Sequence
+
+import sqlalchemy as sa
+from alembic import op
+
+revision: str = "0008_template_base_image_ref"
+down_revision: str | None = "0007_host_service_account"
+branch_labels: str | Sequence[str] | None = None
+depends_on: str | Sequence[str] | None = None
+
+
+def upgrade() -> None:
+    op.add_column("templates", sa.Column("base_image_ref", sa.Text(), nullable=True))
+    op.execute(sa.text("UPDATE templates SET base_image_ref = base_image"))
+    with op.batch_alter_table("templates") as templates:
+        templates.alter_column("base_image_ref", nullable=False)
+        templates.drop_index("ix_templates_provider_base_image_setup_script_hash")
+        templates.create_index(
+            "ix_templates_provider_base_image_ref_setup_script_hash",
+            ["provider", "base_image", "base_image_ref", "setup_script_hash"],
+            unique=True,
+        )
+
+
+def downgrade() -> None:
+    with op.batch_alter_table("templates") as templates:
+        templates.drop_index("ix_templates_provider_base_image_ref_setup_script_hash")
+        templates.drop_column("base_image_ref")
+        templates.create_index(
+            "ix_templates_provider_base_image_setup_script_hash",
+            ["provider", "base_image", "setup_script_hash"],
+            unique=True,
+        )

docs/architecture.md

@@ -77,7 +77,7 @@ the core settings knowing any provider exists.
 
 Not every provider supports every feature. The host contract must not grow
 fields that only one provider uses. Optional features are capability mix-ins.
-`TemplateCapability` declares the template create and delete surface.
+`TemplateCapability` refreshes base images and builds and deletes templates.
 
 `SecretInjectionCapability` is how a secret reaches a provider's boxes. A
 provider carries it in `secrets`. `ProxyInjection` is the default: the box gets
@@ -167,11 +167,24 @@ box before the VM goes, so nothing the seam put anywhere outlives the box. It
 never reads the row's secrets, so a lost key cannot block a teardown. The
 janitor deletes an expired host through the same path.
 
-A template is a persistent provider image keyed by provider, base image,
-and setup-script hash. `POST /templates` creates a `building` record and
-returns `202 Accepted`. Callers poll until the template becomes
-`available` or `failed`. Templates outlive hosts. Each provider builds
-and deletes its own templates behind `TemplateCapability`.
+A template is a persistent provider image keyed by provider, requested base
+image, resolved base image reference, and setup-script hash. Each
+`POST /templates` pulls the base image. Docker Sandboxes also loads that image
+into its own store, once for each reference. Drukbox saves the immutable
+registry digest reference in the template record. An image that no registry
+holds is a local build, and its image ID is the reference. The API returns
+the requested name in `base_image`.
+
+For example, `{"base_image":"sandbox:latest","setup_script":"echo ready"}`
+reuses a template only while the tag resolves to the same digest. A new digest
+creates a `building` record and returns `202 Accepted`. The build uses the
+saved digest reference even if the tag moves again. Callers poll until the
+template becomes `available` or `failed`. A pull or store failure returns
+`502`; a provider without template support returns `400`.
+
+Templates outlive hosts. Each provider owns these operations through
+`TemplateCapability`. Image pulls and store loads finish before the create
+response, so callers must allow enough time for a base image download.
 
 A host request can name an available template by its ID — the ID that
 the create returned. The template's image becomes the host image in place
@@ -197,7 +210,7 @@ Two maintenance commands run as cron jobs from the same image:
   (`POOL_SIZES`, with `POOL_SIZE` as the default provider's target) to
   hide provider cold starts.
 
-When you edit a template setup script, the hash changes. The old
+When the base digest or setup script changes, the old
 template ages out after its last lease. Pool members
 are warmed with the provider's default image and size, so a request that
 customizes its host — `image`, `env`, `template`, `instance_type`, or

src/hosts/tests/test_templates.py

@@ -32,6 +32,7 @@ async def create_template_record(
         id=uuid7(),
         provider=provider,
         base_image=base_image,
+        base_image_ref="stub@sha256:" + "a" * 64,
         setup_script_hash=setup_script_hash,
         setup_script=SETUP_SCRIPT,
         label="",

src/providers/capabilities.py

@@ -108,6 +108,10 @@ def get_public_certificate(self) -> bytes:
 class TemplateCapability(abc.ABC):
     """An ABC, not a Protocol, so ``resolve_capability`` can test inheritance."""
 
+    @abc.abstractmethod
+    async def refresh_base_image(self, image: str) -> str:
+        """Pull the image into the provider's stores and return its immutable reference."""
+
     @abc.abstractmethod
     async def build_template_image(
         self,

src/providers/docker/api.py

@@ -103,6 +103,29 @@ async def build_image(self, image: str, context_tar: bytes) -> None:
         except (aiodocker.DockerError, aiohttp.ClientError) as exc:
             raise DockerTransportError(_detail(exc)) from exc
 
+    async def pull_image(self, image: str, *, registry_auth: dict[str, str] | None = None) -> str:
+        try:
+            progress = await self._get_client().images.pull(image, auth=registry_auth)
+        except aiodocker.DockerError as exc:
+            failure = DockerTransportError(
+                f"could not pull {image!r} from its registry: {_detail(exc)}"
+            )
+            if exc.status != 404:
+                raise failure from exc
+            # No registry holds the image. A local build is pinned by its image ID.
+            try:
+                metadata = await self._get_client().images.inspect(image)
+            except (aiodocker.DockerError, aiohttp.ClientError):
+                raise failure from exc
+            return metadata["Id"]
+        except aiohttp.ClientError as exc:
+            raise DockerTransportError(str(exc)) from exc
+        for event in reversed(progress):
+            status = event.get("status", "")
+            if status.startswith("Digest: "):
+                return f"{image.partition('@')[0]}@{status.removeprefix('Digest: ')}"
+        raise DockerTransportError(f"image {image!r} has no registry digest after pulling")
+
     async def remove_image(self, image: str) -> None:
         try:
             await self._get_client().images.delete(image)

src/providers/docker/provider.py

@@ -139,6 +139,14 @@ async def delete_vm(self, name: str) -> None:
         except DockerProviderError as exc:
             raise ProviderTransportError(str(exc)) from exc
 
+    async def refresh_base_image(self, image: str) -> str:
+        try:
+            return await self.api.pull_image(
+                image, registry_auth=get_settings().get_registry_auth(image)
+            )
+        except DockerProviderError as exc:
+            raise ProviderTransportError(str(exc)) from exc
+
     async def build_template_image(
         self,
         *,

src/providers/docker/tests/test_api.py

@@ -27,6 +27,7 @@ def _fake_docker(**overrides: object) -> SimpleNamespace:
             container=MagicMock(return_value=container),
         ),
         images=SimpleNamespace(
+            pull=AsyncMock(return_value=[{"status": "Digest: sha256:" + "a" * 64}]),
             build=AsyncMock(),
             delete=AsyncMock(),
             push=AsyncMock(),
@@ -133,6 +134,58 @@ async def test_build_failure_keeps_the_engine_detail() -> None:
         await _api(fake).build_image("drukbox-template:123456789abc", b"")
 
 
+@pytest.mark.parametrize(
+    "image",
+    [
+        "sandbox:latest",
+        "sandbox:stable",
+        "registry:5000/sandbox:latest",
+        "sandbox@sha256:" + "a" * 64,
+    ],
+)
+async def test_pull_keeps_the_requested_name_and_pins_the_registry_digest(image) -> None:
+    fake = _fake_docker()
+
+    assert await _api(fake).pull_image(image) == image.partition("@")[0] + "@sha256:" + "a" * 64
+    fake.images.pull.assert_awaited_once_with(image, auth=None)
+
+
+async def test_pull_failure_does_not_reuse_the_cached_image() -> None:
+    fake = _fake_docker()
+    fake.images.pull.side_effect = DockerError(503, "registry unavailable")
+
+    with pytest.raises(
+        DockerTransportError,
+        match=r"could not pull 'sandbox:latest' from its registry: .*registry unavailable",
+    ):
+        await _api(fake).pull_image("sandbox:latest")
+
+
+async def test_pull_pins_a_local_build_by_its_image_id() -> None:
+    fake = _fake_docker()
+    fake.images.pull.side_effect = DockerError(404, "pull access denied")
+    fake.images.inspect = AsyncMock(return_value={"Id": "sha256:" + "b" * 64})
+
+    assert await _api(fake).pull_image("druks-sandbox:local") == "sha256:" + "b" * 64
+
+
+async def test_pull_reports_an_image_that_no_registry_and_no_local_build_has() -> None:
+    fake = _fake_docker()
+    fake.images.pull.side_effect = DockerError(404, "pull access denied")
+    fake.images.inspect = AsyncMock(side_effect=DockerError(404, "No such image"))
+
+    with pytest.raises(DockerTransportError, match="pull access denied"):
+        await _api(fake).pull_image("druks-sandbox:local")
+
+
+async def test_pull_requires_a_registry_digest() -> None:
+    fake = _fake_docker()
+    fake.images.pull.return_value = [{"status": "pull finished"}]
+
+    with pytest.raises(DockerTransportError, match="no registry digest"):
+        await _api(fake).pull_image("sandbox:latest")
+
+
 async def test_missing_image_maps_to_not_found() -> None:
     fake = _fake_docker()
     fake.images.delete.side_effect = DockerError(404, "No such image")

src/providers/docker_sbx/provider.py

@@ -7,6 +7,7 @@
 
 import asyncssh
 
+from core.settings import get_settings
 from providers import environment
 from providers.base import VMCreateResult, VMProvider
 from providers.capabilities import TemplateCapability
@@ -48,6 +49,7 @@ def __init__(
         # A workspace is mounted into its box, so the value files live beside them.
         self.secrets_root = settings.workspace_root / "secrets"
         self.secrets = SbxInjection(api, self.secrets_root)
+        self._loaded_base_images: set[str] = set()
 
     @classmethod
     def from_settings(cls) -> Self:
@@ -172,6 +174,22 @@ async def open_gateway_tunnel(self, name: str) -> asyncssh.SSHClientConnection:
         except (OSError, ValueError, asyncssh.Error) as exc:
             raise ProviderTransportError(f"sbx could not open a tunnel: {exc}") from exc
 
+    async def refresh_base_image(self, image: str) -> str:
+        try:
+            reference = await self.docker.pull_image(
+                image, registry_auth=get_settings().get_registry_auth(image)
+            )
+            # Each template request refreshes the base. sbx needs one load per reference.
+            if reference not in self._loaded_base_images:
+                with tempfile.TemporaryDirectory() as directory:
+                    archive = Path(directory) / "base.tar"
+                    await self.docker.save_image(image, archive)
+                    await self.api.load_template(archive)
+                self._loaded_base_images.add(reference)
+        except (OSError, DockerProviderError, DockerSbxProviderError) as exc:
+            raise ProviderTransportError(str(exc)) from exc
+        return reference
+
     async def build_template_image(
         self,
         *,

src/providers/exe/provider.py

@@ -4,13 +4,15 @@
 from providers.base import VMCreateResult, VMProvider
 from providers.capabilities import TemplateCapability
 from providers.docker.api import DockerAPI
+from providers.docker.exceptions import DockerProviderError
 from providers.docker.images import build_derived_image, remove_derived_image
 from providers.exceptions import (
     ProviderCommandError,
     ProviderHttpProxyExistsError,
     ProviderHttpProxyNotFoundError,
     ProviderNotFoundError,
     ProviderTargetVMNotFoundError,
+    ProviderTransportError,
 )
 from providers.exe.api import ExeAPI
 from providers.exe.exceptions import (
@@ -98,6 +100,14 @@ async def delete_vm(self, name: str) -> None:
         except ExeVMNotFoundError as exc:
             raise ProviderNotFoundError(str(exc)) from exc
 
+    async def refresh_base_image(self, image: str) -> str:
+        try:
+            return await self.docker.pull_image(
+                image, registry_auth=get_settings().get_registry_auth(image)
+            )
+        except DockerProviderError as exc:
+            raise ProviderTransportError(str(exc)) from exc
+
     async def build_template_image(
         self,
         *,

src/providers/tests/test_capabilities.py

@@ -63,6 +63,9 @@ async def aclose(self) -> None:
 
 
 class StubTemplateProvider(StubProvider, TemplateCapability):
+    async def refresh_base_image(self, image: str) -> str:
+        return "stub@sha256:" + "a" * 64
+
     async def build_template_image(self, *, base_image: str, setup_script: str, label: str) -> str:
         return f"{base_image}:{label}"
 

src/templates/api.py

@@ -8,7 +8,7 @@
 
 from core.database import get_session
 from hosts.auth import require_auth
-from providers.exceptions import ProviderError, UnknownProviderError
+from providers.exceptions import CapabilityUnsupportedError, ProviderError, UnknownProviderError
 from templates.exceptions import TemplateTeardownError
 from templates.models import Template
 from templates.schemas import TemplateCreate, TemplateOut
@@ -41,8 +41,10 @@ async def create_template(
             setup_script=payload.setup_script,
             label=payload.label,
         )
-    except UnknownProviderError as exc:
+    except (UnknownProviderError, CapabilityUnsupportedError) as exc:
         raise HTTPException(status_code=400, detail=str(exc)) from exc
+    except ProviderError as exc:
+        raise HTTPException(status_code=502, detail=str(exc)) from exc
     except SQLAlchemyError as exc:
         logger.exception("unexpected database error during template creation")
         raise HTTPException(

src/templates/models.py

@@ -20,9 +20,10 @@ class Template(Base):
     __tablename__ = "templates"
     __table_args__ = (
         Index(
-            "ix_templates_provider_base_image_setup_script_hash",
+            "ix_templates_provider_base_image_ref_setup_script_hash",
             "provider",
             "base_image",
+            "base_image_ref",
             "setup_script_hash",
             unique=True,
         ),
@@ -31,6 +32,7 @@ class Template(Base):
     id: Mapped[uuid.UUID] = mapped_column(Uuid(as_uuid=True), primary_key=True, default=uuid7)
     provider: Mapped[str] = mapped_column(String(20))
     base_image: Mapped[str] = mapped_column(Text)
+    base_image_ref: Mapped[str] = mapped_column(Text)
     setup_script_hash: Mapped[str] = mapped_column(String(64))
     setup_script: Mapped[str] = mapped_column(Text)
     label: Mapped[str] = mapped_column(Text, default="")

src/templates/service.py

@@ -57,11 +57,14 @@ async def get_or_create(
 
         vm = get_vm_provider(provider)
         resolved_base_image = base_image or vm.default_image
+        capability = resolve_capability(vm, TemplateCapability)
+        base_image_ref = await capability.refresh_base_image(resolved_base_image)
         setup_script_hash = hashlib.sha256(setup_script.encode("utf-8")).hexdigest()
         now = utc_now()
         template = Template(
             provider=vm.name,
             base_image=resolved_base_image,
+            base_image_ref=base_image_ref,
             setup_script_hash=setup_script_hash,
             setup_script=setup_script,
             label=label,
@@ -87,6 +90,7 @@ async def get_or_create(
                 select(Template)
                 .where(Template.provider == vm.name)
                 .where(Template.base_image == resolved_base_image)
+                .where(Template.base_image_ref == base_image_ref)
                 .where(Template.setup_script_hash == setup_script_hash)
             )
         ).scalar_one_or_none()
@@ -108,7 +112,7 @@ async def build(self, template_id: uuid.UUID) -> None:
                     TemplateCapability,
                 )
                 image = await capability.build_template_image(
-                    base_image=template.base_image,
+                    base_image=template.base_image_ref,
                     setup_script=template.setup_script,
                     label=template.label,
                 )