From e5456c5ac63995db8467fe06d4cea25107be0738 Mon Sep 17 00:00:00 2001 From: CrypticSwarm Date: Sun, 4 Oct 2026 11:29:45 -0500 Subject: [PATCH 01/14] Let a tong mount a persistent named volume A tong had nowhere to keep state: anything it wrote inside its container was lost whenever the container was recreated, which for a shared tong happens on every definition change. A model store such as ollama's is tens of GB to download again. Tongs may never bind a raw host path, so the new `volume::/target[:mode]` magic word asks for a docker named volume that the launcher names, and docker creates on first use and keeps when the container goes. The docker name is `swarmforge-volume-[-]_`. The digest is the volume's identity: the first 32 hex digits of a SHA-256 over the tong's scope and its raw name. Joining a scope token and a sanitized name with `-` would let distinct tongs collide, both by aliasing (`github.` and `github` sanitize alike) and by brute-forcing a short digest. The hint is the sanitized name, kept only so `docker volume ls` is readable. Volume names may not contain `_` and the digest has a fixed width, so the last `_` always splits the volume name off. A volume mount must be given a scope; choosing one per layer comes next. - `parse_mount` keeps returning `(word, target, mode)` so its callers are untouched; `mount_volume_name` reads the name through the same grammar, so validation and argv assembly cannot disagree on it. - A volume target follows the workspace target's rules: it must be absolute, must not resolve to `/`, and may not overlap the tong's wiring or its other mounts. - A tong mounts each volume once. A second target for the same data, possibly with a different access mode, is much more likely a mistake than something anyone needs. - Volume names are capped at 64 characters and the readable hint at 32, keeping the docker name well inside a 255-byte directory name. --- swarmforge/tongs/__init__.py | 18 +++- swarmforge/tongs/argv.py | 7 +- swarmforge/tongs/mounts.py | 141 +++++++++++++++++++++++++++----- tests/test_tongs_argv.py | 15 ++++ tests/test_tongs_mounts.py | 154 ++++++++++++++++++++++++++++++++++- tests/test_tongs_validate.py | 35 +++++++- 6 files changed, 344 insertions(+), 26 deletions(-) diff --git a/swarmforge/tongs/__init__.py b/swarmforge/tongs/__init__.py index 6a67683..bcb69f3 100644 --- a/swarmforge/tongs/__init__.py +++ b/swarmforge/tongs/__init__.py @@ -15,7 +15,7 @@ discovery reading the layer directories and merging by name validate schema validation secrets secret references, the provider table, FIFO delivery - mounts the `mounts:` magic words and their docker bind specs + mounts the `mounts:` magic words and their docker `-v` specs mcp what each `interface:` contributes to the anvil network per-session network planning approvals config hash, privilege summary, approval keying @@ -111,16 +111,24 @@ from .mounts import ( DEFAULT_DOCKER_SOCKET, DEFAULT_WORKSPACE_MOUNT_TARGET, + LOCAL_VOLUME_SCOPE, MOUNT_MODES, MOUNT_WORDS, + ORG_VOLUME_SCOPE, + VOLUME_DIGEST_LENGTH, + VOLUME_MOUNT, + VOLUME_NAME_MAX_LENGTH, WORKSPACE_MOUNT, + WORKSPACE_VOLUME_SCOPE, mount_destination, mount_target_error, + mount_volume_name, normalize_mount_target, overlapping_mount_error, parse_mount, reserved_mount_targets, tong_mount_specs, + tong_volume_name, workspace_mount_placements, ) from .network import SESSION_NET_PREFIX, plan_network, session_network_name @@ -212,16 +220,24 @@ "warn", "DEFAULT_DOCKER_SOCKET", "DEFAULT_WORKSPACE_MOUNT_TARGET", + "LOCAL_VOLUME_SCOPE", "MOUNT_MODES", "MOUNT_WORDS", + "ORG_VOLUME_SCOPE", + "VOLUME_DIGEST_LENGTH", + "VOLUME_MOUNT", + "VOLUME_NAME_MAX_LENGTH", "WORKSPACE_MOUNT", + "WORKSPACE_VOLUME_SCOPE", "mount_destination", "mount_target_error", + "mount_volume_name", "normalize_mount_target", "overlapping_mount_error", "parse_mount", "reserved_mount_targets", "tong_mount_specs", + "tong_volume_name", "workspace_mount_placements", "SESSION_NET_PREFIX", "plan_network", diff --git a/swarmforge/tongs/argv.py b/swarmforge/tongs/argv.py index 4020bf6..893595b 100644 --- a/swarmforge/tongs/argv.py +++ b/swarmforge/tongs/argv.py @@ -125,6 +125,7 @@ def tong_run_argv( entrypoint=None, command=None, extra_mount_specs=None, + volume_scope=None, ): """Full `docker run -d` argv that starts one tong container. @@ -158,6 +159,9 @@ def tong_run_argv( outside the definition (today the git-dir mounts that ride along with a `workspace` mount -- see swarmforge.anvil); they are appended after the definition's own mounts, mirroring how the Makefile orders the anvil's. + + `volume_scope` is the `(class, path)` pair the orchestrator picked for the + tong's `volume:` mounts from the layer it came from (see `tong_volume_name`). """ if entrypoint is None and command is None: entrypoint, command = declared_run_override(defn) @@ -180,7 +184,8 @@ def tong_run_argv( effective_env.setdefault(WORKSPACE_HOST_ENV, workspace) for key in sorted(effective_env): argv += ["-e", "%s=%s" % (key, effective_env[key])] - for spec in tong_mount_specs(defn, workspace, socket_path=socket_path): + for spec in tong_mount_specs(defn, workspace, socket_path=socket_path, + tong_name=name, volume_scope=volume_scope): argv += ["-v", spec] for spec in extra_mount_specs or []: argv += ["-v", spec] diff --git a/swarmforge/tongs/mounts.py b/swarmforge/tongs/mounts.py index aa6d127..13b4605 100644 --- a/swarmforge/tongs/mounts.py +++ b/swarmforge/tongs/mounts.py @@ -1,25 +1,49 @@ -"""The `mounts:` magic words and the docker bind specs they turn into. +"""The `mounts:` magic words and the docker `-v` specs they turn into. A definition never names a raw host path: it asks for a mount by word, and this module decides where that lands inside the container and what `-v` value docker -is given. Validation and argv assembly both come through here, so a mount is -judged by the same rules whichever asks. +is given: a host path the launcher knows, or a named volume it names. Validation +and argv assembly both come through here, so a mount is judged by the same rules +whichever asks. """ +import hashlib +import json import posixpath +import re + +from swarmforge.names import sanitize_token from .model import SOCKET_MOUNT from .secrets import SECRET_FIFO_DIR, SECRET_INJECT_SHELL, partition_secret_env WORKSPACE_MOUNT = "workspace" -MOUNT_WORDS = (WORKSPACE_MOUNT, SOCKET_MOUNT) +VOLUME_MOUNT = "volume" +MOUNT_WORDS = (WORKSPACE_MOUNT, SOCKET_MOUNT, VOLUME_MOUNT) +TARGETED_MOUNT_WORDS = (WORKSPACE_MOUNT, VOLUME_MOUNT) DEFAULT_WORKSPACE_MOUNT_TARGET = "/workspace" DEFAULT_DOCKER_SOCKET = "/var/run/docker.sock" # An allowlist, so an unvetted mount option cannot reach the daemon. MOUNT_MODES = ("ro", "rw") +VOLUME_NAME_PREFIX = "swarmforge-volume" + +# 128 bits, so no definition can search its way onto another scope's volume. +VOLUME_DIGEST_LENGTH = 32 + +ORG_VOLUME_SCOPE = "org" +WORKSPACE_VOLUME_SCOPE = "workspace" +LOCAL_VOLUME_SCOPE = "local" + +# No `_`, so the last `_` of a docker volume name splits the volume back off. +VOLUME_NAME_RE = re.compile(r"[A-Za-z0-9][A-Za-z0-9.-]*") + +# With the hint cap, keeps a docker volume name far below a 255-byte directory name. +VOLUME_NAME_MAX_LENGTH = 64 +VOLUME_HINT_LIMIT = 32 + def _has_socket_mount(defn): for mount in defn.get("mounts") or []: @@ -55,14 +79,33 @@ def parse_mount(mount, words=MOUNT_WORDS): path naming where the container sees the mount, and an optional access mode, which is always last -- so `workspace`, `workspace:ro`, `workspace:/work` and `workspace:/work:ro` are all valid. `target`/`mode` are None when not declared; - the caller supplies the default target for its magic word. + the caller supplies the default target for its magic word. The `volume` word + is spelled `volume::/target[:mode]`: its name comes first and its target + is required, since a named volume has no natural place to land. The name is + left out of the tuple and read with `mount_volume_name`. The word is checked against `words` before the rest of the entry, since a mount nobody recognizes has no meaningful target or mode; narrow that set to accept fewer words, never widen it to accept a raw host path. Raises `ValueError` for an unaccepted word, a field that is neither an absolute path nor a mode, a mode - that is not last, more than one target, or a target resolving to the root. + that is not last, more than one target, a target resolving to the root, or a + volume mount whose name is malformed or whose target is missing. + """ + word, _, target, mode = _split_mount(mount, words) + return word, target, mode + + +def mount_volume_name(mount): + """The name a `volume::...` entry declares, or None for any other word. + + Judged by the same grammar as `parse_mount`, so it raises `ValueError` for + exactly the entries that does. """ + return _split_mount(mount, MOUNT_WORDS)[1] + + +def _split_mount(mount, words): + """`(word, volume name, target, mode)` for a `mounts:` entry; see `parse_mount`.""" fields = mount.split(":") word, fields = fields[0], fields[1:] if word not in words: @@ -76,6 +119,20 @@ def parse_mount(mount, words=MOUNT_WORDS): "unknown mount %r (expected %s)" % (mount, " or ".join(repr(known) for known in words)) ) + volume = None + if word == VOLUME_MOUNT: + if not fields or not VOLUME_NAME_RE.fullmatch(fields[0]): + raise ValueError( + "mount %r: a volume mount is spelled volume::/target[:mode], " + "with a name of letters, digits, '.' and '-' that starts with a " + "letter or digit" % (mount,) + ) + if len(fields[0]) > VOLUME_NAME_MAX_LENGTH: + raise ValueError( + "mount %r: a volume name is at most %d characters" + % (mount, VOLUME_NAME_MAX_LENGTH) + ) + volume, fields = fields[0], fields[1:] target = None mode = None for index, field in enumerate(fields): @@ -105,7 +162,9 @@ def parse_mount(mount, words=MOUNT_WORDS): "mount %r: %r is neither an absolute target path nor an access " "mode (%s)" % (mount, field, "/".join(MOUNT_MODES)) ) - return word, target, mode + if word == VOLUME_MOUNT and target is None: + raise ValueError("mount %r: a volume mount needs an absolute target path" % (mount,)) + return word, volume, target, mode def reserved_mount_targets(defn, socket_path=DEFAULT_DOCKER_SOCKET): @@ -133,21 +192,24 @@ def mount_destination(word, target, socket_path=DEFAULT_DOCKER_SOCKET): The declared target when there is one, otherwise the word's default: /workspace for the workspace, its own host path for the socket (that is where a docker - client looks for it). Raises `ValueError` for any other word, so a magic word + client looks for it). A volume has no default, so it needs its target. Raises + `ValueError` for a targetless volume and for any other word, so a magic word added without a default cannot inherit the socket's. """ if word == WORKSPACE_MOUNT: return normalize_mount_target(target or DEFAULT_WORKSPACE_MOUNT_TARGET) if word == SOCKET_MOUNT: return normalize_mount_target(socket_path) + if word == VOLUME_MOUNT and target is not None: + return normalize_mount_target(target) raise ValueError("mount %r has no destination" % (word,)) def mount_target_error(mount, word, target, destination, reserved): """Why a mount's target is unusable, or None if it is fine. - Two rules beyond the grammar: only `workspace` takes a target, and no mount may - land on one of the `reserved` destinations (`{path: why}`, from + Two rules beyond the grammar: only `workspace` and `volume` take a target, and + no mount may land on one of the `reserved` destinations (`{path: why}`, from `reserved_mount_targets`) -- docker layers overlapping mounts, which either buries the tong's wiring or has docker create a mountpoint inside the user's workspace on the host. `destination` is where the mount actually lands @@ -155,10 +217,10 @@ def mount_target_error(mount, word, target, destination, reserved): Validation and argv assembly both ask this, so both give the same verdict and the same message for one `reserved` map. """ - if word != WORKSPACE_MOUNT: + if word not in TARGETED_MOUNT_WORDS: if target is not None: - return ("mount %r: only the '%s' mount takes a target path" - % (mount, WORKSPACE_MOUNT)) + return ("mount %r: only the %s mounts take a target path" + % (mount, " and ".join(repr(w) for w in TARGETED_MOUNT_WORDS))) # The socket lands on its own reserved path by construction. return None for path, why in sorted(reserved.items()): @@ -170,27 +232,61 @@ def mount_target_error(mount, word, target, destination, reserved): def overlapping_mount_error(mount, destination, placed): """Why a mount collides with one already placed, or None if it is clear. - `placed` is the `(mount, destination)` pairs accepted so far. Docker refuses two - binds onto one destination outright, and creates the inner mountpoint of nested - ones inside the outer bind -- inside the user's workspace, for a workspace bind. + `placed` is the `(mount, destination)` pairs accepted so far, each entry one + `parse_mount` accepted, as `mount` is. Docker refuses two binds onto one + destination outright, and creates the inner mountpoint of nested ones inside + the outer bind -- inside the user's workspace, for a workspace bind. One volume + is mounted once: a second target for the same data, perhaps with another + access mode, is far likelier a mistake than a need. """ + volume = mount_volume_name(mount) for other, other_destination in placed: if _targets_overlap(destination, other_destination): return ("mount %r: %s overlaps mount %r at %s" % (mount, destination, other, other_destination)) + if volume is not None and mount_volume_name(other) == volume: + return "mount %r: volume %r is already mounted by %r" % (mount, volume, other) return None -def tong_mount_specs(defn, workspace, socket_path=DEFAULT_DOCKER_SOCKET): +def tong_volume_name(tong_name, volume, scope): + """The docker named volume behind one tong's `volume:` mount. + + `swarmforge-volume-[-]_`. The digest is the volume's + identity: the first `VOLUME_DIGEST_LENGTH` hex digits of a SHA-256 over the + scope and the raw tong name, where `scope` is a `(class, path)` pair -- an + `ORG_VOLUME_SCOPE` or `WORKSPACE_VOLUME_SCOPE` with its canonical directory, + or `(LOCAL_VOLUME_SCOPE, None)` for this machine's own layers. Two tongs + share a volume only from one scope under exactly one name. The hint is the + tong name sanitized for docker and cut to `VOLUME_HINT_LIMIT`, there to be + read in `docker volume ls`; it decides nothing, since sanitizing can turn two + names into one. Volume names + exclude `_` and the digest has a fixed width, so the last `_` splits the + volume back off. + """ + scope_class, scope_path = scope + key = json.dumps([scope_class, scope_path, tong_name]) + digest = hashlib.sha256(key.encode("utf-8")).hexdigest()[:VOLUME_DIGEST_LENGTH] + hint = sanitize_token(tong_name)[:VOLUME_HINT_LIMIT].rstrip("-_.") + parts = [VOLUME_NAME_PREFIX] + ([hint] if hint else []) + [digest] + return "%s_%s" % ("-".join(parts), volume) + + +def tong_mount_specs(defn, workspace, socket_path=DEFAULT_DOCKER_SOCKET, + tong_name=None, volume_scope=None): """Concrete docker `-v` specs for a tong's `mounts:` magic words. Returns the list of `-v` *values* (the orchestrator pairs each with a `-v` flag). `workspace[:/target][:mode]` mounts the session workspace, at /workspace unless the definition names another target; `docker-socket[:mode]` - bind-mounts the host docker socket onto the same path it has on the host. + bind-mounts the host docker socket onto the same path it has on the host; + `volume::/target[:mode]` mounts the named volume + `tong_volume_name(tong_name, name, volume_scope)`, which docker creates on + first use and never removes with the container. Raises `ValueError` for a non-string entry, a malformed mount, an unusable or - colliding destination, or a `workspace` mount when no workspace path is known - -- a definition never names a raw host path, so anything else is a mistake that + colliding destination, a `workspace` mount when no workspace path is known, or + a `volume` mount without a `tong_name` and `volume_scope` to name it by -- a + definition never names a raw host path, so anything else is a mistake that should stop the launch. """ specs = [] @@ -211,6 +307,11 @@ def tong_mount_specs(defn, workspace, socket_path=DEFAULT_DOCKER_SOCKET): source = workspace elif word == SOCKET_MOUNT: source = socket_path + elif word == VOLUME_MOUNT: + if not tong_name or volume_scope is None: + raise ValueError( + "mount %r needs the tong's name and scope to name its volume" % (mount,)) + source = tong_volume_name(tong_name, mount_volume_name(mount), volume_scope) else: # Unreachable via `parse_mount`; a new word must not inherit the socket bind. raise ValueError("mount %r has no docker spec" % (mount,)) diff --git a/tests/test_tongs_argv.py b/tests/test_tongs_argv.py index 140d503..3115ce1 100644 --- a/tests/test_tongs_argv.py +++ b/tests/test_tongs_argv.py @@ -193,6 +193,21 @@ def argv_for(mounts): [part for part in default if part != "/ws:/workspace"], ) + def test_run_argv_names_a_volume_after_the_tong_and_its_scope(self): + defn = def_of(NONE_TONG) + defn["mounts"] = ["volume:models:/root/.ollama"] + for scope in ((tongs.LOCAL_VOLUME_SCOPE, None), + (tongs.ORG_VOLUME_SCOPE, "/orgs/acme/.swarmforge/tongs")): + argv = tongs.tong_run_argv( + "ollama", defn, container_name="c", network="n", alias="ollama", + volume_scope=scope, + ) + self.assertEqual( + [argv[i + 1] for i, part in enumerate(argv) if part == "-v"], + [tongs.tong_volume_name("ollama", "models", scope) + ":/root/.ollama"], + scope, + ) + def test_run_argv_extra_mount_specs_follow_definition_mounts(self): defn = def_of(NONE_TONG) defn["mounts"] = ["workspace"] diff --git a/tests/test_tongs_mounts.py b/tests/test_tongs_mounts.py index fc5f631..e9d11dd 100644 --- a/tests/test_tongs_mounts.py +++ b/tests/test_tongs_mounts.py @@ -14,6 +14,9 @@ from swarmforge import tongs +LOCAL = (tongs.LOCAL_VOLUME_SCOPE, None) + + class MountGrammarTests(unittest.TestCase): """The mount grammar and target policy, exercised directly -- `validate_tong` and `tong_mount_specs` both delegate here.""" @@ -29,6 +32,52 @@ def test_parse_mount_splits_the_optional_fields(self): tongs.parse_mount("docker-socket:ro"), ("docker-socket", None, "ro") ) + def test_parse_mount_keeps_a_volume_name_out_of_the_tuple(self): + self.assertEqual( + tongs.parse_mount("volume:models:/data"), ("volume", "/data", None) + ) + self.assertEqual( + tongs.parse_mount("volume:models:/data:ro"), ("volume", "/data", "ro") + ) + self.assertEqual(tongs.mount_volume_name("volume:models:/data:ro"), "models") + self.assertEqual(tongs.mount_volume_name("volume:v1.2-x:/data"), "v1.2-x") + + def test_mount_volume_name_is_none_for_other_words(self): + self.assertIsNone(tongs.mount_volume_name("workspace:/code:ro")) + self.assertIsNone(tongs.mount_volume_name("docker-socket")) + + def test_parse_mount_rejects_a_malformed_volume_name(self): + # `_` is what splits the volume back off its docker name, so it cannot be in one. + for mount in ("volume", "volume::/data", "volume:/data", "volume:my_models:/data", + "volume:-models:/data", "volume:.models:/data", "volume:mo dels:/data", + "volume:models\n:/data"): + with self.assertRaisesRegex(ValueError, "volume::/target", msg=mount): + tongs.parse_mount(mount) + with self.assertRaises(ValueError, msg=mount): + tongs.mount_volume_name(mount) + + def test_parse_mount_caps_a_volume_name(self): + longest = "v" * tongs.VOLUME_NAME_MAX_LENGTH + self.assertEqual(tongs.mount_volume_name("volume:%s:/data" % longest), longest) + with self.assertRaisesRegex(ValueError, "at most 64 characters"): + tongs.parse_mount("volume:%s:/data" % (longest + "v")) + + def test_parse_mount_requires_a_volume_target(self): + for mount in ("volume:models", "volume:models:ro"): + with self.assertRaisesRegex(ValueError, "needs an absolute target path", msg=mount): + tongs.parse_mount(mount) + + def test_parse_mount_judges_a_volume_target_like_a_workspace_one(self): + for mount, message in ( + ("volume:models:data", "neither an absolute target path"), + ("volume:models:/", "not a usable target path"), + ("volume:models:/a:/b", "more than one target path"), + ("volume:models:ro:/data", "must be the last field"), + ("volume:models:/da ta", "whitespace"), + ): + with self.assertRaisesRegex(ValueError, message, msg=mount): + tongs.parse_mount(mount) + def test_parse_mount_rejects_a_raw_host_path(self): with self.assertRaisesRegex(ValueError, "unknown mount"): tongs.parse_mount("/etc/passwd:/etc/passwd") @@ -77,6 +126,65 @@ def test_mount_destination_defaults_per_word(self): self.assertEqual( tongs.mount_destination("docker-socket", None, "/run/d.sock"), "/run/d.sock" ) + self.assertEqual(tongs.mount_destination("volume", "//data/"), "/data") + + def test_mount_destination_refuses_a_volume_without_a_target(self): + with self.assertRaisesRegex(ValueError, "no destination"): + tongs.mount_destination("volume", None) + + def test_mount_target_error_holds_a_volume_to_the_reserved_paths(self): + reserved = {"/run/x": "where the launcher delivers this tong's secrets"} + self.assertIsNone(tongs.mount_target_error( + "volume:v:/data", "volume", "/data", "/data", reserved)) + self.assertIn("overlaps /run/x", tongs.mount_target_error( + "volume:v:/run", "volume", "/run", "/run", reserved)) + + def test_tong_volume_name_is_a_hint_then_a_fixed_width_digest(self): + name = tongs.tong_volume_name("my tong_x", "models", LOCAL) + self.assertRegex(name, r"\Aswarmforge-volume-my-tong_x-[0-9a-f]{32}_models\Z") + self.assertEqual(tongs.VOLUME_DIGEST_LENGTH, 32) + self.assertEqual(name.rsplit("_", 1)[1], "models") + self.assertEqual(name, tongs.tong_volume_name("my tong_x", "models", LOCAL)) + # A name that sanitizes to nothing keeps the digest alone. + self.assertRegex( + tongs.tong_volume_name("__", "v", LOCAL), r"\Aswarmforge-volume-[0-9a-f]{32}_v\Z" + ) + + def test_tong_volume_name_is_pinned(self): + # Literal on purpose: any change to the formula orphans every existing volume. + cases = ( + ("ollama", LOCAL, + "swarmforge-volume-ollama-fbdbeb6d19743ef7c7724f97755c49d1_models"), + ("ollama", (tongs.ORG_VOLUME_SCOPE, "/orgs/acme/.swarmforge/tongs"), + "swarmforge-volume-ollama-a0afff5cc924fc93f5f25e86a52f5888_models"), + ("ollama", (tongs.WORKSPACE_VOLUME_SCOPE, "/home/me/proj"), + "swarmforge-volume-ollama-9d5322a3cc671ff97654fc52727d1dd3_models"), + ("caf\u00e9", LOCAL, + "swarmforge-volume-caf-a3195456dbda485c7f802622dae166e0_models"), + # The hint is cut to 32 and loses the separator it would end on. + ("a" * 31 + "-" + "b" * 10, LOCAL, + "swarmforge-volume-" + "a" * 31 + "-c9719a32926cd73d826a8ed165eb2523_models"), + ) + for tong_name, scope, expected in cases: + self.assertEqual(tongs.tong_volume_name(tong_name, "models", scope), expected) + + def test_tong_volume_name_digests_the_raw_name_not_the_hint(self): + # Both sanitize to `github`; the hint is for reading, never for identity. + dotted = tongs.tong_volume_name("github.", "v", LOCAL) + plain = tongs.tong_volume_name("github", "v", LOCAL) + self.assertTrue(dotted.startswith("swarmforge-volume-github-")) + self.assertTrue(plain.startswith("swarmforge-volume-github-")) + self.assertNotEqual(dotted, plain) + + def test_tong_volume_name_differs_per_scope(self): + names = { + tongs.tong_volume_name("cache", "v", scope) + for scope in (LOCAL, (tongs.ORG_VOLUME_SCOPE, "/orgs/acme/.swarmforge/tongs"), + (tongs.ORG_VOLUME_SCOPE, "/orgs/globex/.swarmforge/tongs"), + (tongs.WORKSPACE_VOLUME_SCOPE, "/orgs/acme/.swarmforge/tongs"), + (tongs.WORKSPACE_VOLUME_SCOPE, "/home/me/proj")) + } + self.assertEqual(len(names), 5) def test_mount_target_error_names_the_mount_and_the_reason(self): reserved = {"/run/x": "where the launcher delivers this tong's secrets"} @@ -90,7 +198,7 @@ def error(mount, word, target): self.assertIsNone(error("workspace:/code", "workspace", "/code")) self.assertIsNone(error("docker-socket", "docker-socket", None)) self.assertIn( - "only the 'workspace' mount takes a target path", + "only the 'workspace' and 'volume' mounts take a target path", error("docker-socket:/s", "docker-socket", "/s"), ) # Overlap in both directions: the target above the reserved path and under it. @@ -111,9 +219,17 @@ def test_overlapping_mount_error_catches_duplicates_and_nesting(self): for destination in ("/code", "/code/sub", "/"): self.assertIn( "overlaps mount 'workspace:/code'", - tongs.overlapping_mount_error("m", destination, placed), + tongs.overlapping_mount_error("workspace", destination, placed), ) + def test_overlapping_mount_error_refuses_one_volume_mounted_twice(self): + placed = [("volume:models:/a", "/a")] + self.assertIsNone(tongs.overlapping_mount_error("volume:cache:/b", "/b", placed)) + self.assertIn( + "volume 'models' is already mounted by 'volume:models:/a'", + tongs.overlapping_mount_error("volume:models:/b:ro", "/b", placed), + ) + class MountSpecTests(unittest.TestCase): """The mount specs a tong contributes to its own `docker run` argv, and the @@ -138,7 +254,7 @@ def test_mount_specs_workspace_custom_target(self): def test_mount_specs_socket_target_raises(self): with self.assertRaises(ValueError) as caught: tongs.tong_mount_specs({"mounts": ["docker-socket:/run/d.sock"]}, "/ws") - self.assertIn("only the 'workspace' mount takes a target path", str(caught.exception)) + self.assertIn("only the 'workspace' and 'volume' mounts take a target path", str(caught.exception)) def test_mount_specs_normalize_the_target(self): # docker cleans a bind destination, so it is handed the spelling the overlap checks judged. @@ -163,6 +279,35 @@ def test_mount_specs_socket_honors_custom_path(self): specs = tongs.tong_mount_specs({"mounts": ["docker-socket:ro"]}, "/ws", socket_path="/run/d.sock") self.assertEqual(specs, ["/run/d.sock:/run/d.sock:ro"]) + def test_mount_specs_volume_names_a_docker_volume(self): + defn = {"mounts": ["volume:models:/root/.ollama", "volume:cache:/cache:ro"]} + scope = (tongs.ORG_VOLUME_SCOPE, "/orgs/acme/.swarmforge/tongs") + self.assertEqual( + tongs.tong_mount_specs(defn, None, tong_name="ollama", volume_scope=scope), + [tongs.tong_volume_name("ollama", "models", scope) + ":/root/.ollama", + tongs.tong_volume_name("ollama", "cache", scope) + ":/cache:ro"], + ) + + def test_mount_specs_volume_target_normalized(self): + self.assertEqual( + tongs.tong_mount_specs( + {"mounts": ["volume:v://data/"]}, None, tong_name="t", volume_scope=LOCAL), + [tongs.tong_volume_name("t", "v", LOCAL) + ":/data"], + ) + + def test_mount_specs_volume_without_a_tong_name_or_scope_raises(self): + # Either would let tongs that must stay apart land on one volume. + for kwargs in ({"volume_scope": LOCAL}, {"tong_name": "t"}): + with self.assertRaisesRegex(ValueError, "name and scope", msg=kwargs): + tongs.tong_mount_specs({"mounts": ["volume:v:/data"]}, None, **kwargs) + + def test_mount_specs_volume_overlapping_another_mount_raises(self): + for mounts in (["workspace", "volume:v:/workspace/cache"], + ["volume:a:/data", "volume:b:/data"]): + with self.assertRaisesRegex(ValueError, "overlaps", msg=mounts): + tongs.tong_mount_specs( + {"mounts": mounts}, "/ws", tong_name="t", volume_scope=LOCAL) + def test_mount_specs_no_mounts_is_empty(self): self.assertEqual(tongs.tong_mount_specs({}, "/ws"), []) @@ -171,6 +316,9 @@ def test_workspace_mount_placements_empty_without_workspace_mount(self): self.assertEqual( tongs.workspace_mount_placements({"mounts": ["docker-socket"]}), [] ) + self.assertEqual( + tongs.workspace_mount_placements({"mounts": ["volume:v:/data"]}), [] + ) def test_workspace_mount_placements_default_target_and_mode(self): self.assertEqual( diff --git a/tests/test_tongs_validate.py b/tests/test_tongs_validate.py index b3ed75c..1b3dc67 100644 --- a/tests/test_tongs_validate.py +++ b/tests/test_tongs_validate.py @@ -179,7 +179,40 @@ def test_two_target_paths_rejected(self): def test_socket_mount_target_rejected(self): # A docker client looks for the socket at its host path, so it cannot move. errors = tongs.validate_tong("t", self._base(mounts=["docker-socket:/run/d.sock"])) - self.assertTrue(any("only the 'workspace' mount takes a target path" in e for e in errors)) + self.assertTrue(any( + "only the 'workspace' and 'volume' mounts take a target path" in e for e in errors + )) + + def test_volume_mount_accepted(self): + for mount in ("volume:models:/root/.ollama", "volume:models:/data:ro"): + self.assertEqual(tongs.validate_tong("t", self._base(mounts=[mount])), [], mount) + + def test_malformed_volume_mount_rejected(self): + for mount, message in ( + ("volume:my_models:/data", "volume::/target"), + ("volume:/data", "volume::/target"), + ("volume:models", "needs an absolute target path"), + ("volume:models:/", "not a usable target path"), + ): + errors = tongs.validate_tong("t", self._base(mounts=[mount])) + self.assertTrue(any(message in e for e in errors), (mount, errors)) + + def test_volume_target_overlapping_the_tongs_wiring_or_mounts_rejected(self): + secret = self._base(mounts=["volume:v:/run"]) + secret["env"] = {"TOKEN": "${secret:op:op://Work/t}"} + for defn in ( + secret, + self._base(mounts=["docker-socket", "volume:v:/var/run"]), + self._base(mounts=["workspace", "volume:v:/workspace/.cache"]), + self._base(mounts=["volume:a:/data", "volume:b:/data/sub"]), + ): + errors = tongs.validate_tong("t", defn) + self.assertTrue(any("overlaps" in e for e in errors), defn["mounts"]) + + def test_one_volume_mounted_twice_rejected(self): + errors = tongs.validate_tong( + "t", self._base(mounts=["volume:models:/a", "volume:models:/b:ro"])) + self.assertTrue(any("already mounted" in e for e in errors), errors) def test_extra_aliases_accepted_on_network_facing_kinds(self): # A client matching a certificate CN dials the dotted name, not the canonical alias. From 6c97a05ab655ad70eab36f0b0a30e031a7e9c585 Mon Sep 17 00:00:00 2001 From: CrypticSwarm Date: Sun, 4 Oct 2026 11:31:13 -0500 Subject: [PATCH 02/14] Keep each org's and checkout's tong volumes apart A volume named only after its tong would hand one org's data to another org that ships a tong with the same filename, and one cloned repo's data to any other clone. `volume_scope` now picks the scope that goes into the volume's digest from the layer the tong came from: - org: the org's tongs directory, on both lifecycles - workspace: the checkout's top-level path, with symlinks resolved, so every session in the checkout shares the volume whichever subdirectory it starts from. The path is the key, as it is for approvals, so a later checkout at the same path inherits it. - user and repo: this machine, so a repo tong and the same-named user tong it replaces in the merge share one volume An org or workspace tong that mounts a volume refuses to start when there is no directory to scope it by, instead of falling back to the machine-wide scope. A workspace-sourced `shared` tong that mounts a volume is refused. Its container name is global but its volume belongs to one checkout, so a session in another checkout that ships the same definition would reuse the container with the first checkout's volume still mounted. Refusing it follows the existing rule against a `shared` tong mounting the workspace. Teardown never touches a volume, and a shared tong recreated after a definition change mounts the same volume again. --- swarmforge/anvil/orchestrate.py | 75 +++++++++++++++---- swarmforge/tongs/__init__.py | 2 + swarmforge/tongs/mounts.py | 33 ++++++++- tests/test_anvil_orchestrate.py | 123 ++++++++++++++++++++++++++++++++ tests/test_tongs_mounts.py | 30 ++++++++ 5 files changed, 247 insertions(+), 16 deletions(-) diff --git a/swarmforge/anvil/orchestrate.py b/swarmforge/anvil/orchestrate.py index 91a486a..5bee868 100644 --- a/swarmforge/anvil/orchestrate.py +++ b/swarmforge/anvil/orchestrate.py @@ -31,14 +31,14 @@ from .secretchan import SecretChannel, make_secret_resolver -def _mounts_workspace(defn): - """True if a tong's `mounts:` request the session workspace. +def _mounts_word(defn, word): + """True if a tong's `mounts:` request the magic `word`. - The magic word may carry a target and/or a mode (e.g. `workspace:/code:ro`), + The word may carry a name, a target and/or a mode (e.g. `workspace:/code:ro`), so compare only the word before the first colon. """ for mount in defn.get("mounts") or []: - if isinstance(mount, str) and mount.split(":", 1)[0] == tongs.WORKSPACE_MOUNT: + if isinstance(mount, str) and mount.split(":", 1)[0] == word: return True return False @@ -90,7 +90,11 @@ def unsupported_tong_reasons(merged): long-lived container reused across sessions, so binding one session's workspace into it would expose that workspace to every later session that reuses the container (a `session` tong is the right home for a - per-workspace mount). + per-workspace mount); + * a workspace-sourced `shared` tong that mounts a `volume` -- the volume + is scoped to one checkout (see `_volume_scopes`) but the container is + not, so a session in another checkout shipping the same definition would + reuse it, volume and all. A refused tong is reported rather than started half-wired. Returns a list of human-readable reason strings (empty == every discovered tong is startable). @@ -104,15 +108,47 @@ def unsupported_tong_reasons(merged): "tong '%s' has a 'volume' interface, which this launcher does not " "wire up" % name ) - if defn.get("lifecycle") == "shared" and _mounts_workspace(defn): + if defn.get("lifecycle") == "shared" and _mounts_word(defn, tongs.WORKSPACE_MOUNT): reasons.append( "tong '%s' is a 'shared' tong that mounts the workspace; a shared " "container is reused across sessions, so it would leak one " "session's workspace into the next" % name ) + if (defn.get("lifecycle") == "shared" + and tongs.is_workspace_sourced(merged[name]["source"]) + and _mounts_word(defn, tongs.VOLUME_MOUNT)): + reasons.append( + "tong '%s' is a workspace 'shared' tong that mounts a volume; its " + "volume belongs to this checkout but its container is reused by " + "sessions in other checkouts, so it would leak this checkout's " + "data into theirs (make it a 'session' tong)" % name + ) return reasons +def _volume_scopes(merged, org_tongs_dir, workspace): + """The scope naming each tong's volumes: `{name: (class, path) or None}`. + + A volume is partitioned like the trust of the layer its tong came from (see + `tongs.volume_scope`): an org tong's by its org's tongs directory, a + workspace tong's by the checkout's top level, and the user and repo layers' + by this machine. None for a tong that mounts no volume. Raises + `OrchestrationError` for a volume with no directory to scope it by, before + anything starts, rather than letting it fall back to the local scope. + """ + scopes = {} + for name in sorted(merged): + scopes[name] = None + if not _mounts_word(merged[name]["definition"], tongs.VOLUME_MOUNT): + continue + try: + scopes[name] = tongs.volume_scope( + merged[name]["source"], org_tongs_dir=org_tongs_dir, workspace=workspace) + except ValueError as exc: + raise OrchestrationError("tong '%s': %s" % (name, exc)) + return scopes + + def ensure_mcp_harness_supported(merged, harness): """Refuse MCP tongs when no emitter exists for the selected harness.""" mcp_names = [ @@ -134,7 +170,7 @@ def ensure_mcp_harness_supported(merged, harness): def _start_one_tong(docker, name, defn, *, container, network, alias, - resolver, workspace, label_hash, make_channel): + resolver, workspace, label_hash, make_channel, volume_scope): """Start one tong container detached, delivering any secret env over a FIFO. Resolves the definition's secret references through `resolver` and splits the @@ -175,7 +211,7 @@ def _start_one_tong(docker, name, defn, *, container, network, alias, name, defn, container_name=container, network=network, alias=alias, env=plain_env, label_hash=label_hash, workspace=workspace, - extra_mount_specs=git_dir_specs, + extra_mount_specs=git_dir_specs, volume_scope=volume_scope, ) except ValueError as exc: raise OrchestrationError("tong '%s': %s" % (name, exc)) @@ -198,7 +234,7 @@ def _start_one_tong(docker, name, defn, *, container, network, alias, container_name=container, network=network, alias=alias, env=plain_env, label_hash=label_hash, workspace=workspace, secret_channel=True, entrypoint=entrypoint, command=command, - extra_mount_specs=git_dir_specs, + extra_mount_specs=git_dir_specs, volume_scope=volume_scope, ) except ValueError as exc: raise OrchestrationError("tong '%s': %s" % (name, exc)) @@ -212,7 +248,8 @@ def _start_one_tong(docker, name, defn, *, container, network, alias, def _ensure_shared_tong(docker, name, defn, *, container, network, alias, - resolver, workspace, label_hash, make_channel): + resolver, workspace, label_hash, make_channel, + volume_scope): """Start a `shared` tong, or reuse the running one, recreating it if stale. A `shared` tong is one long-lived container keyed by `shared_container_name`. @@ -223,6 +260,8 @@ def _ensure_shared_tong(docker, name, defn, *, container, network, alias, so the same long-lived container is reused across sessions while the definition is stable -- and deciding to reuse one never runs a secret-provider CLI, so a rotated secret behind an unchanged reference does not churn it. + The recreated container mounts the same named volumes, so a tong's + persistent state outlives it. """ state = docker.inspect_state(container) if state and state["running"] and state["label"] == label_hash: @@ -231,7 +270,7 @@ def _ensure_shared_tong(docker, name, defn, *, container, network, alias, docker, name, defn, container=container, network=network, alias=alias, resolver=resolver, workspace=workspace, label_hash=label_hash, - make_channel=make_channel, + make_channel=make_channel, volume_scope=volume_scope, ) @@ -309,11 +348,14 @@ def run_with_tongs(merged, anvil_cmd, opts, *, docker, providers=None, On exit -- including SIGINT -- the `session` tongs and the per-session network are torn down (and the connected `shared` tongs disconnected) while the - long-lived `shared` tongs are left running. + long-lived `shared` tongs are left running. No tong's named volumes are ever + removed: they are the state a tong keeps between containers. Returns the anvil's exit code. Raises `OrchestrationError` if a tong never becomes ready (the anvil does not run against a half-up environment) or a - `session` tong is discovered with no anvil `--name` to key the session by, and + `session` tong is discovered with no anvil `--name` to key the session by, or + a tong mounts a volume with no directory to scope it by (an org tong with no + org tongs directory, a workspace tong with no workspace path), and `SecretResolutionError` if a secret reference cannot be resolved. """ ensure_mcp_harness_supported(merged, opts.harness) @@ -331,7 +373,8 @@ def run_with_tongs(merged, anvil_cmd, opts, *, docker, providers=None, ) # No org layer means no token, and every shared tong keeps its global unscoped name. - org_token = tongs.org_scope_token(dict(opts.layer_dirs).get(tongs.ORG)) + org_tongs_dir = dict(opts.layer_dirs).get(tongs.ORG) + org_token = tongs.org_scope_token(org_tongs_dir) has_org_shared = bool(org_token) and any( merged[name]["definition"].get("lifecycle") != "session" and merged[name]["source"] == tongs.ORG @@ -343,6 +386,8 @@ def run_with_tongs(merged, anvil_cmd, opts, *, docker, providers=None, "join their isolated network" ) + volume_scopes = _volume_scopes(merged, org_tongs_dir, opts.workspace) + plan = tongs.plan_network(merged, base_network, session_id) injection = tongs.plan_injection(merged, opts.harness) @@ -368,6 +413,7 @@ def run_with_tongs(merged, anvil_cmd, opts, *, docker, providers=None, container=container, network=plan["network"], alias=alias, resolver=resolver, workspace=opts.workspace, label_hash=tongs.config_hash(defn), make_channel=make_channel, + volume_scope=volume_scopes[name], ) started_sessions.append(container) probe_network = plan["network"] @@ -388,6 +434,7 @@ def run_with_tongs(merged, anvil_cmd, opts, *, docker, providers=None, container=container, network=tong_network, alias=alias, resolver=resolver, workspace=opts.workspace, label_hash=tongs.config_hash(defn), make_channel=make_channel, + volume_scope=volume_scopes[name], ) ready_checks.append((name, defn, alias, container, probe_network)) diff --git a/swarmforge/tongs/__init__.py b/swarmforge/tongs/__init__.py index bcb69f3..05dc302 100644 --- a/swarmforge/tongs/__init__.py +++ b/swarmforge/tongs/__init__.py @@ -129,6 +129,7 @@ reserved_mount_targets, tong_mount_specs, tong_volume_name, + volume_scope, workspace_mount_placements, ) from .network import SESSION_NET_PREFIX, plan_network, session_network_name @@ -238,6 +239,7 @@ "reserved_mount_targets", "tong_mount_specs", "tong_volume_name", + "volume_scope", "workspace_mount_placements", "SESSION_NET_PREFIX", "plan_network", diff --git a/swarmforge/tongs/mounts.py b/swarmforge/tongs/mounts.py index 13b4605..5b3cafc 100644 --- a/swarmforge/tongs/mounts.py +++ b/swarmforge/tongs/mounts.py @@ -9,12 +9,13 @@ import hashlib import json +import os import posixpath import re -from swarmforge.names import sanitize_token +from swarmforge.names import canonical_path, sanitize_token -from .model import SOCKET_MOUNT +from .model import ORG, REPO, SOCKET_MOUNT, USER, WORKSPACE from .secrets import SECRET_FIFO_DIR, SECRET_INJECT_SHELL, partition_secret_env @@ -272,6 +273,34 @@ def tong_volume_name(tong_name, volume, scope): return "%s_%s" % ("-".join(parts), volume) +def volume_scope(layer, org_tongs_dir=None, workspace=None): + """The `(class, path)` scope that names the volumes of a tong from `layer`. + + An org tong's volumes belong to its org's tongs directory (canonical, as + `org_scope_token` reads it); a workspace tong's to the checkout's top level + `workspace` with symlinks resolved, so every session in that checkout, from + any subdirectory, shares them -- as does any later checkout at that path, + since the path is the key. The user and repo layers are this machine's own + and share `(LOCAL_VOLUME_SCOPE, None)`, so a repo tong and the same-named + user tong it replaces in the merge use one volume. Raises `ValueError` for an org or + workspace tong with no directory to scope it by, rather than falling back to + the local scope, and for an unknown layer. + """ + if layer == ORG: + if not org_tongs_dir: + raise ValueError("an org tong's volume is scoped to its org, but no org " + "tongs directory is known") + return (ORG_VOLUME_SCOPE, canonical_path(org_tongs_dir)) + if layer == WORKSPACE: + if not workspace: + raise ValueError("a workspace tong's volume is scoped to its checkout, but " + "no workspace path is known") + return (WORKSPACE_VOLUME_SCOPE, os.path.realpath(workspace)) + if layer in (USER, REPO): + return (LOCAL_VOLUME_SCOPE, None) + raise ValueError("unknown layer %r has no volume scope" % (layer,)) + + def tong_mount_specs(defn, workspace, socket_path=DEFAULT_DOCKER_SOCKET, tong_name=None, volume_scope=None): """Concrete docker `-v` specs for a tong's `mounts:` magic words. diff --git a/tests/test_anvil_orchestrate.py b/tests/test_anvil_orchestrate.py index 20d627e..83517e4 100644 --- a/tests/test_anvil_orchestrate.py +++ b/tests/test_anvil_orchestrate.py @@ -104,6 +104,30 @@ def test_shared_workspace_refused_with_a_custom_mount_target(self): }) )) + def test_workspace_shared_tong_with_a_volume_refused(self): + # Its container is shared by every checkout while its volume belongs to one. + defn = { + "lifecycle": "shared", "image": "x", "mounts": ["volume:v:/data"], + "interface": {"kind": "none"}, "readiness": {"mode": "none"}, + } + reasons = launcher.unsupported_tong_reasons(_merged("t", defn)) + self.assertTrue( + any("workspace 'shared' tong that mounts a volume" in r for r in reasons), reasons + ) + + def test_volume_allowed_where_container_and_volume_scope_agree(self): + shared = { + "lifecycle": "shared", "image": "x", "mounts": ["volume:v:/data"], + "interface": {"kind": "none"}, "readiness": {"mode": "none"}, + } + for source in (tongs.USER, tongs.ORG, tongs.REPO): + self.assertEqual( + launcher.unsupported_tong_reasons(_merged("t", shared, source=source)), + [], source, + ) + session = dict(shared, lifecycle="session") + self.assertEqual(launcher.unsupported_tong_reasons(_merged("t", session)), []) + def test_workspace_refusal_is_shared_scoped(self): # A `session` tong is torn down with the anvil, so it cannot leak. self.assertEqual( @@ -859,6 +883,105 @@ def test_org_shared_tong_without_anvil_name_raises_before_any_docker_call(self): self.assertEqual(docker.calls, []) +SESSION_VOLUME = { + "lifecycle": "session", + "image": "model-cache", + "mounts": ["volume:models:/models"], + "interface": {"kind": "none"}, + "readiness": {"mode": "none"}, +} + + +def _mounted(argv): + return [argv[i + 1] for i, part in enumerate(argv) if part == "-v"] + + +class VolumeScopeTests(unittest.TestCase): + """A tong's named volumes are scoped by the layer it came from, and kept.""" + + _ACME = "/orgs/acme/.swarmforge/tongs" + + def _run(self, docker, merged, workspace=None, org_dir=None, anvil=None): + layer_dirs = [(tongs.ORG, org_dir)] if org_dir else [] + opts = launcher.LauncherOptions( + layer_dirs=layer_dirs, workspace=workspace, approvals=None, + providers=None, harness="opencode", anvil_image="anvil:img", no_prompt=False, + ) + return launcher.run_with_tongs( + merged, anvil or ANVIL_ARGV, opts, + docker=docker, sleep=lambda _s: None, monotonic=_Clock(), + ) + + def test_user_and_repo_tong_volumes_share_the_local_scope(self): + local = [tongs.tong_volume_name("cache", "models", (tongs.LOCAL_VOLUME_SCOPE, None)) + + ":/models"] + for source in (tongs.USER, tongs.REPO): + docker = FakeDocker() + self._run(docker, _merged("cache", SESSION_VOLUME, source=source), + workspace="/home/me/proj") + self.assertEqual(_mounted(docker.run_argvs[0]), local, source) + + def test_org_tong_volume_scoped_by_its_org_on_either_lifecycle(self): + org = [tongs.tong_volume_name("cache", "models", (tongs.ORG_VOLUME_SCOPE, self._ACME)) + + ":/models"] + shared = dict(SESSION_VOLUME, lifecycle="shared") + for defn in (SESSION_VOLUME, shared): + docker = FakeDocker() + self._run(docker, _merged("cache", defn, source=tongs.ORG), org_dir=self._ACME) + self.assertEqual(_mounted(docker.run_argvs[0]), org, defn["lifecycle"]) + + def test_workspace_tong_volume_scoped_by_its_checkout(self): + mounted = [] + for workspace in ("/home/me/proj", "/home/me/fork"): + docker = FakeDocker() + self._run(docker, _merged("cache", SESSION_VOLUME), workspace=workspace) + mounted.append(_mounted(docker.run_argvs[0])) + scope = (tongs.WORKSPACE_VOLUME_SCOPE, os.path.realpath("/home/me/proj")) + self.assertEqual(mounted[0], [tongs.tong_volume_name("cache", "models", scope) + + ":/models"]) + self.assertNotEqual(mounted[0], mounted[1]) + + def test_workspace_tong_volume_without_a_workspace_path_starts_nothing(self): + docker = FakeDocker() + with self.assertRaisesRegex(launcher.OrchestrationError, "tong 'cache'.*workspace"): + self._run(docker, _merged("cache", SESSION_VOLUME)) + self.assertEqual(docker.calls, []) + + def test_org_tong_volume_without_an_org_directory_starts_nothing(self): + docker = FakeDocker() + with self.assertRaisesRegex(launcher.OrchestrationError, "tong 'cache'.*org"): + self._run(docker, _merged("cache", SESSION_VOLUME, source=tongs.ORG)) + self.assertEqual(docker.calls, []) + + def test_workspace_tong_without_a_volume_needs_no_workspace_path(self): + docker = FakeDocker() + self.assertEqual(self._run(docker, _merged("pg", SESSION_PORT)), 0) + + def test_recreated_shared_tong_mounts_the_same_volume(self): + # A changed definition recreates the container, never the volume. + defn = dict(SESSION_VOLUME, lifecycle="shared") + states = {"swarmforge-shared-cache": {"running": True, "label": "stale"}} + docker = FakeDocker(states=states) + self._run(docker, _merged("cache", defn, source=tongs.REPO)) + self.assertIn(("rm_force", "swarmforge-shared-cache"), docker.calls) + self.assertEqual( + _mounted(docker.run_argvs[0]), + [tongs.tong_volume_name("cache", "models", (tongs.LOCAL_VOLUME_SCOPE, None)) + + ":/models"], + ) + + def test_teardown_only_force_removes_the_container_and_network(self): + # No volume step: a named volume outlives `docker rm` of its container. + docker = FakeDocker() + self._run(docker, _merged("cache", SESSION_VOLUME, source=tongs.REPO)) + teardown = docker.calls[[c[0] for c in docker.calls].index("run_foreground_multi") + 1:] + self.assertEqual(teardown, [ + ("rm_force", "claude-myproject-tong-cache"), + ("rm_force", "claude-myproject"), + ("network_rm", tongs.session_network_name("claude-myproject")), + ]) + + class WorkspaceGitDirSpecTests(unittest.TestCase): """_workspace_git_dir_specs: the git-dir mounts riding along with `workspace`.""" diff --git a/tests/test_tongs_mounts.py b/tests/test_tongs_mounts.py index e9d11dd..69cd2e8 100644 --- a/tests/test_tongs_mounts.py +++ b/tests/test_tongs_mounts.py @@ -3,6 +3,7 @@ import os import sys +import tempfile import unittest REPO_ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) @@ -186,6 +187,35 @@ def test_tong_volume_name_differs_per_scope(self): } self.assertEqual(len(names), 5) + def test_volume_scope_follows_the_layer(self): + self.assertEqual(tongs.volume_scope(tongs.USER), LOCAL) + self.assertEqual(tongs.volume_scope(tongs.REPO, "/o/tongs", "/ws"), LOCAL) + self.assertEqual( + tongs.volume_scope(tongs.ORG, org_tongs_dir="/orgs//acme/./tongs/"), + (tongs.ORG_VOLUME_SCOPE, "/orgs/acme/tongs"), + ) + with self.assertRaisesRegex(ValueError, "org tongs directory"): + tongs.volume_scope(tongs.ORG, workspace="/ws") + with self.assertRaisesRegex(ValueError, "workspace path"): + tongs.volume_scope(tongs.WORKSPACE, org_tongs_dir="/o/tongs") + with self.assertRaisesRegex(ValueError, "unknown layer"): + tongs.volume_scope("elsewhere") + + def test_volume_scope_resolves_a_symlinked_checkout(self): + with tempfile.TemporaryDirectory() as tmp: + checkout = os.path.join(tmp, "checkout") + os.mkdir(checkout) + link = os.path.join(tmp, "link") + os.symlink(checkout, link) + self.assertEqual( + tongs.volume_scope(tongs.WORKSPACE, workspace=link), + tongs.volume_scope(tongs.WORKSPACE, workspace=checkout), + ) + self.assertEqual( + tongs.volume_scope(tongs.WORKSPACE, workspace=link)[1], + os.path.realpath(checkout), + ) + def test_mount_target_error_names_the_mount_and_the_reason(self): reserved = {"/run/x": "where the launcher delivers this tong's secrets"} From ab8f8cf6b5548b99aaa2465fafe88350ccf24a7b Mon Sep 17 00:00:00 2001 From: CrypticSwarm Date: Sun, 4 Oct 2026 11:31:47 -0500 Subject: [PATCH 03/14] Call out tong volumes in the approval prompt A workspace tong that mounts a volume leaves data on the host's docker after the tong is removed. The prompt already lists the raw `volume:...` mount, but nothing in that line says the data persists. The prompt now adds a `volumes:` line that names each volume and says it is kept. The prompt does not show the resolved docker volume name. The launcher derives that name from the tong's name and the checkout, so the definition cannot choose it, and it would not help the reviewer decide whether to approve. Leaving it out also keeps `privilege_summary` a function of the definition alone. The docs explain how volumes are named, for anyone who wants to find or remove one. The gate runs before validation, so a malformed volume entry is left off the `volumes:` line rather than raising. It still appears among the raw mounts, and validation refuses it before anything starts. --- swarmforge/anvil/approval.py | 5 +++++ swarmforge/tongs/approvals.py | 30 ++++++++++++++++++++++++++---- tests/test_anvil_approval.py | 9 +++++++++ tests/test_tongs_approvals.py | 14 ++++++++++++++ 4 files changed, 54 insertions(+), 4 deletions(-) diff --git a/swarmforge/anvil/approval.py b/swarmforge/anvil/approval.py index 164bec2..a5908e4 100644 --- a/swarmforge/anvil/approval.py +++ b/swarmforge/anvil/approval.py @@ -41,6 +41,11 @@ def render_privilege_summary(name, summary): mounts = summary.get("mounts") or [] if mounts: lines.append(" mounts: %s" % ", ".join(str(m) for m in mounts)) + volumes = summary.get("volumes") or [] + if volumes: + lines.append(" volumes: %s (persistent: outlives the tong and reuses any data " + "left by an earlier definition or checkout at this path)" + % ", ".join(volumes)) networks = summary.get("networks") or [] if networks: lines.append(" networks: %s" % ", ".join(str(n) for n in networks)) diff --git a/swarmforge/tongs/approvals.py b/swarmforge/tongs/approvals.py index 37d9a4d..35c61c1 100644 --- a/swarmforge/tongs/approvals.py +++ b/swarmforge/tongs/approvals.py @@ -12,7 +12,7 @@ import os from .model import WORKSPACE -from .mounts import _has_socket_mount +from .mounts import _has_socket_mount, mount_volume_name from .secrets import find_secret_refs @@ -32,14 +32,16 @@ def privilege_summary(defn): """Structured summary of what a definition asks for, for the approval gate. Gathers the privileges a reviewer must see before approving a - workspace-sourced tong: image, secret references, mounts, networks, - docker-socket access, and host GPU access. Rendering and prompting are the - caller's job; this just assembles the facts. + workspace-sourced tong: image, secret references, mounts (and the + persistent volumes among them), networks, docker-socket access, and host + GPU access. Rendering and prompting are the caller's job; this just + assembles the facts. """ return { "image": defn.get("image"), "secrets": [{"provider": p, "ref": r} for p, r in find_secret_refs(defn)], "mounts": list(defn.get("mounts") or []), + "volumes": _declared_volumes(defn), "networks": list(defn.get("networks") or []), "socket": _has_socket_mount(defn), "gpus": _requested_gpus(defn), @@ -56,6 +58,26 @@ def _requested_gpus(defn): return resources.get("gpus") if isinstance(resources, dict) else None +def _declared_volumes(defn): + """Names of the volumes a definition's well-formed `volume:` mounts declare. + + The gate runs before validation, so a malformed entry is skipped here rather + than raised: it is still listed among the raw mounts, and validation refuses + it before anything starts. + """ + volumes = [] + for mount in defn.get("mounts") or []: + if not isinstance(mount, str): + continue + try: + volume = mount_volume_name(mount) + except ValueError: + continue + if volume: + volumes.append(volume) + return volumes + + def is_workspace_sourced(source_layer): """True if a tong's winning layer is the (untrusted) workspace and so gates.""" return source_layer == WORKSPACE diff --git a/tests/test_anvil_approval.py b/tests/test_anvil_approval.py index ba72be9..7fe84f6 100644 --- a/tests/test_anvil_approval.py +++ b/tests/test_anvil_approval.py @@ -58,10 +58,19 @@ def test_non_mapping_resources_renders_no_gpus_line(self): text = launcher.render_privilege_summary("github", tongs.privilege_summary(defn)) self.assertNotIn("gpus", text) + def test_calls_out_a_volume_as_persistent(self): + defn = dict(WORKSPACE_TONG, mounts=["volume:models:/models"]) + text = launcher.render_privilege_summary("cache", tongs.privilege_summary(defn)) + self.assertIn("volume:models:/models", text) + self.assertIn( + " volumes: models (persistent: outlives the tong and reuses any data left by " + "an earlier definition or checkout at this path)", text.splitlines()) + def test_omits_unrequested_sections(self): defn = {"image": "x", "interface": {"kind": "none"}, "resources": {"memory": "1g"}} text = launcher.render_privilege_summary("x", tongs.privilege_summary(defn)) self.assertNotIn("secrets:", text) + self.assertNotIn("volumes:", text) self.assertNotIn("docker socket", text) self.assertNotIn("gpus", text) diff --git a/tests/test_tongs_approvals.py b/tests/test_tongs_approvals.py index c109c89..68ef464 100644 --- a/tests/test_tongs_approvals.py +++ b/tests/test_tongs_approvals.py @@ -73,6 +73,20 @@ def test_summary_reports_an_unvalidated_gpus_value_as_declared(self): defn["resources"] = {"gpus": 0} self.assertEqual(tongs.privilege_summary(defn)["gpus"], 0) + def test_summary_names_each_declared_volume(self): + defn = def_of(GITHUB_TONG) + defn["mounts"] = ["workspace:ro", "volume:models:/a", "volume:cache:/c:ro"] + self.assertEqual(tongs.privilege_summary(defn)["volumes"], ["models", "cache"]) + self.assertEqual(tongs.privilege_summary(def_of(GITHUB_TONG))["volumes"], []) + + def test_summary_tolerates_malformed_mounts(self): + # The gate runs before validation, which refuses these on its own. + defn = def_of(GITHUB_TONG) + defn["mounts"] = ["volume:bad_name:/a", "volume:v", 7, "/etc:/etc", "volume:ok:/b"] + summary = tongs.privilege_summary(defn) + self.assertEqual(summary["volumes"], ["ok"]) + self.assertEqual(summary["mounts"], defn["mounts"]) + class ApprovalKeyingTests(unittest.TestCase): def setUp(self): From 7c53f343c462365be974d673e2019ce12f59bb27 Mon Sep 17 00:00:00 2001 From: CrypticSwarm Date: Sun, 4 Oct 2026 11:32:34 -0500 Subject: [PATCH 04/14] Document persistent volumes for tongs The definition reference covers the `volume::/target[:mode]` mount. It describes how the docker volume is named and scoped per layer, explains that the launcher never removes a volume and how to remove one with `docker volume rm`, and says why the mount is unrelated to the not-yet-wired `interface.kind: volume`. The lifecycle notes now tell `shared` tong authors where to keep state that must survive a recreation, and the approval page mentions the new `volumes:` line. --- docs/tongs/README.md | 2 +- docs/tongs/approval.md | 2 +- docs/tongs/definitions.md | 16 ++++++++++++++-- 3 files changed, 16 insertions(+), 4 deletions(-) diff --git a/docs/tongs/README.md b/docs/tongs/README.md index 990ec9c..3d27616 100644 --- a/docs/tongs/README.md +++ b/docs/tongs/README.md @@ -30,7 +30,7 @@ The workspace layer is gated by [first-run approval](approval.md). ## Lifecycle - **`session`** — started with the anvil, torn down when it exits. Per-session isolation; the default for credential tongs. -- **`shared`** — long-lived, survives across anvil sessions (ollama-style). Started on first use, connected to each session's network via a network alias, and left running on teardown (no refcounting). A running `shared` container whose config-hash docker label still matches the current definition is reused untouched; a missing, stopped, or stale one is recreated automatically. A rotated secret behind an unchanged reference does **not** churn it — force a restart with `docker rm -f `. A `shared` tong may not mount the `workspace` (it would leak one session's workspace into the next); use a `session` tong for per-workspace mounts. +- **`shared`** — long-lived, survives across anvil sessions (ollama-style). Started on first use, connected to each session's network via a network alias, and left running on teardown (no refcounting). A running `shared` container whose config-hash docker label still matches the current definition is reused untouched; a missing, stopped, or stale one is recreated automatically. A rotated secret behind an unchanged reference does **not** churn it — force a restart with `docker rm -f `. Recreating the container loses everything the tong wrote inside it, so a tong that has state to keep, such as a model store, declares a [`volume:` mount](definitions.md#mounts). Its named volume is mounted again by the next container and is never removed by the launcher (a workspace-sourced `shared` tong may not mount one; see [Volume naming and scope](definitions.md#volume-naming-and-scope)). A `shared` tong may not mount the `workspace` (it would leak one session's workspace into the next); use a `session` tong for per-workspace mounts. ## Reference diff --git a/docs/tongs/approval.md b/docs/tongs/approval.md index be7de8f..1cca4b8 100644 --- a/docs/tongs/approval.md +++ b/docs/tongs/approval.md @@ -3,7 +3,7 @@ The user, org, and repo layers are installed deliberately and are trusted; they skip the gate. A **workspace**-sourced tong (from a repo you cloned) could otherwise request your secrets, host mounts, the docker socket, or the host's GPUs simply by being present, so the launcher gates it: -- Before starting, it prints exactly what the tong requests — image, secret references, mounts, networks, docker-socket access, and GPU access — and asks you to approve. Control and formatting characters in those values are shown as escapes such as `\x1b` or `\n`, so a value cannot move the cursor, restyle, or hide any part of the summary. +- Before starting, it prints exactly what the tong requests — image, secret references, mounts, networks, docker-socket access, and GPU access — and asks you to approve. Control and formatting characters in those values are shown as escapes such as `\x1b` or `\n`, so a value cannot move the cursor, restyle, or hide any part of the summary. A `volume:` mount is also listed by name on a `volumes:` line, which warns that its data outlives the tong and that any data already in it, left by an earlier definition or an earlier checkout at this path, is reused. - Approval is keyed by workspace path + tong name + a hash of the merged definition, stored in `~/.swarmforge/approvals.json`. Any change to the definition re-prompts. - The gate defaults to **No**, and a non-interactive stdin reads as No. A scripted `--no-prompt` run **fails closed** rather than auto-approving. - Approving `image: foo:latest` approves a moving target; **pinned digests are the recommended convention** for workspace tongs. diff --git a/docs/tongs/definitions.md b/docs/tongs/definitions.md index 5d4687c..d9a852e 100644 --- a/docs/tongs/definitions.md +++ b/docs/tongs/definitions.md @@ -16,6 +16,7 @@ interface: # required; how (or whether) the anvil reaches the # aliases: [gh, git.example] # optional extra DNS names the tong also answers to mounts: # opt-in magic words only, never raw host paths - workspace:ro # or workspace:/code:ro to bind it somewhere else +# - volume:cache:/var/cache # a persistent named volume, kept between containers resources: memory: 512m # string or number # gpus: all # optional host GPU access (see Resources) @@ -33,7 +34,7 @@ The `interface:` block drives what gets injected into the anvil, how readiness i - **`mcp`** — an HTTP MCP server (the common case). Requires `port` and `name`; `transport` defaults to `http`. Injects per-harness MCP config pointing at `http://:/mcp` on the session network (`interface.path` overrides the `/mcp` suffix). TCP readiness probe by default. - **`port`** — a non-MCP network service. Requires `port`; optional `protocol` is informational. Injects `SWARMFORGE_TONG__HOST` (the canonical alias) and `SWARMFORGE_TONG__PORT` so the anvil composes its own connection string. TCP readiness probe by default. -- **`volume`** — a shared named volume, no network. Requires `volume` and `mountpoint`; readiness must be declared. The schema accepts it, but **the launcher does not wire it up yet** and refuses to start such a tong with a clear message. +- **`volume`** — a shared named volume, no network. Requires `volume` and `mountpoint`; readiness must be declared. The schema accepts it, but **the launcher does not wire it up yet** and refuses to start such a tong with a clear message. (A tong that only needs to keep its own state uses a `volume:` mount, described under Mounts below.) - **`none`** — a background side-effect with no anvil-facing surface. Injects nothing. Readiness must be declared. `` is the filename uppercased with hyphens turned into underscores (`github-creds` → `SWARMFORGE_TONG_GITHUB_CREDS_*`). @@ -65,10 +66,21 @@ readiness: ## Mounts -Mounts are opt-in **magic words**, never raw host paths. Only two are recognized, spelled `[:/target][:mode]` with the access mode (`ro`/`rw`) last: +Mounts are opt-in **magic words**, never raw host paths. Only three are recognized, each spelled as shown below with the optional access mode (`ro`/`rw`) last: - `workspace[:/target][:mode]` — bind-mounts the session workspace, at `/workspace` unless an absolute `target` says otherwise (e.g. `workspace:ro`, `workspace:/code`, `workspace:/code:ro`). A custom target lets an image that expects its sources elsewhere be used unmodified (it does not set the working directory — the process still starts in the image's own `WORKDIR`). A target is refused unless it is an absolute path free of whitespace, and refused if it resolves to `/`, overlaps another of the tong's mounts, or overlaps a path the tong's own wiring occupies — the secret-delivery tmpfs at `/run/swarmforge` and the `/bin/sh` its wrapper execs (for a tong with secret references), or the docker socket (for a tong that mounts it). The workspace bind is paired with the same git-dir mounts the anvil gets from `swarmforge/gitguard.py`: read-only guards over the config and hooks the host's git obeys, and — when the workspace is a linked worktree or another checkout whose git dir lives outside it — that git dir at its own absolute path, which is where the checkout's `.git` pointer file says to look (without it, git inside the tong fails with "not a git repository"). The one mount the anvil gets and a tong does not is the restated worktree registration — a tong keeps the host's copy, so `git worktree list` inside one names the checkout by its host path (the [git guard](../git-guard.md) has the rest). When every `workspace` mount is `ro`, the ride-along git-dir mounts are forced read-only too. - `docker-socket[:mode]` — bind-mounts the host docker socket onto the same path inside the container (so it takes no target). This is full host docker control and is always called out explicitly in the workspace approval prompt; it is the grant a broker tong needs. +- `volume::/target[:mode]` — mounts a docker **named volume** at `target`, for state the tong keeps between containers (a model store, a cache). The name comes first and is at most 64 letters, digits, `.` and `-`, starting with a letter or digit; the target is required and held to the same rules as a `workspace` target, and a tong mounts each volume only once. Docker creates the volume on first use and seeds a new, empty one with whatever the image has at `target`. The launcher never removes a volume — not on teardown, and not when a `shared` tong is recreated because its definition changed — so the state outlives every container that mounts it. Remove one by hand with `docker volume rm ` (`docker volume ls --filter name=swarmforge-volume-` lists them). A `volume:` mount is private to its tong; it is unrelated to `interface.kind: volume`, which would share a volume with the anvil and is not wired up yet. + +### Volume naming and scope + +The docker volume is named `swarmforge-volume-[-]_`. The hint is the tong's filename reduced to the characters docker allows and cut to 32 of them, and is there only to be read in `docker volume ls`. The digest is the volume's identity: the first 32 hex digits of a SHA-256 over the tong's exact filename and the scope it came from. A volume name never contains `_` and the digest has a fixed width, so the last `_` always separates the volume name. The scope depends on the tong's layer: + +- **org** — the org's tongs directory. This applies to both lifecycles. +- **workspace** — the checkout's top-level path, with symlinks resolved. The path is the key: every session started anywhere in that checkout uses the volume, and a clone or worktree at a different path gets its own. A later checkout at the same path inherits it, even if it is a different repository, just as it inherits the approvals keyed by that path. Deleting a checkout leaves its volumes behind; remove them with `docker volume rm`. A workspace tong that mounts a volume may not be `shared`: its container would be reused by sessions in other checkouts, still mounting this checkout's volume. +- **user** and **repo** — this machine. A repo tong and the same-named user tong it replaces use one volume. + +Two tongs therefore share a volume only when they have the same filename and the same scope. A tong never gets the volume of another org, a checkout at another path, or a differently named tong, and at 128 bits a definition cannot search its way onto another scope's name. Within its scope a volume belongs to the tong's name, not to one version of its definition: an edited definition, or a later checkout at the same path, reuses whatever data is already there. ## Resources From 28dc43f360653b8f9ef6d305d275252e80c75413 Mon Sep 17 00:00:00 2001 From: CrypticSwarm Date: Sun, 4 Oct 2026 12:01:15 -0500 Subject: [PATCH 05/14] Show a GPU model server tong in the definition docs With GPUs and volumes both available, an ollama-style model server can run as a tong: a `shared` `port` tong with `resources.gpus` and its model store in a `volume:` mount, so recreating the container does not download the models again. The definition reference now shows one complete, minimal example. The example tong is named `models` rather than `ollama`. `make run_ollama` already answers to `ollama` on the same network, so a tong by that name would collide with it. --- docs/tongs/definitions.md | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/docs/tongs/definitions.md b/docs/tongs/definitions.md index d9a852e..5fa65ee 100644 --- a/docs/tongs/definitions.md +++ b/docs/tongs/definitions.md @@ -99,3 +99,22 @@ Both are optional and passed to `docker run` as written. `gpus` grants the tong Docker's other `--gpus` keys are deliberately refused. `driver=`, `capabilities=`, and `options=` can grant more than GPU access — `driver=cdi`, for one, injects any device registered on the host — and `count=` is redundant with a plain count. GPU access needs a docker host set up for GPU containers — the NVIDIA Container Toolkit on Linux; Docker Desktop's WSL 2 backend provides GPU support itself. Without it `docker run` fails and the launch stops. A workspace tong's GPU request is called out in its [approval prompt](approval.md). + +## Example: a GPU model server + +A `shared` `port` tong that serves models from the host's GPUs and keeps its multi-GB model store in a volume, so a recreated container does not download it again: + +```yaml +# ~/.swarmforge/tongs/models.yaml +lifecycle: shared +image: ollama/ollama@sha256:... +interface: + kind: port + port: 11434 +mounts: + - volume:store:/root/.ollama +resources: + gpus: all +``` + +The anvil gets `SWARMFORGE_TONG_MODELS_HOST=models` and `SWARMFORGE_TONG_MODELS_PORT=11434`, and the store lives in the volume `swarmforge-volume-models-_store`. The tong is named `models` rather than `ollama` because `make run_ollama` already answers to `ollama` on the same network; a tong of that name would clash with it. From 2395ce2dcfa5120fb2cc72bdc74c71166659c88e Mon Sep 17 00:00:00 2001 From: CrypticSwarm Date: Thu, 8 Oct 2026 23:05:53 -0500 Subject: [PATCH 06/14] Escape every tongs warning, not one call site A workspace tong's filename reached stderr raw through the merge, refusal and readiness warnings; only the skipped-file warning was escaped. Escaping inside `tongs.warn` covers them all, and the git-dir and `tongs validate` messages now route through the same escape. --- docs/tongs/approval.md | 2 +- swarmforge/anvil/orchestrate.py | 4 +--- swarmforge/tongs/cli.py | 4 ++-- swarmforge/tongs/discovery.py | 4 ++-- swarmforge/tongs/model.py | 7 ++++--- tests/test_tongs_model.py | 8 ++++++++ 6 files changed, 18 insertions(+), 11 deletions(-) diff --git a/docs/tongs/approval.md b/docs/tongs/approval.md index 1cca4b8..cd3c07d 100644 --- a/docs/tongs/approval.md +++ b/docs/tongs/approval.md @@ -3,7 +3,7 @@ The user, org, and repo layers are installed deliberately and are trusted; they skip the gate. A **workspace**-sourced tong (from a repo you cloned) could otherwise request your secrets, host mounts, the docker socket, or the host's GPUs simply by being present, so the launcher gates it: -- Before starting, it prints exactly what the tong requests — image, secret references, mounts, networks, docker-socket access, and GPU access — and asks you to approve. Control and formatting characters in those values are shown as escapes such as `\x1b` or `\n`, so a value cannot move the cursor, restyle, or hide any part of the summary. A `volume:` mount is also listed by name on a `volumes:` line, which warns that its data outlives the tong and that any data already in it, left by an earlier definition or an earlier checkout at this path, is reused. +- Before starting, it prints exactly what the tong requests — image, secret references, mounts, networks, docker-socket access, and GPU access — and asks you to approve. Control and formatting characters in those values are shown as escapes such as `\x1b` or `\n`, so a value cannot move the cursor, restyle, or hide any part of the summary. Every `tongs:` warning escapes them the same way, since many quote a tong's filename. A `volume:` mount is also listed by name on a `volumes:` line, which warns that its data outlives the tong and that any data already in it, left by an earlier definition or an earlier checkout at this path, is reused. - Approval is keyed by workspace path + tong name + a hash of the merged definition, stored in `~/.swarmforge/approvals.json`. Any change to the definition re-prompts. - The gate defaults to **No**, and a non-interactive stdin reads as No. A scripted `--no-prompt` run **fails closed** rather than auto-approving. - Approving `image: foo:latest` approves a moving target; **pinned digests are the recommended convention** for workspace tongs. diff --git a/swarmforge/anvil/orchestrate.py b/swarmforge/anvil/orchestrate.py index 54fcee9..a002a1b 100644 --- a/swarmforge/anvil/orchestrate.py +++ b/swarmforge/anvil/orchestrate.py @@ -18,7 +18,6 @@ import json import os import shutil -import sys import tempfile import time @@ -201,8 +200,7 @@ def _start_one_tong(docker, name, defn, *, container, network, alias, try: git_dir_specs = _workspace_git_dir_specs( defn, workspace, - warn=lambda message: print("tong '%s': %s" % (name, message), - file=sys.stderr)) + warn=lambda message: tongs.warn("tong '%s': %s" % (name, message))) except ValueError as exc: raise OrchestrationError("tong '%s': %s" % (name, exc)) diff --git a/swarmforge/tongs/cli.py b/swarmforge/tongs/cli.py index 22c7945..e231b32 100644 --- a/swarmforge/tongs/cli.py +++ b/swarmforge/tongs/cli.py @@ -8,7 +8,7 @@ import sys from .discovery import discover, merge_tongs -from .model import LAYERS +from .model import LAYERS, printable from .validate import validate_tong @@ -31,7 +31,7 @@ def main(argv): problems = 0 for name in sorted(merged): for error in validate_tong(name, merged[name]["definition"]): - print(error) + print(printable(error)) problems += 1 if not problems: print("ok: %d tong(s) valid" % len(merged)) diff --git a/swarmforge/tongs/discovery.py b/swarmforge/tongs/discovery.py index f9dd1be..6d27d21 100644 --- a/swarmforge/tongs/discovery.py +++ b/swarmforge/tongs/discovery.py @@ -11,7 +11,7 @@ from swarmforge.yamlite import parse_map -from .model import TRUSTED_LAYERS, WORKSPACE, printable, warn +from .model import TRUSTED_LAYERS, WORKSPACE, warn def load_yaml(text): @@ -48,7 +48,7 @@ def load_tong_dir(path): try: out[name] = load_tong_file(full) except (ValueError, OSError) as exc: - warn(printable("skipping %s: %s" % (full, exc))) + warn("skipping %s: %s" % (full, exc)) return out diff --git a/swarmforge/tongs/model.py b/swarmforge/tongs/model.py index 7cd4f5c..5a85eb0 100644 --- a/swarmforge/tongs/model.py +++ b/swarmforge/tongs/model.py @@ -5,8 +5,8 @@ dispatches on (layers, lifecycles, interface kinds, readiness modes), the docker labels it stamps, and the pure resolution of a definition's `readiness:` block and `resources.gpus` request. -`warn` lives here too, so the one `tongs: ` stderr prefix has a single home, with -`printable` beside it for echoing definition-sourced text to the terminal. +`warn` lives here too, so the one `tongs: ` stderr prefix has a single home, and +every message passes through `printable` beside it. """ import re @@ -36,7 +36,8 @@ def warn(message): - print("tongs: %s" % message, file=sys.stderr) + """Print `message` to stderr through `printable`, since it may quote definition-sourced text.""" + print("tongs: %s" % printable(message), file=sys.stderr) def printable(text): diff --git a/tests/test_tongs_model.py b/tests/test_tongs_model.py index fe73ca8..2dbdfc1 100644 --- a/tests/test_tongs_model.py +++ b/tests/test_tongs_model.py @@ -1,6 +1,8 @@ #!/usr/bin/env python3 """Unit tests for swarmforge.tongs.model. Run: python3 tests/test_tongs_model.py""" +import contextlib +import io import os import sys import unittest @@ -36,6 +38,12 @@ def test_output_is_printable_for_every_code_point(self): every = "".join(chr(cp) for cp in range(0x110000)) self.assertTrue(tongs.printable(every).isprintable()) + def test_warn_escapes_its_message(self): + err = io.StringIO() + with contextlib.redirect_stderr(err): + tongs.warn("tong 'x\x1b[2K\napproved' is refused") + self.assertEqual(err.getvalue(), "tongs: tong 'x\\x1b[2K\\napproved' is refused\n") + class ReadinessTests(unittest.TestCase): def test_parse_duration_units(self): From bf2ea349e935cbb66390366ebed485c2e612610a Mon Sep 17 00:00:00 2001 From: CrypticSwarm Date: Thu, 8 Oct 2026 23:06:44 -0500 Subject: [PATCH 07/14] Share one mounts-word check across the tongs layers `_has_socket_mount` was `_mounts_word` with the word fixed. Move the general check down to `tongs.mounts` and have the socket, workspace and volume checks all use it. --- swarmforge/anvil/orchestrate.py | 18 +++--------------- swarmforge/tongs/__init__.py | 2 ++ swarmforge/tongs/approvals.py | 6 +++--- swarmforge/tongs/argv.py | 12 +++++++++--- swarmforge/tongs/mounts.py | 11 ++++++++--- tests/test_tongs_mounts.py | 7 +++++++ 6 files changed, 32 insertions(+), 24 deletions(-) diff --git a/swarmforge/anvil/orchestrate.py b/swarmforge/anvil/orchestrate.py index a002a1b..92ea54f 100644 --- a/swarmforge/anvil/orchestrate.py +++ b/swarmforge/anvil/orchestrate.py @@ -31,18 +31,6 @@ from .secretchan import SecretChannel, make_secret_resolver -def _mounts_word(defn, word): - """True if a tong's `mounts:` request the magic `word`. - - The word may carry a name, a target and/or a mode (e.g. `workspace:/code:ro`), - so compare only the word before the first colon. - """ - for mount in defn.get("mounts") or []: - if isinstance(mount, str) and mount.split(":", 1)[0] == word: - return True - return False - - def _workspace_git_dir_specs(defn, workspace, warn=None): """Git-dir mounts a workspace-mounting tong needs beyond the workspace bind. @@ -108,7 +96,7 @@ def unsupported_tong_reasons(merged): "tong '%s' has a 'volume' interface, which this launcher does not " "wire up" % name ) - if defn.get("lifecycle") == "shared" and _mounts_word(defn, tongs.WORKSPACE_MOUNT): + if defn.get("lifecycle") == "shared" and tongs.mounts_word(defn, tongs.WORKSPACE_MOUNT): reasons.append( "tong '%s' is a 'shared' tong that mounts the workspace; a shared " "container is reused across sessions, so it would leak one " @@ -116,7 +104,7 @@ def unsupported_tong_reasons(merged): ) if (defn.get("lifecycle") == "shared" and tongs.is_workspace_sourced(merged[name]["source"]) - and _mounts_word(defn, tongs.VOLUME_MOUNT)): + and tongs.mounts_word(defn, tongs.VOLUME_MOUNT)): reasons.append( "tong '%s' is a workspace 'shared' tong that mounts a volume; its " "volume belongs to this checkout but its container is reused by " @@ -139,7 +127,7 @@ def _volume_scopes(merged, org_tongs_dir, workspace): scopes = {} for name in sorted(merged): scopes[name] = None - if not _mounts_word(merged[name]["definition"], tongs.VOLUME_MOUNT): + if not tongs.mounts_word(merged[name]["definition"], tongs.VOLUME_MOUNT): continue try: scopes[name] = tongs.volume_scope( diff --git a/swarmforge/tongs/__init__.py b/swarmforge/tongs/__init__.py index 05dc302..8c2603f 100644 --- a/swarmforge/tongs/__init__.py +++ b/swarmforge/tongs/__init__.py @@ -123,6 +123,7 @@ mount_destination, mount_target_error, mount_volume_name, + mounts_word, normalize_mount_target, overlapping_mount_error, parse_mount, @@ -233,6 +234,7 @@ "mount_destination", "mount_target_error", "mount_volume_name", + "mounts_word", "normalize_mount_target", "overlapping_mount_error", "parse_mount", diff --git a/swarmforge/tongs/approvals.py b/swarmforge/tongs/approvals.py index 35c61c1..9527b5e 100644 --- a/swarmforge/tongs/approvals.py +++ b/swarmforge/tongs/approvals.py @@ -11,8 +11,8 @@ import json import os -from .model import WORKSPACE -from .mounts import _has_socket_mount, mount_volume_name +from .model import SOCKET_MOUNT, WORKSPACE +from .mounts import mount_volume_name, mounts_word from .secrets import find_secret_refs @@ -43,7 +43,7 @@ def privilege_summary(defn): "mounts": list(defn.get("mounts") or []), "volumes": _declared_volumes(defn), "networks": list(defn.get("networks") or []), - "socket": _has_socket_mount(defn), + "socket": mounts_word(defn, SOCKET_MOUNT), "gpus": _requested_gpus(defn), } diff --git a/swarmforge/tongs/argv.py b/swarmforge/tongs/argv.py index 893595b..2e11473 100644 --- a/swarmforge/tongs/argv.py +++ b/swarmforge/tongs/argv.py @@ -12,8 +12,14 @@ from swarmforge.names import canonical_path, dir_hint, path_token, sanitize_token from .mcp import _is_network_facing, _ordered_aliases -from .model import LABEL_CONFIG_HASH, LABEL_TONG_NAME, WORKSPACE_HOST_ENV, parse_gpus -from .mounts import DEFAULT_DOCKER_SOCKET, _has_socket_mount, tong_mount_specs +from .model import ( + LABEL_CONFIG_HASH, + LABEL_TONG_NAME, + SOCKET_MOUNT, + WORKSPACE_HOST_ENV, + parse_gpus, +) +from .mounts import DEFAULT_DOCKER_SOCKET, mounts_word, tong_mount_specs from .secrets import SECRET_FIFO_TMPFS, declared_run_override @@ -180,7 +186,7 @@ def tong_run_argv( if secret_channel: argv += ["--tmpfs", SECRET_FIFO_TMPFS] effective_env = dict(env or {}) - if workspace and _has_socket_mount(defn) and defn.get("lifecycle") == "session": + if workspace and mounts_word(defn, SOCKET_MOUNT) and defn.get("lifecycle") == "session": effective_env.setdefault(WORKSPACE_HOST_ENV, workspace) for key in sorted(effective_env): argv += ["-e", "%s=%s" % (key, effective_env[key])] diff --git a/swarmforge/tongs/mounts.py b/swarmforge/tongs/mounts.py index 5b3cafc..413f72d 100644 --- a/swarmforge/tongs/mounts.py +++ b/swarmforge/tongs/mounts.py @@ -46,9 +46,14 @@ VOLUME_HINT_LIMIT = 32 -def _has_socket_mount(defn): +def mounts_word(defn, word): + """True if a tong's `mounts:` request the magic `word`. + + The word may carry a name, a target and/or a mode (e.g. `workspace:/code:ro`), + so compare only the word before the first colon. + """ for mount in defn.get("mounts") or []: - if isinstance(mount, str) and mount.split(":", 1)[0] == SOCKET_MOUNT: + if isinstance(mount, str) and mount.split(":", 1)[0] == word: return True return False @@ -183,7 +188,7 @@ def reserved_mount_targets(defn, socket_path=DEFAULT_DOCKER_SOCKET): if isinstance(env, dict) and partition_secret_env(env)[1]: paths[SECRET_FIFO_DIR] = "the tmpfs where the launcher delivers this tong's secrets" paths[SECRET_INJECT_SHELL] = "the shell the secret wrapper execs" - if _has_socket_mount(defn): + if mounts_word(defn, SOCKET_MOUNT): paths[socket_path] = "where the docker socket is mounted" return paths diff --git a/tests/test_tongs_mounts.py b/tests/test_tongs_mounts.py index 69cd2e8..a830f5f 100644 --- a/tests/test_tongs_mounts.py +++ b/tests/test_tongs_mounts.py @@ -92,6 +92,13 @@ def test_parse_mount_word_set_can_be_narrowed(self): with self.assertRaisesRegex(ValueError, "not allowed here"): tongs.parse_mount("docker-socket", words=narrowed) + def test_mounts_word_compares_only_the_word(self): + defn = {"mounts": ["volume:workspace:/data", "docker-socket:ro", 7]} + self.assertTrue(tongs.mounts_word(defn, tongs.VOLUME_MOUNT)) + self.assertTrue(tongs.mounts_word(defn, "docker-socket")) + self.assertFalse(tongs.mounts_word(defn, tongs.WORKSPACE_MOUNT)) + self.assertFalse(tongs.mounts_word({}, tongs.WORKSPACE_MOUNT)) + def test_mount_destination_refuses_a_word_with_no_default(self): # A word without its own default must not inherit the socket's destination. with self.assertRaisesRegex(ValueError, "no destination"): From c1c5f97cd7f073d2aa0ede4b9e5a163ee3547765 Mon Sep 17 00:00:00 2001 From: CrypticSwarm Date: Thu, 8 Oct 2026 23:07:35 -0500 Subject: [PATCH 08/14] Read a volume mount's name in the one parse `parse_mount` dropped the volume name, so every overlap check re-parsed the new mount and each placed one to get it back. Return the name from the parse and carry it in the placed entries instead. --- swarmforge/tongs/__init__.py | 2 -- swarmforge/tongs/approvals.py | 4 ++-- swarmforge/tongs/mounts.py | 44 +++++++++++----------------------- swarmforge/tongs/validate.py | 8 +++---- tests/test_tongs_mounts.py | 45 ++++++++++++++++------------------- 5 files changed, 41 insertions(+), 62 deletions(-) diff --git a/swarmforge/tongs/__init__.py b/swarmforge/tongs/__init__.py index 8c2603f..4b6c0aa 100644 --- a/swarmforge/tongs/__init__.py +++ b/swarmforge/tongs/__init__.py @@ -122,7 +122,6 @@ WORKSPACE_VOLUME_SCOPE, mount_destination, mount_target_error, - mount_volume_name, mounts_word, normalize_mount_target, overlapping_mount_error, @@ -233,7 +232,6 @@ "WORKSPACE_VOLUME_SCOPE", "mount_destination", "mount_target_error", - "mount_volume_name", "mounts_word", "normalize_mount_target", "overlapping_mount_error", diff --git a/swarmforge/tongs/approvals.py b/swarmforge/tongs/approvals.py index 9527b5e..01a4c89 100644 --- a/swarmforge/tongs/approvals.py +++ b/swarmforge/tongs/approvals.py @@ -12,7 +12,7 @@ import os from .model import SOCKET_MOUNT, WORKSPACE -from .mounts import mount_volume_name, mounts_word +from .mounts import mounts_word, parse_mount from .secrets import find_secret_refs @@ -70,7 +70,7 @@ def _declared_volumes(defn): if not isinstance(mount, str): continue try: - volume = mount_volume_name(mount) + volume = parse_mount(mount)[1] except ValueError: continue if volume: diff --git a/swarmforge/tongs/mounts.py b/swarmforge/tongs/mounts.py index 413f72d..787416a 100644 --- a/swarmforge/tongs/mounts.py +++ b/swarmforge/tongs/mounts.py @@ -79,7 +79,7 @@ def _targets_overlap(one, other): def parse_mount(mount, words=MOUNT_WORDS): - """Split a `mounts:` entry into `(word, target, mode)`. + """Split a `mounts:` entry into `(word, volume, target, mode)`. The grammar is `[:/target][:mode]`: the magic word, an optional absolute path naming where the container sees the mount, and an optional access mode, @@ -87,8 +87,8 @@ def parse_mount(mount, words=MOUNT_WORDS): `workspace:/work:ro` are all valid. `target`/`mode` are None when not declared; the caller supplies the default target for its magic word. The `volume` word is spelled `volume::/target[:mode]`: its name comes first and its target - is required, since a named volume has no natural place to land. The name is - left out of the tuple and read with `mount_volume_name`. + is required, since a named volume has no natural place to land. `volume` is + that name, and None for every other word. The word is checked against `words` before the rest of the entry, since a mount nobody recognizes has no meaningful target or mode; narrow that set to accept @@ -97,21 +97,6 @@ def parse_mount(mount, words=MOUNT_WORDS): that is not last, more than one target, a target resolving to the root, or a volume mount whose name is malformed or whose target is missing. """ - word, _, target, mode = _split_mount(mount, words) - return word, target, mode - - -def mount_volume_name(mount): - """The name a `volume::...` entry declares, or None for any other word. - - Judged by the same grammar as `parse_mount`, so it raises `ValueError` for - exactly the entries that does. - """ - return _split_mount(mount, MOUNT_WORDS)[1] - - -def _split_mount(mount, words): - """`(word, volume name, target, mode)` for a `mounts:` entry; see `parse_mount`.""" fields = mount.split(":") word, fields = fields[0], fields[1:] if word not in words: @@ -235,22 +220,21 @@ def mount_target_error(mount, word, target, destination, reserved): return None -def overlapping_mount_error(mount, destination, placed): +def overlapping_mount_error(mount, destination, volume, placed): """Why a mount collides with one already placed, or None if it is clear. - `placed` is the `(mount, destination)` pairs accepted so far, each entry one - `parse_mount` accepted, as `mount` is. Docker refuses two binds onto one + `volume` is the name `parse_mount` read from `mount`, and `placed` the + `(mount, destination, volume)` triples accepted so far. Docker refuses two binds onto one destination outright, and creates the inner mountpoint of nested ones inside the outer bind -- inside the user's workspace, for a workspace bind. One volume is mounted once: a second target for the same data, perhaps with another access mode, is far likelier a mistake than a need. """ - volume = mount_volume_name(mount) - for other, other_destination in placed: + for other, other_destination, other_volume in placed: if _targets_overlap(destination, other_destination): return ("mount %r: %s overlaps mount %r at %s" % (mount, destination, other, other_destination)) - if volume is not None and mount_volume_name(other) == volume: + if volume is not None and other_volume == volume: return "mount %r: volume %r is already mounted by %r" % (mount, volume, other) return None @@ -325,14 +309,14 @@ def tong_mount_specs(defn, workspace, socket_path=DEFAULT_DOCKER_SOCKET, """ specs = [] reserved = reserved_mount_targets(defn, socket_path) - placed = [] # (mount, destination) already emitted + placed = [] # (mount, destination, volume) already emitted for mount in defn.get("mounts") or []: if not isinstance(mount, str): raise ValueError("mount entries must be strings, got %r" % (mount,)) - word, target, mode = parse_mount(mount) + word, volume, target, mode = parse_mount(mount) destination = mount_destination(word, target, socket_path) reason = (mount_target_error(mount, word, target, destination, reserved) - or overlapping_mount_error(mount, destination, placed)) + or overlapping_mount_error(mount, destination, volume, placed)) if reason: raise ValueError(reason) if word == WORKSPACE_MOUNT: @@ -345,7 +329,7 @@ def tong_mount_specs(defn, workspace, socket_path=DEFAULT_DOCKER_SOCKET, if not tong_name or volume_scope is None: raise ValueError( "mount %r needs the tong's name and scope to name its volume" % (mount,)) - source = tong_volume_name(tong_name, mount_volume_name(mount), volume_scope) + source = tong_volume_name(tong_name, volume, volume_scope) else: # Unreachable via `parse_mount`; a new word must not inherit the socket bind. raise ValueError("mount %r has no docker spec" % (mount,)) @@ -353,7 +337,7 @@ def tong_mount_specs(defn, workspace, socket_path=DEFAULT_DOCKER_SOCKET, if mode: spec += ":" + mode specs.append(spec) - placed.append((mount, destination)) + placed.append((mount, destination, volume)) return specs @@ -372,7 +356,7 @@ def workspace_mount_placements(defn, socket_path=DEFAULT_DOCKER_SOCKET): for mount in defn.get("mounts") or []: if not isinstance(mount, str): raise ValueError("mount entries must be strings, got %r" % (mount,)) - word, target, mode = parse_mount(mount) + word, _, target, mode = parse_mount(mount) if word == WORKSPACE_MOUNT: placements.append((mount_destination(word, target, socket_path), mode)) return placements diff --git a/swarmforge/tongs/validate.py b/swarmforge/tongs/validate.py index 02dfe99..4107d75 100644 --- a/swarmforge/tongs/validate.py +++ b/swarmforge/tongs/validate.py @@ -150,24 +150,24 @@ def err(msg): mounts = defn.get("mounts") if isinstance(mounts, list): reserved = reserved_mount_targets(defn) - placed = [] # (mount, destination) already accounted for + placed = [] # (mount, destination, volume) already accounted for for mount in mounts: if not isinstance(mount, str): err("mount entries must be strings, got %r" % (mount,)) continue try: - word, target, _ = parse_mount(mount) + word, volume, target, _ = parse_mount(mount) destination = mount_destination(word, target) except ValueError as exc: err(str(exc)) continue reason = mount_target_error(mount, word, target, destination, reserved) if reason is None: - reason = overlapping_mount_error(mount, destination, placed) + reason = overlapping_mount_error(mount, destination, volume, placed) if reason: err(reason) continue - placed.append((mount, destination)) + placed.append((mount, destination, volume)) networks = defn.get("networks") if isinstance(networks, list): diff --git a/tests/test_tongs_mounts.py b/tests/test_tongs_mounts.py index a830f5f..af9845d 100644 --- a/tests/test_tongs_mounts.py +++ b/tests/test_tongs_mounts.py @@ -23,29 +23,25 @@ class MountGrammarTests(unittest.TestCase): and `tong_mount_specs` both delegate here.""" def test_parse_mount_splits_the_optional_fields(self): - self.assertEqual(tongs.parse_mount("workspace"), ("workspace", None, None)) - self.assertEqual(tongs.parse_mount("workspace:ro"), ("workspace", None, "ro")) - self.assertEqual(tongs.parse_mount("workspace:/code"), ("workspace", "/code", None)) + self.assertEqual(tongs.parse_mount("workspace"), ("workspace", None, None, None)) + self.assertEqual(tongs.parse_mount("workspace:ro"), ("workspace", None, None, "ro")) self.assertEqual( - tongs.parse_mount("workspace:/code:rw"), ("workspace", "/code", "rw") + tongs.parse_mount("workspace:/code"), ("workspace", None, "/code", None) ) self.assertEqual( - tongs.parse_mount("docker-socket:ro"), ("docker-socket", None, "ro") + tongs.parse_mount("workspace:/code:rw"), ("workspace", None, "/code", "rw") + ) + self.assertEqual( + tongs.parse_mount("docker-socket:ro"), ("docker-socket", None, None, "ro") ) - def test_parse_mount_keeps_a_volume_name_out_of_the_tuple(self): + def test_parse_mount_reads_a_volume_name(self): self.assertEqual( - tongs.parse_mount("volume:models:/data"), ("volume", "/data", None) + tongs.parse_mount("volume:models:/data"), ("volume", "models", "/data", None) ) self.assertEqual( - tongs.parse_mount("volume:models:/data:ro"), ("volume", "/data", "ro") + tongs.parse_mount("volume:v1.2-x:/data:ro"), ("volume", "v1.2-x", "/data", "ro") ) - self.assertEqual(tongs.mount_volume_name("volume:models:/data:ro"), "models") - self.assertEqual(tongs.mount_volume_name("volume:v1.2-x:/data"), "v1.2-x") - - def test_mount_volume_name_is_none_for_other_words(self): - self.assertIsNone(tongs.mount_volume_name("workspace:/code:ro")) - self.assertIsNone(tongs.mount_volume_name("docker-socket")) def test_parse_mount_rejects_a_malformed_volume_name(self): # `_` is what splits the volume back off its docker name, so it cannot be in one. @@ -54,12 +50,10 @@ def test_parse_mount_rejects_a_malformed_volume_name(self): "volume:models\n:/data"): with self.assertRaisesRegex(ValueError, "volume::/target", msg=mount): tongs.parse_mount(mount) - with self.assertRaises(ValueError, msg=mount): - tongs.mount_volume_name(mount) def test_parse_mount_caps_a_volume_name(self): longest = "v" * tongs.VOLUME_NAME_MAX_LENGTH - self.assertEqual(tongs.mount_volume_name("volume:%s:/data" % longest), longest) + self.assertEqual(tongs.parse_mount("volume:%s:/data" % longest)[1], longest) with self.assertRaisesRegex(ValueError, "at most 64 characters"): tongs.parse_mount("volume:%s:/data" % (longest + "v")) @@ -87,7 +81,8 @@ def test_parse_mount_word_set_can_be_narrowed(self): # A known word refused by a narrowed set is a policy refusal, not a misspelling. narrowed = (tongs.WORKSPACE_MOUNT,) self.assertEqual( - tongs.parse_mount("workspace:/code", words=narrowed), ("workspace", "/code", None) + tongs.parse_mount("workspace:/code", words=narrowed), + ("workspace", None, "/code", None), ) with self.assertRaisesRegex(ValueError, "not allowed here"): tongs.parse_mount("docker-socket", words=narrowed) @@ -251,20 +246,22 @@ def test_mount_target_error_judges_the_default_destination_too(self): ) def test_overlapping_mount_error_catches_duplicates_and_nesting(self): - placed = [("workspace:/code", "/code")] - self.assertIsNone(tongs.overlapping_mount_error("workspace:/src", "/src", placed)) + placed = [("workspace:/code", "/code", None)] + self.assertIsNone( + tongs.overlapping_mount_error("workspace:/src", "/src", None, placed)) for destination in ("/code", "/code/sub", "/"): self.assertIn( "overlaps mount 'workspace:/code'", - tongs.overlapping_mount_error("workspace", destination, placed), + tongs.overlapping_mount_error("workspace", destination, None, placed), ) def test_overlapping_mount_error_refuses_one_volume_mounted_twice(self): - placed = [("volume:models:/a", "/a")] - self.assertIsNone(tongs.overlapping_mount_error("volume:cache:/b", "/b", placed)) + placed = [("volume:models:/a", "/a", "models")] + self.assertIsNone( + tongs.overlapping_mount_error("volume:cache:/b", "/b", "cache", placed)) self.assertIn( "volume 'models' is already mounted by 'volume:models:/a'", - tongs.overlapping_mount_error("volume:models:/b:ro", "/b", placed), + tongs.overlapping_mount_error("volume:models:/b:ro", "/b", "models", placed), ) From 73477237866bfe73600e2c8513d2a91659be4680 Mon Sep 17 00:00:00 2001 From: CrypticSwarm Date: Thu, 8 Oct 2026 23:09:03 -0500 Subject: [PATCH 09/14] Key a checkout's approvals and volumes by one path Volumes resolved symlinks in the workspace path but approvals did not, so one checkout reached two ways needed two approvals yet shared one volume. Both now use `workspace_key`. A missing org or workspace dir is refused once, in `tong_mount_specs`, instead of at three layers. --- docs/tongs/approval.md | 2 +- swarmforge/anvil/orchestrate.py | 35 +++++++-------------------------- swarmforge/tongs/__init__.py | 2 ++ swarmforge/tongs/approvals.py | 6 +++--- swarmforge/tongs/model.py | 6 ++++++ swarmforge/tongs/mounts.py | 26 ++++++++++-------------- tests/test_anvil_orchestrate.py | 19 ++++++++---------- tests/test_tongs_approvals.py | 11 +++++++++++ tests/test_tongs_mounts.py | 6 ++---- 9 files changed, 50 insertions(+), 63 deletions(-) diff --git a/docs/tongs/approval.md b/docs/tongs/approval.md index cd3c07d..83a7a21 100644 --- a/docs/tongs/approval.md +++ b/docs/tongs/approval.md @@ -4,6 +4,6 @@ The user, org, and repo layers are installed deliberately and are trusted; they A **workspace**-sourced tong (from a repo you cloned) could otherwise request your secrets, host mounts, the docker socket, or the host's GPUs simply by being present, so the launcher gates it: - Before starting, it prints exactly what the tong requests — image, secret references, mounts, networks, docker-socket access, and GPU access — and asks you to approve. Control and formatting characters in those values are shown as escapes such as `\x1b` or `\n`, so a value cannot move the cursor, restyle, or hide any part of the summary. Every `tongs:` warning escapes them the same way, since many quote a tong's filename. A `volume:` mount is also listed by name on a `volumes:` line, which warns that its data outlives the tong and that any data already in it, left by an earlier definition or an earlier checkout at this path, is reused. -- Approval is keyed by workspace path + tong name + a hash of the merged definition, stored in `~/.swarmforge/approvals.json`. Any change to the definition re-prompts. +- Approval is keyed by workspace path (symlinks resolved, as for its [volumes](definitions.md#volume-naming-and-scope)) + tong name + a hash of the merged definition, stored in `~/.swarmforge/approvals.json`. Any change to the definition re-prompts. - The gate defaults to **No**, and a non-interactive stdin reads as No. A scripted `--no-prompt` run **fails closed** rather than auto-approving. - Approving `image: foo:latest` approves a moving target; **pinned digests are the recommended convention** for workspace tongs. diff --git a/swarmforge/anvil/orchestrate.py b/swarmforge/anvil/orchestrate.py index 92ea54f..e854c24 100644 --- a/swarmforge/anvil/orchestrate.py +++ b/swarmforge/anvil/orchestrate.py @@ -80,7 +80,7 @@ def unsupported_tong_reasons(merged): reuses the container (a `session` tong is the right home for a per-workspace mount); * a workspace-sourced `shared` tong that mounts a `volume` -- the volume - is scoped to one checkout (see `_volume_scopes`) but the container is + is scoped to one checkout (see `tongs.volume_scope`) but the container is not, so a session in another checkout shipping the same definition would reuse it, volume and all. @@ -114,29 +114,6 @@ def unsupported_tong_reasons(merged): return reasons -def _volume_scopes(merged, org_tongs_dir, workspace): - """The scope naming each tong's volumes: `{name: (class, path) or None}`. - - A volume is partitioned like the trust of the layer its tong came from (see - `tongs.volume_scope`): an org tong's by its org's tongs directory, a - workspace tong's by the checkout's top level, and the user and repo layers' - by this machine. None for a tong that mounts no volume. Raises - `OrchestrationError` for a volume with no directory to scope it by, before - anything starts, rather than letting it fall back to the local scope. - """ - scopes = {} - for name in sorted(merged): - scopes[name] = None - if not tongs.mounts_word(merged[name]["definition"], tongs.VOLUME_MOUNT): - continue - try: - scopes[name] = tongs.volume_scope( - merged[name]["source"], org_tongs_dir=org_tongs_dir, workspace=workspace) - except ValueError as exc: - raise OrchestrationError("tong '%s': %s" % (name, exc)) - return scopes - - def ensure_mcp_harness_supported(merged, harness): """Refuse MCP tongs when no emitter exists for the selected harness.""" mcp_names = [ @@ -343,9 +320,7 @@ def run_with_tongs(merged, anvil_cmd, opts, *, docker, providers=None, Returns the anvil's exit code. Raises `OrchestrationError` if a tong never becomes ready (the anvil does not run against a half-up environment) or a - `session` tong is discovered with no anvil `--name` to key the session by, or - a tong mounts a volume with no directory to scope it by (an org tong with no - org tongs directory, a workspace tong with no workspace path), and + `session` tong is discovered with no anvil `--name` to key the session by, and `SecretResolutionError` if a secret reference cannot be resolved. """ ensure_mcp_harness_supported(merged, opts.harness) @@ -376,7 +351,11 @@ def run_with_tongs(merged, anvil_cmd, opts, *, docker, providers=None, "join their isolated network" ) - volume_scopes = _volume_scopes(merged, org_tongs_dir, opts.workspace) + volume_scopes = { + name: tongs.volume_scope(merged[name]["source"], org_tongs_dir=org_tongs_dir, + workspace=opts.workspace) + for name in merged + } plan = tongs.plan_network(merged, base_network, session_id) injection = tongs.plan_injection(merged, opts.harness) diff --git a/swarmforge/tongs/__init__.py b/swarmforge/tongs/__init__.py index 4b6c0aa..6b5468e 100644 --- a/swarmforge/tongs/__init__.py +++ b/swarmforge/tongs/__init__.py @@ -107,6 +107,7 @@ printable, readiness_settings, warn, + workspace_key, ) from .mounts import ( DEFAULT_DOCKER_SOCKET, @@ -219,6 +220,7 @@ "printable", "readiness_settings", "warn", + "workspace_key", "DEFAULT_DOCKER_SOCKET", "DEFAULT_WORKSPACE_MOUNT_TARGET", "LOCAL_VOLUME_SCOPE", diff --git a/swarmforge/tongs/approvals.py b/swarmforge/tongs/approvals.py index 01a4c89..51e8b59 100644 --- a/swarmforge/tongs/approvals.py +++ b/swarmforge/tongs/approvals.py @@ -11,7 +11,7 @@ import json import os -from .model import SOCKET_MOUNT, WORKSPACE +from .model import SOCKET_MOUNT, WORKSPACE, workspace_key from .mounts import mounts_word, parse_mount from .secrets import find_secret_refs @@ -109,7 +109,7 @@ def is_approved(approvals, workspace_path, name, defn): Fails closed (returns False) on a missing or malformed store entry rather than raising -- a hand-edited approvals.json must never crash the gate. """ - entry = approvals.get(workspace_path) + entry = approvals.get(workspace_key(workspace_path)) if not isinstance(entry, dict): return False return entry.get(name) == config_hash(defn) @@ -120,5 +120,5 @@ def record_approval(approvals, workspace_path, name, defn): Mutates and returns the store (same object) so callers can persist it. """ - approvals.setdefault(workspace_path, {})[name] = config_hash(defn) + approvals.setdefault(workspace_key(workspace_path), {})[name] = config_hash(defn) return approvals diff --git a/swarmforge/tongs/model.py b/swarmforge/tongs/model.py index 5a85eb0..269eff1 100644 --- a/swarmforge/tongs/model.py +++ b/swarmforge/tongs/model.py @@ -9,6 +9,7 @@ every message passes through `printable` beside it. """ +import os import re import sys @@ -35,6 +36,11 @@ WORKSPACE_HOST_ENV = "SWARMFORGE_WORKSPACE_HOST_PATH" +def workspace_key(workspace): + """The key a checkout's approvals and volumes are stored under: its path, symlinks resolved.""" + return os.path.realpath(workspace) + + def warn(message): """Print `message` to stderr through `printable`, since it may quote definition-sourced text.""" print("tongs: %s" % printable(message), file=sys.stderr) diff --git a/swarmforge/tongs/mounts.py b/swarmforge/tongs/mounts.py index 787416a..b3b851e 100644 --- a/swarmforge/tongs/mounts.py +++ b/swarmforge/tongs/mounts.py @@ -9,13 +9,12 @@ import hashlib import json -import os import posixpath import re from swarmforge.names import canonical_path, sanitize_token -from .model import ORG, REPO, SOCKET_MOUNT, USER, WORKSPACE +from .model import ORG, REPO, SOCKET_MOUNT, USER, WORKSPACE, workspace_key from .secrets import SECRET_FIFO_DIR, SECRET_INJECT_SHELL, partition_secret_env @@ -267,24 +266,19 @@ def volume_scope(layer, org_tongs_dir=None, workspace=None): An org tong's volumes belong to its org's tongs directory (canonical, as `org_scope_token` reads it); a workspace tong's to the checkout's top level - `workspace` with symlinks resolved, so every session in that checkout, from - any subdirectory, shares them -- as does any later checkout at that path, - since the path is the key. The user and repo layers are this machine's own + `workspace`, keyed by `workspace_key` like its approvals, so every session in + that checkout, from any subdirectory, shares them -- as does any later + checkout at that path. The user and repo layers are this machine's own and share `(LOCAL_VOLUME_SCOPE, None)`, so a repo tong and the same-named - user tong it replaces in the merge use one volume. Raises `ValueError` for an org or - workspace tong with no directory to scope it by, rather than falling back to - the local scope, and for an unknown layer. + user tong it replaces in the merge use one volume. None for an org or + workspace tong with no directory to scope it by, which `tong_mount_specs` + refuses rather than falling back to the local scope. Raises `ValueError` for + an unknown layer. """ if layer == ORG: - if not org_tongs_dir: - raise ValueError("an org tong's volume is scoped to its org, but no org " - "tongs directory is known") - return (ORG_VOLUME_SCOPE, canonical_path(org_tongs_dir)) + return (ORG_VOLUME_SCOPE, canonical_path(org_tongs_dir)) if org_tongs_dir else None if layer == WORKSPACE: - if not workspace: - raise ValueError("a workspace tong's volume is scoped to its checkout, but " - "no workspace path is known") - return (WORKSPACE_VOLUME_SCOPE, os.path.realpath(workspace)) + return (WORKSPACE_VOLUME_SCOPE, workspace_key(workspace)) if workspace else None if layer in (USER, REPO): return (LOCAL_VOLUME_SCOPE, None) raise ValueError("unknown layer %r has no volume scope" % (layer,)) diff --git a/tests/test_anvil_orchestrate.py b/tests/test_anvil_orchestrate.py index 9a2f066..22eebc1 100644 --- a/tests/test_anvil_orchestrate.py +++ b/tests/test_anvil_orchestrate.py @@ -1003,17 +1003,14 @@ def test_workspace_tong_volume_scoped_by_its_checkout(self): + ":/models"]) self.assertNotEqual(mounted[0], mounted[1]) - def test_workspace_tong_volume_without_a_workspace_path_starts_nothing(self): - docker = FakeDocker() - with self.assertRaisesRegex(launcher.OrchestrationError, "tong 'cache'.*workspace"): - self._run(docker, _merged("cache", SESSION_VOLUME)) - self.assertEqual(docker.calls, []) - - def test_org_tong_volume_without_an_org_directory_starts_nothing(self): - docker = FakeDocker() - with self.assertRaisesRegex(launcher.OrchestrationError, "tong 'cache'.*org"): - self._run(docker, _merged("cache", SESSION_VOLUME, source=tongs.ORG)) - self.assertEqual(docker.calls, []) + def test_volume_without_a_directory_to_scope_it_starts_no_tong(self): + # Never the local scope: that would share the volume across orgs or checkouts. + for source in (tongs.WORKSPACE, tongs.ORG): + docker = FakeDocker() + with self.assertRaisesRegex(launcher.OrchestrationError, + "tong 'cache'.*name and scope", msg=source): + self._run(docker, _merged("cache", SESSION_VOLUME, source=source)) + self.assertEqual(docker.run_argvs, [], source) def test_workspace_tong_without_a_volume_needs_no_workspace_path(self): docker = FakeDocker() diff --git a/tests/test_tongs_approvals.py b/tests/test_tongs_approvals.py index 68ef464..2e4efd9 100644 --- a/tests/test_tongs_approvals.py +++ b/tests/test_tongs_approvals.py @@ -118,6 +118,17 @@ def test_keyed_by_workspace_path(self): tongs.record_approval(approvals, self.ws, "github", self.defn) self.assertFalse(tongs.is_approved(approvals, "/other/ws", "github", self.defn)) + def test_a_symlinked_checkout_shares_its_approvals_and_volumes(self): + with tempfile.TemporaryDirectory() as tmp: + checkout = os.path.join(tmp, "checkout") + os.mkdir(checkout) + link = os.path.join(tmp, "link") + os.symlink(checkout, link) + approvals = tongs.record_approval({}, link, "github", self.defn) + self.assertTrue(tongs.is_approved(approvals, checkout, "github", self.defn)) + self.assertEqual(list(approvals), + [tongs.volume_scope(tongs.WORKSPACE, workspace=link)[1]]) + def test_load_save_roundtrip(self): with tempfile.TemporaryDirectory() as tmp: path = os.path.join(tmp, "nested", "approvals.json") diff --git a/tests/test_tongs_mounts.py b/tests/test_tongs_mounts.py index af9845d..b774c67 100644 --- a/tests/test_tongs_mounts.py +++ b/tests/test_tongs_mounts.py @@ -196,10 +196,8 @@ def test_volume_scope_follows_the_layer(self): tongs.volume_scope(tongs.ORG, org_tongs_dir="/orgs//acme/./tongs/"), (tongs.ORG_VOLUME_SCOPE, "/orgs/acme/tongs"), ) - with self.assertRaisesRegex(ValueError, "org tongs directory"): - tongs.volume_scope(tongs.ORG, workspace="/ws") - with self.assertRaisesRegex(ValueError, "workspace path"): - tongs.volume_scope(tongs.WORKSPACE, org_tongs_dir="/o/tongs") + self.assertIsNone(tongs.volume_scope(tongs.ORG, workspace="/ws")) + self.assertIsNone(tongs.volume_scope(tongs.WORKSPACE, org_tongs_dir="/o/tongs")) with self.assertRaisesRegex(ValueError, "unknown layer"): tongs.volume_scope("elsewhere") From 3423f66a18ab9f1aeca90fe9309176a9bc2569d7 Mon Sep 17 00:00:00 2001 From: CrypticSwarm Date: Thu, 8 Oct 2026 23:09:28 -0500 Subject: [PATCH 10/14] Cut a volume's hint with the names helper `tong_volume_name` repeated `dir_hint`'s truncate-and-trim rule. Split that rule out as `names.name_hint` and use it for both. --- swarmforge/names.py | 11 ++++++++--- swarmforge/tongs/mounts.py | 4 ++-- tests/test_names.py | 10 ++++++++++ 3 files changed, 20 insertions(+), 5 deletions(-) diff --git a/swarmforge/names.py b/swarmforge/names.py index 78a3ff7..1966eda 100644 --- a/swarmforge/names.py +++ b/swarmforge/names.py @@ -53,12 +53,17 @@ def path_digest(path): return digest.hexdigest()[:DIGEST_LENGTH] -def dir_hint(path, limit=None): - """`path`'s basename as a docker name, cut to `limit` and not left on a separator.""" - hint = sanitize_token(os.path.basename(canonical_path(path))) +def name_hint(name, limit=None): + """`name` as a docker name, cut to `limit` and not left on a separator.""" + hint = sanitize_token(name) return hint[:limit].rstrip("-_.") if limit is not None else hint +def dir_hint(path, limit=None): + """`path`'s basename as a `name_hint`.""" + return name_hint(os.path.basename(canonical_path(path)), limit) + + def path_token(path, hint): """Docker-name token for one host directory: `hint`, then `path`'s digest. diff --git a/swarmforge/tongs/mounts.py b/swarmforge/tongs/mounts.py index b3b851e..0450290 100644 --- a/swarmforge/tongs/mounts.py +++ b/swarmforge/tongs/mounts.py @@ -12,7 +12,7 @@ import posixpath import re -from swarmforge.names import canonical_path, sanitize_token +from swarmforge.names import canonical_path, name_hint from .model import ORG, REPO, SOCKET_MOUNT, USER, WORKSPACE, workspace_key from .secrets import SECRET_FIFO_DIR, SECRET_INJECT_SHELL, partition_secret_env @@ -256,7 +256,7 @@ def tong_volume_name(tong_name, volume, scope): scope_class, scope_path = scope key = json.dumps([scope_class, scope_path, tong_name]) digest = hashlib.sha256(key.encode("utf-8")).hexdigest()[:VOLUME_DIGEST_LENGTH] - hint = sanitize_token(tong_name)[:VOLUME_HINT_LIMIT].rstrip("-_.") + hint = name_hint(tong_name, VOLUME_HINT_LIMIT) parts = [VOLUME_NAME_PREFIX] + ([hint] if hint else []) + [digest] return "%s_%s" % ("-".join(parts), volume) diff --git a/tests/test_names.py b/tests/test_names.py index 79c820a..50c65d2 100644 --- a/tests/test_names.py +++ b/tests/test_names.py @@ -82,6 +82,16 @@ def test_directories_sharing_a_basename_digest_differently(self): names.path_digest("/home/me/repo2/master")) +class NameHint(unittest.TestCase): + """The readable half of a token, from any name.""" + + def test_a_name_is_reduced_to_what_docker_accepts(self): + self.assertEqual(names.name_hint("my tong!"), "my-tong") + + def test_a_cut_never_leaves_the_hint_on_a_separator(self): + self.assertEqual(names.name_hint("abc-defg", limit=4), "abc") + + class DirHint(unittest.TestCase): """The readable half of a token.""" From 7a4ef26d30947b6773582172ae16f6fe1727120a Mon Sep 17 00:00:00 2001 From: CrypticSwarm Date: Thu, 8 Oct 2026 23:10:06 -0500 Subject: [PATCH 11/14] Refuse a volume over a workspace's git-dir mounts A linked worktree's git dir is bind-mounted at its host path beside the workspace, outside the definition's own mounts, so the overlap check never saw it. A volume at /home could enclose it, and docker would leave its mountpoints inside the persistent volume. --- docs/tongs/definitions.md | 2 +- swarmforge/tongs/argv.py | 3 ++- swarmforge/tongs/mounts.py | 14 ++++++++++++-- tests/test_tongs_argv.py | 10 ++++++++++ tests/test_tongs_mounts.py | 13 +++++++++++++ 5 files changed, 38 insertions(+), 4 deletions(-) diff --git a/docs/tongs/definitions.md b/docs/tongs/definitions.md index 5fa65ee..3626ad1 100644 --- a/docs/tongs/definitions.md +++ b/docs/tongs/definitions.md @@ -70,7 +70,7 @@ Mounts are opt-in **magic words**, never raw host paths. Only three are recogniz - `workspace[:/target][:mode]` — bind-mounts the session workspace, at `/workspace` unless an absolute `target` says otherwise (e.g. `workspace:ro`, `workspace:/code`, `workspace:/code:ro`). A custom target lets an image that expects its sources elsewhere be used unmodified (it does not set the working directory — the process still starts in the image's own `WORKDIR`). A target is refused unless it is an absolute path free of whitespace, and refused if it resolves to `/`, overlaps another of the tong's mounts, or overlaps a path the tong's own wiring occupies — the secret-delivery tmpfs at `/run/swarmforge` and the `/bin/sh` its wrapper execs (for a tong with secret references), or the docker socket (for a tong that mounts it). The workspace bind is paired with the same git-dir mounts the anvil gets from `swarmforge/gitguard.py`: read-only guards over the config and hooks the host's git obeys, and — when the workspace is a linked worktree or another checkout whose git dir lives outside it — that git dir at its own absolute path, which is where the checkout's `.git` pointer file says to look (without it, git inside the tong fails with "not a git repository"). The one mount the anvil gets and a tong does not is the restated worktree registration — a tong keeps the host's copy, so `git worktree list` inside one names the checkout by its host path (the [git guard](../git-guard.md) has the rest). When every `workspace` mount is `ro`, the ride-along git-dir mounts are forced read-only too. - `docker-socket[:mode]` — bind-mounts the host docker socket onto the same path inside the container (so it takes no target). This is full host docker control and is always called out explicitly in the workspace approval prompt; it is the grant a broker tong needs. -- `volume::/target[:mode]` — mounts a docker **named volume** at `target`, for state the tong keeps between containers (a model store, a cache). The name comes first and is at most 64 letters, digits, `.` and `-`, starting with a letter or digit; the target is required and held to the same rules as a `workspace` target, and a tong mounts each volume only once. Docker creates the volume on first use and seeds a new, empty one with whatever the image has at `target`. The launcher never removes a volume — not on teardown, and not when a `shared` tong is recreated because its definition changed — so the state outlives every container that mounts it. Remove one by hand with `docker volume rm ` (`docker volume ls --filter name=swarmforge-volume-` lists them). A `volume:` mount is private to its tong; it is unrelated to `interface.kind: volume`, which would share a volume with the anvil and is not wired up yet. +- `volume::/target[:mode]` — mounts a docker **named volume** at `target`, for state the tong keeps between containers (a model store, a cache). The name comes first and is at most 64 letters, digits, `.` and `-`, starting with a letter or digit; the target is required and held to the same rules as a `workspace` target, may not overlap the git dir that a `workspace` mount brings along, and a tong mounts each volume only once. Docker creates the volume on first use and seeds a new, empty one with whatever the image has at `target`. The launcher never removes a volume — not on teardown, and not when a `shared` tong is recreated because its definition changed — so the state outlives every container that mounts it. Remove one by hand with `docker volume rm ` (`docker volume ls --filter name=swarmforge-volume-` lists them). A `volume:` mount is private to its tong; it is unrelated to `interface.kind: volume`, which would share a volume with the anvil and is not wired up yet. ### Volume naming and scope diff --git a/swarmforge/tongs/argv.py b/swarmforge/tongs/argv.py index 2e11473..9f9e4bd 100644 --- a/swarmforge/tongs/argv.py +++ b/swarmforge/tongs/argv.py @@ -191,7 +191,8 @@ def tong_run_argv( for key in sorted(effective_env): argv += ["-e", "%s=%s" % (key, effective_env[key])] for spec in tong_mount_specs(defn, workspace, socket_path=socket_path, - tong_name=name, volume_scope=volume_scope): + tong_name=name, volume_scope=volume_scope, + extra_mount_specs=extra_mount_specs or ()): argv += ["-v", spec] for spec in extra_mount_specs or []: argv += ["-v", spec] diff --git a/swarmforge/tongs/mounts.py b/swarmforge/tongs/mounts.py index 0450290..4e1b112 100644 --- a/swarmforge/tongs/mounts.py +++ b/swarmforge/tongs/mounts.py @@ -285,7 +285,7 @@ def volume_scope(layer, org_tongs_dir=None, workspace=None): def tong_mount_specs(defn, workspace, socket_path=DEFAULT_DOCKER_SOCKET, - tong_name=None, volume_scope=None): + tong_name=None, volume_scope=None, extra_mount_specs=()): """Concrete docker `-v` specs for a tong's `mounts:` magic words. Returns the list of `-v` *values* (the orchestrator pairs each with a `-v` @@ -294,7 +294,10 @@ def tong_mount_specs(defn, workspace, socket_path=DEFAULT_DOCKER_SOCKET, bind-mounts the host docker socket onto the same path it has on the host; `volume::/target[:mode]` mounts the named volume `tong_volume_name(tong_name, name, volume_scope)`, which docker creates on - first use and never removes with the container. + first use and never removes with the container. A volume may not overlap + the destination of any of `extra_mount_specs`, the `-v` values that ride + along with a workspace mount (see `tong_run_argv`): docker would create + their mountpoints inside the volume, where they outlive the container. Raises `ValueError` for a non-string entry, a malformed mount, an unusable or colliding destination, a `workspace` mount when no workspace path is known, or a `volume` mount without a `tong_name` and `volume_scope` to name it by -- a @@ -324,6 +327,13 @@ def tong_mount_specs(defn, workspace, socket_path=DEFAULT_DOCKER_SOCKET, raise ValueError( "mount %r needs the tong's name and scope to name its volume" % (mount,)) source = tong_volume_name(tong_name, volume, volume_scope) + for extra in extra_mount_specs: + # Ride-along sources never contain a colon; gitguard refuses those. + extra_destination = extra.split(":")[1] + if _targets_overlap(destination, extra_destination): + raise ValueError( + "mount %r: %s overlaps %s, which is mounted alongside the " + "workspace" % (mount, destination, extra_destination)) else: # Unreachable via `parse_mount`; a new word must not inherit the socket bind. raise ValueError("mount %r has no docker spec" % (mount,)) diff --git a/tests/test_tongs_argv.py b/tests/test_tongs_argv.py index 3115ce1..d9aa42e 100644 --- a/tests/test_tongs_argv.py +++ b/tests/test_tongs_argv.py @@ -221,6 +221,16 @@ def test_run_argv_extra_mount_specs_follow_definition_mounts(self): ["/ws:/workspace", "/ws/.git/config:/workspace/.git/config:ro"], ) + def test_run_argv_refuses_a_volume_over_an_extra_mount(self): + defn = def_of(NONE_TONG) + defn["mounts"] = ["workspace", "volume:v:/home"] + with self.assertRaisesRegex(ValueError, "mounted alongside the workspace"): + tongs.tong_run_argv( + "w", defn, container_name="c", network="n", alias="w", workspace="/ws", + extra_mount_specs=["/home/u/repo/.git:/home/u/repo/.git"], + volume_scope=(tongs.LOCAL_VOLUME_SCOPE, None), + ) + def test_run_argv_without_extra_mount_specs_is_unchanged(self): defn = def_of(NONE_TONG) defn["mounts"] = ["workspace"] diff --git a/tests/test_tongs_mounts.py b/tests/test_tongs_mounts.py index b774c67..1ca93fc 100644 --- a/tests/test_tongs_mounts.py +++ b/tests/test_tongs_mounts.py @@ -340,6 +340,19 @@ def test_mount_specs_volume_overlapping_another_mount_raises(self): tongs.tong_mount_specs( {"mounts": mounts}, "/ws", tong_name="t", volume_scope=LOCAL) + def test_mount_specs_volume_overlapping_a_ride_along_mount_raises(self): + git_dir = "/home/u/repo/.git:/home/u/repo/.git" + defn = {"mounts": ["workspace", "volume:v:/home"]} + with self.assertRaisesRegex(ValueError, "/home overlaps /home/u/repo/.git"): + tongs.tong_mount_specs(defn, "/ws", tong_name="t", volume_scope=LOCAL, + extra_mount_specs=[git_dir]) + self.assertEqual( + len(tongs.tong_mount_specs( + {"mounts": ["workspace", "volume:v:/data"]}, "/ws", tong_name="t", + volume_scope=LOCAL, extra_mount_specs=[git_dir])), + 2, + ) + def test_mount_specs_no_mounts_is_empty(self): self.assertEqual(tongs.tong_mount_specs({}, "/ws"), []) From 3ccd64bb56a8d72b84cfe7e5968532db0f3f6ce9 Mon Sep 17 00:00:00 2001 From: CrypticSwarm Date: Thu, 8 Oct 2026 23:11:37 -0500 Subject: [PATCH 12/14] Show each volume's target and docker name at approval The prompt listed bare volume names, so a reviewer could not see where the data lands or which existing docker volume it reuses. It now names both and says that concurrent sessions mount the same volume. --- docs/tongs/approval.md | 2 +- swarmforge/anvil/approval.py | 15 +++++++++++---- swarmforge/tongs/approvals.py | 20 ++++++++++++-------- tests/test_anvil_approval.py | 20 ++++++++++++++++---- tests/test_tongs_approvals.py | 15 +++++++++++++-- 5 files changed, 53 insertions(+), 19 deletions(-) diff --git a/docs/tongs/approval.md b/docs/tongs/approval.md index 83a7a21..7cf1573 100644 --- a/docs/tongs/approval.md +++ b/docs/tongs/approval.md @@ -3,7 +3,7 @@ The user, org, and repo layers are installed deliberately and are trusted; they skip the gate. A **workspace**-sourced tong (from a repo you cloned) could otherwise request your secrets, host mounts, the docker socket, or the host's GPUs simply by being present, so the launcher gates it: -- Before starting, it prints exactly what the tong requests — image, secret references, mounts, networks, docker-socket access, and GPU access — and asks you to approve. Control and formatting characters in those values are shown as escapes such as `\x1b` or `\n`, so a value cannot move the cursor, restyle, or hide any part of the summary. Every `tongs:` warning escapes them the same way, since many quote a tong's filename. A `volume:` mount is also listed by name on a `volumes:` line, which warns that its data outlives the tong and that any data already in it, left by an earlier definition or an earlier checkout at this path, is reused. +- Before starting, it prints exactly what the tong requests — image, secret references, mounts, networks, docker-socket access, and GPU access — and asks you to approve. Control and formatting characters in those values are shown as escapes such as `\x1b` or `\n`, so a value cannot move the cursor, restyle, or hide any part of the summary. Every `tongs:` warning escapes them the same way, since many quote a tong's filename. Each `volume:` mount also gets its own `volume:` line naming its target and the docker volume it mounts, so you can check `docker volume ls` for leftover data first. A note under them warns that the data outlives the tong, is mounted by each of its concurrent sessions, and that any data already there, left by an earlier definition or an earlier checkout at this path, is reused. - Approval is keyed by workspace path (symlinks resolved, as for its [volumes](definitions.md#volume-naming-and-scope)) + tong name + a hash of the merged definition, stored in `~/.swarmforge/approvals.json`. Any change to the definition re-prompts. - The gate defaults to **No**, and a non-interactive stdin reads as No. A scripted `--no-prompt` run **fails closed** rather than auto-approving. - Approving `image: foo:latest` approves a moving target; **pinned digests are the recommended convention** for workspace tongs. diff --git a/swarmforge/anvil/approval.py b/swarmforge/anvil/approval.py index a5908e4..ba88ff5 100644 --- a/swarmforge/anvil/approval.py +++ b/swarmforge/anvil/approval.py @@ -42,10 +42,15 @@ def render_privilege_summary(name, summary): if mounts: lines.append(" mounts: %s" % ", ".join(str(m) for m in mounts)) volumes = summary.get("volumes") or [] + for volume in volumes: + where = "%s at %s" % (volume["name"], volume["target"]) + if volume["docker"]: + where += " (docker volume %s)" % volume["docker"] + lines.append(" volume: %s" % where) if volumes: - lines.append(" volumes: %s (persistent: outlives the tong and reuses any data " - "left by an earlier definition or checkout at this path)" - % ", ".join(volumes)) + lines.append(" persistent: outlives the tong, is mounted by each of its " + "concurrent sessions, and reuses any data left by an earlier " + "definition or checkout at this path") networks = summary.get("networks") or [] if networks: lines.append(" networks: %s" % ", ".join(str(n) for n in networks)) @@ -113,7 +118,9 @@ def gate_workspace_tongs(merged, workspace, approvals_path, prompt=True, out=Non for name, defn in pending: if tongs.is_approved(approvals, workspace, name, defn): continue - out.write(render_privilege_summary(name, tongs.privilege_summary(defn)) + "\n") + summary = tongs.privilege_summary( + defn, name, tongs.volume_scope(tongs.WORKSPACE, workspace=workspace)) + out.write(render_privilege_summary(name, summary) + "\n") if not prompt: raise ApprovalDenied( "workspace tong %r is unapproved and --no-prompt fails closed" % name diff --git a/swarmforge/tongs/approvals.py b/swarmforge/tongs/approvals.py index 51e8b59..a448e05 100644 --- a/swarmforge/tongs/approvals.py +++ b/swarmforge/tongs/approvals.py @@ -12,7 +12,7 @@ import os from .model import SOCKET_MOUNT, WORKSPACE, workspace_key -from .mounts import mounts_word, parse_mount +from .mounts import mounts_word, parse_mount, tong_volume_name from .secrets import find_secret_refs @@ -28,20 +28,22 @@ def config_hash(defn): return hashlib.sha256(canonical.encode("utf-8")).hexdigest() -def privilege_summary(defn): +def privilege_summary(defn, name=None, volume_scope=None): """Structured summary of what a definition asks for, for the approval gate. Gathers the privileges a reviewer must see before approving a workspace-sourced tong: image, secret references, mounts (and the persistent volumes among them), networks, docker-socket access, and host - GPU access. Rendering and prompting are the caller's job; this just + GPU access. Each volume is `{"name", "target", "docker"}`, where `docker` is + the docker volume the tong `name` mounts from `volume_scope`, or None + without both. Rendering and prompting are the caller's job; this just assembles the facts. """ return { "image": defn.get("image"), "secrets": [{"provider": p, "ref": r} for p, r in find_secret_refs(defn)], "mounts": list(defn.get("mounts") or []), - "volumes": _declared_volumes(defn), + "volumes": _declared_volumes(defn, name, volume_scope), "networks": list(defn.get("networks") or []), "socket": mounts_word(defn, SOCKET_MOUNT), "gpus": _requested_gpus(defn), @@ -58,8 +60,8 @@ def _requested_gpus(defn): return resources.get("gpus") if isinstance(resources, dict) else None -def _declared_volumes(defn): - """Names of the volumes a definition's well-formed `volume:` mounts declare. +def _declared_volumes(defn, name, volume_scope): + """The volumes a definition's well-formed `volume:` mounts declare. The gate runs before validation, so a malformed entry is skipped here rather than raised: it is still listed among the raw mounts, and validation refuses @@ -70,11 +72,13 @@ def _declared_volumes(defn): if not isinstance(mount, str): continue try: - volume = parse_mount(mount)[1] + _, volume, target, _ = parse_mount(mount) except ValueError: continue if volume: - volumes.append(volume) + docker = (tong_volume_name(name, volume, volume_scope) + if name and volume_scope else None) + volumes.append({"name": volume, "target": target, "docker": docker}) return volumes diff --git a/tests/test_anvil_approval.py b/tests/test_anvil_approval.py index 7fe84f6..d3551df 100644 --- a/tests/test_anvil_approval.py +++ b/tests/test_anvil_approval.py @@ -60,17 +60,23 @@ def test_non_mapping_resources_renders_no_gpus_line(self): def test_calls_out_a_volume_as_persistent(self): defn = dict(WORKSPACE_TONG, mounts=["volume:models:/models"]) - text = launcher.render_privilege_summary("cache", tongs.privilege_summary(defn)) + scope = (tongs.WORKSPACE_VOLUME_SCOPE, "/home/me/proj") + text = launcher.render_privilege_summary( + "cache", tongs.privilege_summary(defn, "cache", scope)) self.assertIn("volume:models:/models", text) + lines = text.splitlines() + self.assertIn(" volume: models at /models (docker volume %s)" + % tongs.tong_volume_name("cache", "models", scope), lines) self.assertIn( - " volumes: models (persistent: outlives the tong and reuses any data left by " - "an earlier definition or checkout at this path)", text.splitlines()) + " persistent: outlives the tong, is mounted by each of its concurrent " + "sessions, and reuses any data left by an earlier definition or checkout at " + "this path", lines) def test_omits_unrequested_sections(self): defn = {"image": "x", "interface": {"kind": "none"}, "resources": {"memory": "1g"}} text = launcher.render_privilege_summary("x", tongs.privilege_summary(defn)) self.assertNotIn("secrets:", text) - self.assertNotIn("volumes:", text) + self.assertNotIn("volume", text) self.assertNotIn("docker socket", text) self.assertNotIn("gpus", text) @@ -198,6 +204,12 @@ def test_accepted_workspace_tong_records_and_persists(self): stored = tongs.load_approvals(self.approvals) self.assertTrue(tongs.is_approved(stored, self.ws, "gh", WORKSPACE_TONG)) + def test_prompt_names_the_volume_this_checkout_would_mount(self): + merged = _merged("cache", dict(WORKSPACE_TONG, mounts=["volume:models:/models"])) + out = self._gate(merged, answer="y\n") + scope = tongs.volume_scope(tongs.WORKSPACE, workspace=self.ws) + self.assertIn(tongs.tong_volume_name("cache", "models", scope), out) + def test_declined_workspace_tong_raises_and_does_not_persist(self): merged = _merged("gh", WORKSPACE_TONG) with self.assertRaises(launcher.ApprovalDenied): diff --git a/tests/test_tongs_approvals.py b/tests/test_tongs_approvals.py index 2e4efd9..3d310c8 100644 --- a/tests/test_tongs_approvals.py +++ b/tests/test_tongs_approvals.py @@ -76,15 +76,26 @@ def test_summary_reports_an_unvalidated_gpus_value_as_declared(self): def test_summary_names_each_declared_volume(self): defn = def_of(GITHUB_TONG) defn["mounts"] = ["workspace:ro", "volume:models:/a", "volume:cache:/c:ro"] - self.assertEqual(tongs.privilege_summary(defn)["volumes"], ["models", "cache"]) + self.assertEqual(tongs.privilege_summary(defn)["volumes"], [ + {"name": "models", "target": "/a", "docker": None}, + {"name": "cache", "target": "/c", "docker": None}, + ]) self.assertEqual(tongs.privilege_summary(def_of(GITHUB_TONG))["volumes"], []) + def test_summary_names_the_docker_volume_given_its_scope(self): + defn = dict(def_of(GITHUB_TONG), mounts=["volume:models:/a"]) + scope = (tongs.WORKSPACE_VOLUME_SCOPE, "/home/me/proj") + self.assertEqual( + tongs.privilege_summary(defn, "cache", scope)["volumes"][0]["docker"], + tongs.tong_volume_name("cache", "models", scope), + ) + def test_summary_tolerates_malformed_mounts(self): # The gate runs before validation, which refuses these on its own. defn = def_of(GITHUB_TONG) defn["mounts"] = ["volume:bad_name:/a", "volume:v", 7, "/etc:/etc", "volume:ok:/b"] summary = tongs.privilege_summary(defn) - self.assertEqual(summary["volumes"], ["ok"]) + self.assertEqual([v["name"] for v in summary["volumes"]], ["ok"]) self.assertEqual(summary["mounts"], defn["mounts"]) From 827b01d4634880126488b058712e249e4232b31d Mon Sep 17 00:00:00 2001 From: CrypticSwarm Date: Thu, 8 Oct 2026 23:11:46 -0500 Subject: [PATCH 13/14] Document how far a tong volume is shared State that every concurrent session mounts a tong's volume, that the user and repo layers share one across checkouts, and that docker seeds a volume from the image only once, so targets should hold data only. --- docs/tongs/definitions.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/docs/tongs/definitions.md b/docs/tongs/definitions.md index 3626ad1..30aa1f1 100644 --- a/docs/tongs/definitions.md +++ b/docs/tongs/definitions.md @@ -70,7 +70,7 @@ Mounts are opt-in **magic words**, never raw host paths. Only three are recogniz - `workspace[:/target][:mode]` — bind-mounts the session workspace, at `/workspace` unless an absolute `target` says otherwise (e.g. `workspace:ro`, `workspace:/code`, `workspace:/code:ro`). A custom target lets an image that expects its sources elsewhere be used unmodified (it does not set the working directory — the process still starts in the image's own `WORKDIR`). A target is refused unless it is an absolute path free of whitespace, and refused if it resolves to `/`, overlaps another of the tong's mounts, or overlaps a path the tong's own wiring occupies — the secret-delivery tmpfs at `/run/swarmforge` and the `/bin/sh` its wrapper execs (for a tong with secret references), or the docker socket (for a tong that mounts it). The workspace bind is paired with the same git-dir mounts the anvil gets from `swarmforge/gitguard.py`: read-only guards over the config and hooks the host's git obeys, and — when the workspace is a linked worktree or another checkout whose git dir lives outside it — that git dir at its own absolute path, which is where the checkout's `.git` pointer file says to look (without it, git inside the tong fails with "not a git repository"). The one mount the anvil gets and a tong does not is the restated worktree registration — a tong keeps the host's copy, so `git worktree list` inside one names the checkout by its host path (the [git guard](../git-guard.md) has the rest). When every `workspace` mount is `ro`, the ride-along git-dir mounts are forced read-only too. - `docker-socket[:mode]` — bind-mounts the host docker socket onto the same path inside the container (so it takes no target). This is full host docker control and is always called out explicitly in the workspace approval prompt; it is the grant a broker tong needs. -- `volume::/target[:mode]` — mounts a docker **named volume** at `target`, for state the tong keeps between containers (a model store, a cache). The name comes first and is at most 64 letters, digits, `.` and `-`, starting with a letter or digit; the target is required and held to the same rules as a `workspace` target, may not overlap the git dir that a `workspace` mount brings along, and a tong mounts each volume only once. Docker creates the volume on first use and seeds a new, empty one with whatever the image has at `target`. The launcher never removes a volume — not on teardown, and not when a `shared` tong is recreated because its definition changed — so the state outlives every container that mounts it. Remove one by hand with `docker volume rm ` (`docker volume ls --filter name=swarmforge-volume-` lists them). A `volume:` mount is private to its tong; it is unrelated to `interface.kind: volume`, which would share a volume with the anvil and is not wired up yet. +- `volume::/target[:mode]` — mounts a docker **named volume** at `target`, for state the tong keeps between containers (a model store, a cache). The name comes first and is at most 64 letters, digits, `.` and `-`, starting with a letter or digit; the target is required and held to the same rules as a `workspace` target, may not overlap the git dir that a `workspace` mount brings along, and a tong mounts each volume only once. Docker creates the volume on first use and seeds a new, empty one with whatever the image has at `target`. Only that first seeding happens: a later image's files at `target` never replace what the volume holds, so point `target` at a data directory, not at files the image ships. The launcher never removes a volume — not on teardown, and not when a `shared` tong is recreated because its definition changed — so the state outlives every container that mounts it. Remove one by hand with `docker volume rm ` (`docker volume ls --filter name=swarmforge-volume-` lists them). A `volume:` mount is private to its tong; it is unrelated to `interface.kind: volume`, which would share a volume with the anvil and is not wired up yet. ### Volume naming and scope @@ -78,9 +78,9 @@ The docker volume is named `swarmforge-volume-[-]_`. The hin - **org** — the org's tongs directory. This applies to both lifecycles. - **workspace** — the checkout's top-level path, with symlinks resolved. The path is the key: every session started anywhere in that checkout uses the volume, and a clone or worktree at a different path gets its own. A later checkout at the same path inherits it, even if it is a different repository, just as it inherits the approvals keyed by that path. Deleting a checkout leaves its volumes behind; remove them with `docker volume rm`. A workspace tong that mounts a volume may not be `shared`: its container would be reused by sessions in other checkouts, still mounting this checkout's volume. -- **user** and **repo** — this machine. A repo tong and the same-named user tong it replaces use one volume. +- **user** and **repo** — this machine, across every checkout. A repo tong and the same-named user tong it replaces use one volume. Anything such a tong copies from one workspace into its volume is visible to its sessions in every other checkout. -Two tongs therefore share a volume only when they have the same filename and the same scope. A tong never gets the volume of another org, a checkout at another path, or a differently named tong, and at 128 bits a definition cannot search its way onto another scope's name. Within its scope a volume belongs to the tong's name, not to one version of its definition: an edited definition, or a later checkout at the same path, reuses whatever data is already there. +Two tongs therefore share a volume only when they have the same filename and the same scope. A tong never gets the volume of another org, a checkout at another path, or a differently named tong, and at 128 bits a definition cannot search its way onto another scope's name. Within its scope a volume belongs to the tong's name, not to one version of its definition or to one session: an edited definition, or a later checkout at the same path, reuses whatever data is already there, and concurrent sessions of a `session` tong all mount it read-write at once. Keep state there that tolerates that, or that the program locks itself; a second database server on one data directory, for instance, fails to start. ## Resources From cd6f9b23dd93a3f2ed9cf798281f0b921acf0c0f Mon Sep 17 00:00:00 2001 From: CrypticSwarm Date: Fri, 9 Oct 2026 22:26:42 -0500 Subject: [PATCH 14/14] Trim the volume prose to one statement per rule The never-removed, workspace-shared-refusal and scope rules were each restated across several docstrings and doc pages. Keep one copy of each and cut the rest back. --- docs/tongs/README.md | 2 +- docs/tongs/approval.md | 2 +- docs/tongs/definitions.md | 14 +++++----- swarmforge/anvil/orchestrate.py | 14 +++------- swarmforge/tongs/approvals.py | 10 +++---- swarmforge/tongs/argv.py | 3 +- swarmforge/tongs/mounts.py | 49 +++++++++++---------------------- 7 files changed, 34 insertions(+), 60 deletions(-) diff --git a/docs/tongs/README.md b/docs/tongs/README.md index 1023993..fe03a58 100644 --- a/docs/tongs/README.md +++ b/docs/tongs/README.md @@ -30,7 +30,7 @@ The workspace layer is gated by [first-run approval](approval.md). ## Lifecycle - **`session`** — started with the anvil, torn down when it exits. Per-session isolation; the default for credential tongs. -- **`shared`** — long-lived, survives across anvil sessions (ollama-style). Started on first use, connected to each session's network via a network alias, and left running on teardown (no refcounting). A running `shared` container whose config-hash docker label still matches the current definition is reused untouched; a missing, stopped, or stale one is recreated automatically. A rotated secret behind an unchanged reference does **not** churn it — force a restart with `docker rm -f `. Recreating the container loses everything the tong wrote inside it, so a tong that has state to keep, such as a model store, declares a [`volume:` mount](definitions.md#mounts). Its named volume is mounted again by the next container and is never removed by the launcher (a workspace-sourced `shared` tong may not mount one; see [Volume naming and scope](definitions.md#volume-naming-and-scope)). A `shared` tong may not mount the `workspace` (it would leak one session's workspace into the next); use a `session` tong for per-workspace mounts. +- **`shared`** — long-lived, survives across anvil sessions (ollama-style). Started on first use, connected to each session's network via a network alias, and left running on teardown (no refcounting). A running `shared` container whose config-hash docker label still matches the current definition is reused untouched; a missing, stopped, or stale one is recreated automatically. A rotated secret behind an unchanged reference does **not** churn it — force a restart with `docker rm -f `. Recreating the container loses what the tong wrote inside it; state worth keeping, such as a model store, belongs in a [`volume:` mount](definitions.md#mounts). A `shared` tong may not mount the `workspace` (it would leak one session's workspace into the next); use a `session` tong for per-workspace mounts. ## Reference diff --git a/docs/tongs/approval.md b/docs/tongs/approval.md index 7cf1573..8982a89 100644 --- a/docs/tongs/approval.md +++ b/docs/tongs/approval.md @@ -3,7 +3,7 @@ The user, org, and repo layers are installed deliberately and are trusted; they skip the gate. A **workspace**-sourced tong (from a repo you cloned) could otherwise request your secrets, host mounts, the docker socket, or the host's GPUs simply by being present, so the launcher gates it: -- Before starting, it prints exactly what the tong requests — image, secret references, mounts, networks, docker-socket access, and GPU access — and asks you to approve. Control and formatting characters in those values are shown as escapes such as `\x1b` or `\n`, so a value cannot move the cursor, restyle, or hide any part of the summary. Every `tongs:` warning escapes them the same way, since many quote a tong's filename. Each `volume:` mount also gets its own `volume:` line naming its target and the docker volume it mounts, so you can check `docker volume ls` for leftover data first. A note under them warns that the data outlives the tong, is mounted by each of its concurrent sessions, and that any data already there, left by an earlier definition or an earlier checkout at this path, is reused. +- Before starting, it prints exactly what the tong requests — image, secret references, mounts, networks, docker-socket access, and GPU access — and asks you to approve. Control and formatting characters in those values are shown as escapes such as `\x1b` or `\n`, so a value cannot move the cursor, restyle, or hide any part of the summary. Every `tongs:` warning escapes them the same way. Each volume is listed with its target and docker volume name, with a note that its data persists and is shared by concurrent sessions. - Approval is keyed by workspace path (symlinks resolved, as for its [volumes](definitions.md#volume-naming-and-scope)) + tong name + a hash of the merged definition, stored in `~/.swarmforge/approvals.json`. Any change to the definition re-prompts. - The gate defaults to **No**, and a non-interactive stdin reads as No. A scripted `--no-prompt` run **fails closed** rather than auto-approving. - Approving `image: foo:latest` approves a moving target; **pinned digests are the recommended convention** for workspace tongs. diff --git a/docs/tongs/definitions.md b/docs/tongs/definitions.md index 30aa1f1..757af7b 100644 --- a/docs/tongs/definitions.md +++ b/docs/tongs/definitions.md @@ -34,7 +34,7 @@ The `interface:` block drives what gets injected into the anvil, how readiness i - **`mcp`** — an HTTP MCP server (the common case). Requires `port` and `name`; `transport` defaults to `http`. Injects per-harness MCP config pointing at `http://:/mcp` on the session network (`interface.path` overrides the `/mcp` suffix). TCP readiness probe by default. - **`port`** — a non-MCP network service. Requires `port`; optional `protocol` is informational. Injects `SWARMFORGE_TONG__HOST` (the canonical alias) and `SWARMFORGE_TONG__PORT` so the anvil composes its own connection string. TCP readiness probe by default. -- **`volume`** — a shared named volume, no network. Requires `volume` and `mountpoint`; readiness must be declared. The schema accepts it, but **the launcher does not wire it up yet** and refuses to start such a tong with a clear message. (A tong that only needs to keep its own state uses a `volume:` mount, described under Mounts below.) +- **`volume`** — a shared named volume, no network. Requires `volume` and `mountpoint`; readiness must be declared. The schema accepts it, but **the launcher does not wire it up yet** and refuses to start such a tong with a clear message. - **`none`** — a background side-effect with no anvil-facing surface. Injects nothing. Readiness must be declared. `` is the filename uppercased with hyphens turned into underscores (`github-creds` → `SWARMFORGE_TONG_GITHUB_CREDS_*`). @@ -70,17 +70,17 @@ Mounts are opt-in **magic words**, never raw host paths. Only three are recogniz - `workspace[:/target][:mode]` — bind-mounts the session workspace, at `/workspace` unless an absolute `target` says otherwise (e.g. `workspace:ro`, `workspace:/code`, `workspace:/code:ro`). A custom target lets an image that expects its sources elsewhere be used unmodified (it does not set the working directory — the process still starts in the image's own `WORKDIR`). A target is refused unless it is an absolute path free of whitespace, and refused if it resolves to `/`, overlaps another of the tong's mounts, or overlaps a path the tong's own wiring occupies — the secret-delivery tmpfs at `/run/swarmforge` and the `/bin/sh` its wrapper execs (for a tong with secret references), or the docker socket (for a tong that mounts it). The workspace bind is paired with the same git-dir mounts the anvil gets from `swarmforge/gitguard.py`: read-only guards over the config and hooks the host's git obeys, and — when the workspace is a linked worktree or another checkout whose git dir lives outside it — that git dir at its own absolute path, which is where the checkout's `.git` pointer file says to look (without it, git inside the tong fails with "not a git repository"). The one mount the anvil gets and a tong does not is the restated worktree registration — a tong keeps the host's copy, so `git worktree list` inside one names the checkout by its host path (the [git guard](../git-guard.md) has the rest). When every `workspace` mount is `ro`, the ride-along git-dir mounts are forced read-only too. - `docker-socket[:mode]` — bind-mounts the host docker socket onto the same path inside the container (so it takes no target). This is full host docker control and is always called out explicitly in the workspace approval prompt; it is the grant a broker tong needs. -- `volume::/target[:mode]` — mounts a docker **named volume** at `target`, for state the tong keeps between containers (a model store, a cache). The name comes first and is at most 64 letters, digits, `.` and `-`, starting with a letter or digit; the target is required and held to the same rules as a `workspace` target, may not overlap the git dir that a `workspace` mount brings along, and a tong mounts each volume only once. Docker creates the volume on first use and seeds a new, empty one with whatever the image has at `target`. Only that first seeding happens: a later image's files at `target` never replace what the volume holds, so point `target` at a data directory, not at files the image ships. The launcher never removes a volume — not on teardown, and not when a `shared` tong is recreated because its definition changed — so the state outlives every container that mounts it. Remove one by hand with `docker volume rm ` (`docker volume ls --filter name=swarmforge-volume-` lists them). A `volume:` mount is private to its tong; it is unrelated to `interface.kind: volume`, which would share a volume with the anvil and is not wired up yet. +- `volume::/target[:mode]` — mounts a docker **named volume** at `target`, for state the tong keeps between containers (a model store, a cache). The name comes first and is at most 64 letters, digits, `.` and `-`, starting with a letter or digit; the target is required and held to the same rules as a `workspace` target, may not overlap the git dir that a `workspace` mount brings along, and a tong mounts each volume only once. Docker creates the volume on first use and seeds a new, empty one with whatever the image has at `target`. Only that first seeding happens: a later image's files at `target` never replace what the volume holds, so point `target` at a data directory, not at files the image ships. The launcher never removes a volume; remove one by hand with `docker volume rm ` (`docker volume ls --filter name=swarmforge-volume-` lists them). A `volume:` mount is private to its tong; it is unrelated to `interface.kind: volume`, which would share a volume with the anvil and is not wired up yet. ### Volume naming and scope -The docker volume is named `swarmforge-volume-[-]_`. The hint is the tong's filename reduced to the characters docker allows and cut to 32 of them, and is there only to be read in `docker volume ls`. The digest is the volume's identity: the first 32 hex digits of a SHA-256 over the tong's exact filename and the scope it came from. A volume name never contains `_` and the digest has a fixed width, so the last `_` always separates the volume name. The scope depends on the tong's layer: +The docker volume is named `swarmforge-volume-[-]_`. The hint is the tong's filename made docker-safe and cut to 32 characters, only for reading. The digest, a 128-bit SHA-256 prefix over the tong's exact filename and its scope, is the identity, so two tongs share a volume only with the same filename and scope. The scope depends on the tong's layer: - **org** — the org's tongs directory. This applies to both lifecycles. -- **workspace** — the checkout's top-level path, with symlinks resolved. The path is the key: every session started anywhere in that checkout uses the volume, and a clone or worktree at a different path gets its own. A later checkout at the same path inherits it, even if it is a different repository, just as it inherits the approvals keyed by that path. Deleting a checkout leaves its volumes behind; remove them with `docker volume rm`. A workspace tong that mounts a volume may not be `shared`: its container would be reused by sessions in other checkouts, still mounting this checkout's volume. -- **user** and **repo** — this machine, across every checkout. A repo tong and the same-named user tong it replaces use one volume. Anything such a tong copies from one workspace into its volume is visible to its sessions in every other checkout. +- **workspace** — the checkout's top-level path, with symlinks resolved. A clone or worktree at another path gets its own volume; a later checkout at the same path, even of a different repository, inherits it, as it does the approvals. Deleting a checkout leaves its volumes behind. A workspace tong that mounts a volume may not be `shared`: its container would be reused by other checkouts, still mounting this one's volume. +- **user** and **repo** — this machine, across every checkout, so anything such a tong copies from one workspace is visible in the others. A repo tong and the same-named user tong it replaces use one volume. -Two tongs therefore share a volume only when they have the same filename and the same scope. A tong never gets the volume of another org, a checkout at another path, or a differently named tong, and at 128 bits a definition cannot search its way onto another scope's name. Within its scope a volume belongs to the tong's name, not to one version of its definition or to one session: an edited definition, or a later checkout at the same path, reuses whatever data is already there, and concurrent sessions of a `session` tong all mount it read-write at once. Keep state there that tolerates that, or that the program locks itself; a second database server on one data directory, for instance, fails to start. +A volume belongs to the tong's name, not to one definition or one session: an edited definition reuses its data, and concurrent sessions of a `session` tong mount it read-write at once. Keep state there that tolerates that, or that the program locks itself. ## Resources @@ -117,4 +117,4 @@ resources: gpus: all ``` -The anvil gets `SWARMFORGE_TONG_MODELS_HOST=models` and `SWARMFORGE_TONG_MODELS_PORT=11434`, and the store lives in the volume `swarmforge-volume-models-_store`. The tong is named `models` rather than `ollama` because `make run_ollama` already answers to `ollama` on the same network; a tong of that name would clash with it. +The anvil gets `SWARMFORGE_TONG_MODELS_HOST=models` and `SWARMFORGE_TONG_MODELS_PORT=11434`, and the store lives in the volume `swarmforge-volume-models-_store`. It is named `models` because `make run_ollama` already answers to `ollama` on that network. diff --git a/swarmforge/anvil/orchestrate.py b/swarmforge/anvil/orchestrate.py index e854c24..9d6fe09 100644 --- a/swarmforge/anvil/orchestrate.py +++ b/swarmforge/anvil/orchestrate.py @@ -79,10 +79,8 @@ def unsupported_tong_reasons(merged): workspace into it would expose that workspace to every later session that reuses the container (a `session` tong is the right home for a per-workspace mount); - * a workspace-sourced `shared` tong that mounts a `volume` -- the volume - is scoped to one checkout (see `tongs.volume_scope`) but the container is - not, so a session in another checkout shipping the same definition would - reuse it, volume and all. + * a workspace-sourced `shared` tong that mounts a `volume` -- its volume + is scoped to one checkout but its container is not. A refused tong is reported rather than started half-wired. Returns a list of human-readable reason strings (empty == every discovered tong is startable). @@ -224,8 +222,6 @@ def _ensure_shared_tong(docker, name, defn, *, container, network, alias, so the same long-lived container is reused across sessions while the definition is stable -- and deciding to reuse one never runs a secret-provider CLI, so a rotated secret behind an unchanged reference does not churn it. - The recreated container mounts the same named volumes, so a tong's - persistent state outlives it. """ state = docker.inspect_state(container) if state and state["running"] and state["label"] == label_hash: @@ -313,10 +309,8 @@ def run_with_tongs(merged, anvil_cmd, opts, *, docker, providers=None, On exit -- including SIGINT, and SIGHUP/SIGTERM once `cli.main` makes them raise -- the `session` tongs and the per-session network are torn down (and the connected `shared` tongs disconnected) while the long-lived `shared` - tongs are left running. No tong's named volumes are ever removed: they are - the state a tong keeps between containers. Teardown runs inside the caller's - `teardown_guard()`, which keeps signals from cutting it short; the default - guards nothing. + tongs are left running. Teardown runs inside the caller's `teardown_guard()`, + which keeps signals from cutting it short; the default guards nothing. Returns the anvil's exit code. Raises `OrchestrationError` if a tong never becomes ready (the anvil does not run against a half-up environment) or a diff --git a/swarmforge/tongs/approvals.py b/swarmforge/tongs/approvals.py index a448e05..3ba552c 100644 --- a/swarmforge/tongs/approvals.py +++ b/swarmforge/tongs/approvals.py @@ -32,12 +32,10 @@ def privilege_summary(defn, name=None, volume_scope=None): """Structured summary of what a definition asks for, for the approval gate. Gathers the privileges a reviewer must see before approving a - workspace-sourced tong: image, secret references, mounts (and the - persistent volumes among them), networks, docker-socket access, and host - GPU access. Each volume is `{"name", "target", "docker"}`, where `docker` is - the docker volume the tong `name` mounts from `volume_scope`, or None - without both. Rendering and prompting are the caller's job; this just - assembles the facts. + workspace-sourced tong: image, secret references, mounts, volumes, networks, + docker-socket access, and host GPU access. Each volume is `{"name", "target", + "docker"}`; `docker` is None unless `name` and `volume_scope` are given. + Rendering and prompting are the caller's job; this just assembles the facts. """ return { "image": defn.get("image"), diff --git a/swarmforge/tongs/argv.py b/swarmforge/tongs/argv.py index 9f9e4bd..019b75c 100644 --- a/swarmforge/tongs/argv.py +++ b/swarmforge/tongs/argv.py @@ -166,8 +166,7 @@ def tong_run_argv( `workspace` mount -- see swarmforge.anvil); they are appended after the definition's own mounts, mirroring how the Makefile orders the anvil's. - `volume_scope` is the `(class, path)` pair the orchestrator picked for the - tong's `volume:` mounts from the layer it came from (see `tong_volume_name`). + `volume_scope` names the tong's `volume:` mounts (see `volume_scope`). """ if entrypoint is None and command is None: entrypoint, command = declared_run_override(defn) diff --git a/swarmforge/tongs/mounts.py b/swarmforge/tongs/mounts.py index 4e1b112..6873a78 100644 --- a/swarmforge/tongs/mounts.py +++ b/swarmforge/tongs/mounts.py @@ -85,8 +85,7 @@ def parse_mount(mount, words=MOUNT_WORDS): which is always last -- so `workspace`, `workspace:ro`, `workspace:/work` and `workspace:/work:ro` are all valid. `target`/`mode` are None when not declared; the caller supplies the default target for its magic word. The `volume` word - is spelled `volume::/target[:mode]`: its name comes first and its target - is required, since a named volume has no natural place to land. `volume` is + is spelled `volume::/target[:mode]`, its target required; `volume` is that name, and None for every other word. The word is checked against `words` before the rest of the entry, since a mount @@ -182,9 +181,8 @@ def mount_destination(word, target, socket_path=DEFAULT_DOCKER_SOCKET): The declared target when there is one, otherwise the word's default: /workspace for the workspace, its own host path for the socket (that is where a docker - client looks for it). A volume has no default, so it needs its target. Raises - `ValueError` for a targetless volume and for any other word, so a magic word - added without a default cannot inherit the socket's. + client looks for it); a volume has none. Raises `ValueError` for a targetless + volume and any other word, so a new word cannot inherit the socket's default. """ if word == WORKSPACE_MOUNT: return normalize_mount_target(target or DEFAULT_WORKSPACE_MOUNT_TARGET) @@ -225,9 +223,8 @@ def overlapping_mount_error(mount, destination, volume, placed): `volume` is the name `parse_mount` read from `mount`, and `placed` the `(mount, destination, volume)` triples accepted so far. Docker refuses two binds onto one destination outright, and creates the inner mountpoint of nested ones inside - the outer bind -- inside the user's workspace, for a workspace bind. One volume - is mounted once: a second target for the same data, perhaps with another - access mode, is far likelier a mistake than a need. + the outer bind -- inside the user's workspace, for a workspace bind. A volume + mounts only once. """ for other, other_destination, other_volume in placed: if _targets_overlap(destination, other_destination): @@ -241,17 +238,9 @@ def overlapping_mount_error(mount, destination, volume, placed): def tong_volume_name(tong_name, volume, scope): """The docker named volume behind one tong's `volume:` mount. - `swarmforge-volume-[-]_`. The digest is the volume's - identity: the first `VOLUME_DIGEST_LENGTH` hex digits of a SHA-256 over the - scope and the raw tong name, where `scope` is a `(class, path)` pair -- an - `ORG_VOLUME_SCOPE` or `WORKSPACE_VOLUME_SCOPE` with its canonical directory, - or `(LOCAL_VOLUME_SCOPE, None)` for this machine's own layers. Two tongs - share a volume only from one scope under exactly one name. The hint is the - tong name sanitized for docker and cut to `VOLUME_HINT_LIMIT`, there to be - read in `docker volume ls`; it decides nothing, since sanitizing can turn two - names into one. Volume names - exclude `_` and the digest has a fixed width, so the last `_` splits the - volume back off. + `swarmforge-volume-[-]_`. The digest, over `scope` + (from `volume_scope`) and the raw tong name, is the identity; the hint is the + sanitized name, only for reading in `docker volume ls`. """ scope_class, scope_path = scope key = json.dumps([scope_class, scope_path, tong_name]) @@ -264,16 +253,11 @@ def tong_volume_name(tong_name, volume, scope): def volume_scope(layer, org_tongs_dir=None, workspace=None): """The `(class, path)` scope that names the volumes of a tong from `layer`. - An org tong's volumes belong to its org's tongs directory (canonical, as - `org_scope_token` reads it); a workspace tong's to the checkout's top level - `workspace`, keyed by `workspace_key` like its approvals, so every session in - that checkout, from any subdirectory, shares them -- as does any later - checkout at that path. The user and repo layers are this machine's own - and share `(LOCAL_VOLUME_SCOPE, None)`, so a repo tong and the same-named - user tong it replaces in the merge use one volume. None for an org or - workspace tong with no directory to scope it by, which `tong_mount_specs` - refuses rather than falling back to the local scope. Raises `ValueError` for - an unknown layer. + Org: its tongs directory, canonical as `org_scope_token` reads it. + Workspace: the checkout's `workspace_key`, like its approvals. User and + repo: `(LOCAL_VOLUME_SCOPE, None)`. None when the org or workspace directory + is unknown, which `tong_mount_specs` refuses rather than falling back to the + local scope. Raises `ValueError` for an unknown layer. """ if layer == ORG: return (ORG_VOLUME_SCOPE, canonical_path(org_tongs_dir)) if org_tongs_dir else None @@ -294,10 +278,9 @@ def tong_mount_specs(defn, workspace, socket_path=DEFAULT_DOCKER_SOCKET, bind-mounts the host docker socket onto the same path it has on the host; `volume::/target[:mode]` mounts the named volume `tong_volume_name(tong_name, name, volume_scope)`, which docker creates on - first use and never removes with the container. A volume may not overlap - the destination of any of `extra_mount_specs`, the `-v` values that ride - along with a workspace mount (see `tong_run_argv`): docker would create - their mountpoints inside the volume, where they outlive the container. + first use and the launcher never removes. A volume may not overlap one of + `extra_mount_specs` (see `tong_run_argv`), or docker would leave their + mountpoints inside it. Raises `ValueError` for a non-string entry, a malformed mount, an unusable or colliding destination, a `workspace` mount when no workspace path is known, or a `volume` mount without a `tong_name` and `volume_scope` to name it by -- a