Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion dokimi/docs/architecture.md
Original file line number Diff line number Diff line change
Expand Up @@ -73,7 +73,7 @@ points up:
| tracks | `runner`, `report`, `subjects`, `novelty`, `harnesses`, `context`, `repos`, `explore`, `tool_preference`, `external`, `dashboard`, `orchestrator`, `datasets_export`, `coverage`, `task_formats`, and the checkers `config_check`, `schema_check`, `schema_compat` |
| evaluation | `eval`, `personas`, `questions`, `treatments`, `plugins` |
| components | `ingest`, `metrics`, `gate`, `mcp`, `skills`, `agent_profiles` |
| foundation | `__init__` (the `dokimi` package itself), `_version`, `artifacts`, `records`, `llm`, `process`, `caching`, `config`, `registry`, `intents`, `defaults`, `console`, `utils`, `metrics/usage.py` |
| foundation | `__init__` (the `dokimi` package itself), `_version`, `artifacts`, `records`, `llm`, `process`, `container_policy`, `caching`, `config`, `registry`, `intents`, `defaults`, `console`, `utils`, `metrics/usage.py` |

So `report` (a track) can call `gate` (a component), and `gate` cannot import
`report`. The sections below follow the tracks, not the layers.
Expand Down
58 changes: 58 additions & 0 deletions dokimi/dokimi/container_policy.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,58 @@
# Copyright (C) 2026 Intel Corporation
# SPDX-License-Identifier: Apache-2.0

"""The hardening and resource flags of a ``docker run``, spelled once.

Two call sites start containers with the docker CLI: :class:`dokimi.repos.container.DockerRunner`
and the docker backend of :mod:`dokimi.metrics.example_runner`. They use different subsets of
the same flags, and each has its own argv order. :class:`ContainerPolicy` owns how every flag
is spelled. A call site sets the fields it wants and splices the groups in the order its argv
has always had; that order is pinned by ``tests/test_container_argv_characterization.py``.

The builder chooses nothing and validates nothing: a field left at its default adds no flag,
so each call site opts in to what it runs with.
"""

from __future__ import annotations

from dataclasses import dataclass


@dataclass(frozen=True)
class ContainerPolicy:
Comment thread
antonwolfy marked this conversation as resolved.
Resolved
network: str | None = None # --network; None adds no flag
init: bool = False # --init: a daemonizing build server is reaped, its cache writes are kept
no_new_privileges: bool = False # --security-opt no-new-privileges: a setuid binary cannot gain root
cap_drop_all: bool = False # --cap-drop ALL
pids_limit: int | None = None # --pids-limit: a ceiling against a fork bomb
cpus: float | None = None # --cpus
memory_mb: int | None = None # --memory <n>m
read_only: bool = False # --read-only: the container's root filesystem

def init_argv(self) -> list[str]:
return ["--init"] if self.init else []

def hardening_argv(self) -> list[str]:
out: list[str] = []
if self.no_new_privileges:
out += ["--security-opt", "no-new-privileges"]
if self.cap_drop_all:
out += ["--cap-drop", "ALL"]
return out

def pids_argv(self) -> list[str]:
return [] if self.pids_limit is None else ["--pids-limit", str(self.pids_limit)]

def network_argv(self) -> list[str]:
return [] if self.network is None else ["--network", self.network]

def resource_argv(self) -> list[str]:
out: list[str] = []
if self.cpus is not None:
out += ["--cpus", str(self.cpus)]
if self.memory_mb is not None:
out += ["--memory", f"{self.memory_mb}m"]
return out

def read_only_argv(self) -> list[str]:
return ["--read-only"] if self.read_only else []
12 changes: 7 additions & 5 deletions dokimi/dokimi/metrics/example_runner.py
Original file line number Diff line number Diff line change
Expand Up @@ -30,6 +30,7 @@
from pathlib import Path

from dokimi import process
from dokimi.container_policy import ContainerPolicy


# Supported languages -> (file suffix, interpreter argv prefix for a file path).
Expand Down Expand Up @@ -191,13 +192,14 @@ def _run_docker(lang: str, code: str, policy: ExecutionPolicy,
with tempfile.TemporaryDirectory() as tmp:
script = Path(tmp) / f"snippet{suffix}"
script.write_text(code, encoding="utf-8")
flags = ContainerPolicy(network="none", cpus=policy.cpus, memory_mb=policy.mem_mb, pids_limit=64,
read_only=True)
cmd = [
"docker", "run", "--rm",
"--network", "none",
"--cpus", str(policy.cpus),
"--memory", f"{policy.mem_mb}m",
"--pids-limit", "64",
"--read-only",
*flags.network_argv(),
*flags.resource_argv(),
*flags.pids_argv(),
*flags.read_only_argv(),
"-v", f"{tmp}:/work:ro",
policy.docker_image,
"timeout", str(policy.timeout),
Expand Down
7 changes: 5 additions & 2 deletions dokimi/dokimi/repos/container.py
Original file line number Diff line number Diff line change
Expand Up @@ -40,6 +40,7 @@
from typing import Protocol

from dokimi import process
from dokimi.container_policy import ContainerPolicy

RUN_DIR = "/eval/run"
WORKSPACE = f"{RUN_DIR}/workspace"
Expand Down Expand Up @@ -124,8 +125,10 @@ def __init__(self, docker: str = "docker", runner: process.Runner = process.run)
def argv(self, spec: ContainerSpec) -> list[str]:
if spec.network not in NETWORKS:
raise ContainerError(f"unknown network {spec.network!r} (known: {', '.join(NETWORKS)})")
out = [self.docker, "run", "--rm", "--init", "--security-opt", "no-new-privileges", "--cap-drop", "ALL",
"--pids-limit", str(PIDS_LIMIT), "--network", spec.network, "-w", spec.workdir]
flags = ContainerPolicy(network=spec.network, init=True, no_new_privileges=True, cap_drop_all=True,
pids_limit=PIDS_LIMIT)
out = [self.docker, "run", "--rm", *flags.init_argv(), *flags.hardening_argv(), *flags.pids_argv(),
*flags.network_argv(), "-w", spec.workdir]
if spec.name:
out += ["--name", spec.name]
user = spec.user or host_user()
Expand Down
164 changes: 164 additions & 0 deletions dokimi/tests/test_container_argv_characterization.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,164 @@
# Copyright (C) 2026 Intel Corporation
# SPDX-License-Identifier: Apache-2.0

"""Characterization of the ``docker run`` argv each call site builds today.

These pin the exact argv, flag order included, of the two container call sites that
the container policy builder serves: :class:`dokimi.repos.container.DockerRunner` and
the docker backend of :mod:`dokimi.metrics.example_runner`. They were written against
the code before the builder existed, and must keep passing through the refactor: a
change to any expected list here is an argv change, which is a separate decision.
"""

from __future__ import annotations

from pathlib import Path

import pytest

from dokimi.metrics.example_runner import EXECUTORS, ExecutionPolicy, run_example
from dokimi.process import FakeRunner, reply
from dokimi.repos import container
from dokimi.repos.container import (
CONTENT_DIR,
WORKSPACE,
ContainerError,
ContainerSpec,
DockerRunner,
Mount,
)

# --- repos/container.py: DockerRunner.argv -----------------------------------------


def _no_host_user(monkeypatch: pytest.MonkeyPatch, uid_gid: str | None) -> None:
monkeypatch.setattr(container, "host_user", lambda: uid_gid)


def test_repo_argv_full_spec_pins_every_flag_in_order(monkeypatch):
_no_host_user(monkeypatch, None)
spec = ContainerSpec(
image="img",
argv=["python3", "x"],
mounts=[Mount(Path("/h/run"), "/eval/run"), Mount(Path("/h/content"), CONTENT_DIR, read_only=True)],
env={"HOME": "/eval/run/home"},
pass_env=("AWS_BEARER_TOKEN_BEDROCK",),
network="none",
name="dokimi-x",
user="1000:1000",
)
assert DockerRunner().argv(spec) == [
"docker", "run", "--rm", "--init",
"--security-opt", "no-new-privileges", "--cap-drop", "ALL",
"--pids-limit", "32768",
"--network", "none",
"-w", WORKSPACE,
"--name", "dokimi-x",
"--user", "1000:1000",
"--mount", "type=bind,src=/h/run,dst=/eval/run",
"--mount", f"type=bind,src=/h/content,dst={CONTENT_DIR},readonly",
"-e", "HOME=/eval/run/home",
"-e", "AWS_BEARER_TOKEN_BEDROCK",
"--entrypoint", "", "img", "python3", "x",
]


@pytest.mark.parametrize("network", ["none", "bridge", "host"])
def test_repo_argv_network_mode_is_the_only_network_difference(monkeypatch, network):
_no_host_user(monkeypatch, None)
argv = DockerRunner().argv(ContainerSpec(image="img", argv=["make"], network=network))
assert argv == [
"docker", "run", "--rm", "--init",
"--security-opt", "no-new-privileges", "--cap-drop", "ALL",
"--pids-limit", "32768",
"--network", network,
"-w", WORKSPACE,
"--entrypoint", "", "img", "make",
]


def test_repo_argv_takes_the_host_uid_gid_when_the_spec_has_no_user(monkeypatch):
_no_host_user(monkeypatch, "4242:4343")
argv = DockerRunner().argv(ContainerSpec(image="img", argv=["true"], network="none"))
i = argv.index("--user")
assert argv[i - 2:i + 2] == ["-w", WORKSPACE, "--user", "4242:4343"]


def test_repo_argv_omits_user_when_there_is_no_posix_uid(monkeypatch):
_no_host_user(monkeypatch, None)
assert "--user" not in DockerRunner().argv(ContainerSpec(image="img", argv=["true"]))


def test_repo_argv_unknown_network_is_refused():
with pytest.raises(ContainerError, match="unknown network 'wide-open'"):
DockerRunner().argv(ContainerSpec(image="i", argv=[], network="wide-open"))


def test_repo_argv_writable_mount_has_no_readonly_suffix(monkeypatch):
_no_host_user(monkeypatch, None)
argv = DockerRunner().argv(ContainerSpec(image="i", argv=[], mounts=[Mount(Path("/h/w"), "/eval/w")]))
assert "type=bind,src=/h/w,dst=/eval/w" in argv
assert "readonly" not in " ".join(argv)


# --- metrics/example_runner.py: docker backend argv ------------------------------

_TMP_MOUNT = "<tmp>:/work:ro"


def _docker_cmd(policy: ExecutionPolicy, lang: str = "python") -> tuple[list[str], float]:
"""The argv and timeout of one docker-backend run, with the per-run temp dir masked."""
runner = FakeRunner(lambda call: reply())
res = run_example(lang, "print(1)", policy, runner)
assert res.passed is True
(call,) = runner.calls
argv = [_TMP_MOUNT if a.endswith(":/work:ro") else a for a in call.argv]
return argv, call.timeout


def test_example_docker_argv_default_policy_pins_every_flag_in_order():
argv, timeout = _docker_cmd(ExecutionPolicy(backend="docker", timeout=7))
assert argv == [
"docker", "run", "--rm",
"--network", "none",
"--cpus", "1.0",
"--memory", "512m",
"--pids-limit", "64",
"--read-only",
"-v", _TMP_MOUNT,
"python:3.11-slim",
"timeout", "7",
*EXECUTORS["python"][1], "/work/snippet.py",
]
assert timeout == 37 # the snippet's timeout plus the docker CLI's grace


def test_example_docker_argv_resource_values_come_from_the_policy():
policy = ExecutionPolicy(backend="docker", timeout=3, docker_image="repo/img:1", mem_mb=1024, cpus=2.5)
argv, _ = _docker_cmd(policy, lang="bash")
assert argv == [
"docker", "run", "--rm",
"--network", "none",
"--cpus", "2.5",
"--memory", "1024m",
"--pids-limit", "64",
"--read-only",
"-v", _TMP_MOUNT,
"repo/img:1",
"timeout", "3",
*EXECUTORS["bash"][1], "/work/snippet.sh",
]


def test_example_docker_argv_never_sets_network_from_the_policy():
# The docker backend has no network mode: it is always "none".
argv, _ = _docker_cmd(ExecutionPolicy(backend="docker", timeout=5))
assert argv[argv.index("--network") + 1] == "none"


def test_example_docker_argv_has_no_init_no_cap_drop_no_new_privileges():
# Recorded gap: this site carries none of the repos/container.py hardening today.
# Adding it is an argv change and is a separate follow-up, so this test pins the absence.
argv, _ = _docker_cmd(ExecutionPolicy(backend="docker", timeout=5))
for flag in ("--init", "--cap-drop", "no-new-privileges"):
assert flag not in argv
67 changes: 67 additions & 0 deletions dokimi/tests/test_container_policy.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,67 @@
# Copyright (C) 2026 Intel Corporation
# SPDX-License-Identifier: Apache-2.0

"""The defaults and opt-out branches of :class:`dokimi.container_policy.ContainerPolicy`.

The two call sites exercise one field combination each, so each field is pinned here alone.
"""

from __future__ import annotations

import pytest

from dokimi.container_policy import ContainerPolicy

_GROUPS = (
"init_argv",
"hardening_argv",
"pids_argv",
"network_argv",
"resource_argv",
"read_only_argv",
)


@pytest.mark.parametrize("group", _GROUPS)
def test_default_policy_adds_no_flag(group):
assert getattr(ContainerPolicy(), group)() == []


@pytest.mark.parametrize(
("policy", "group", "expected"),
[
(ContainerPolicy(init=True), "init_argv", ["--init"]),
(
ContainerPolicy(no_new_privileges=True),
"hardening_argv",
["--security-opt", "no-new-privileges"],
),
(ContainerPolicy(cap_drop_all=True), "hardening_argv", ["--cap-drop", "ALL"]),
(ContainerPolicy(pids_limit=64), "pids_argv", ["--pids-limit", "64"]),
(ContainerPolicy(pids_limit=0), "pids_argv", ["--pids-limit", "0"]),
(ContainerPolicy(network="none"), "network_argv", ["--network", "none"]),
(ContainerPolicy(cpus=1), "resource_argv", ["--cpus", "1"]),
(ContainerPolicy(cpus=1.5), "resource_argv", ["--cpus", "1.5"]),
(ContainerPolicy(cpus=0.0), "resource_argv", ["--cpus", "0.0"]),
(ContainerPolicy(memory_mb=512), "resource_argv", ["--memory", "512m"]),
(ContainerPolicy(memory_mb=0), "resource_argv", ["--memory", "0m"]),
(ContainerPolicy(read_only=True), "read_only_argv", ["--read-only"]),
],
)
def test_each_field_alone_yields_exactly_its_flag(policy, group, expected):
assert getattr(policy, group)() == expected
assert all(getattr(policy, g)() == [] for g in _GROUPS if g != group)


def test_hardening_flags_combine_in_order():
policy = ContainerPolicy(no_new_privileges=True, cap_drop_all=True)
assert policy.hardening_argv() == ["--security-opt", "no-new-privileges", "--cap-drop", "ALL"]


def test_resource_flags_combine_in_order():
assert ContainerPolicy(cpus=2, memory_mb=256).resource_argv() == [
"--cpus",
"2",
"--memory",
"256m",
]
2 changes: 1 addition & 1 deletion dokimi/tests/test_import_architecture.py
Original file line number Diff line number Diff line change
Expand Up @@ -69,7 +69,7 @@
# process chokepoints, console output.
("foundation", (
"dokimi", "dokimi._version", "dokimi.artifacts", "dokimi.caching", "dokimi.config",
"dokimi.console", "dokimi.defaults", "dokimi.intents", "dokimi.llm", "dokimi.llm_pricing",
"dokimi.console", "dokimi.container_policy", "dokimi.defaults", "dokimi.intents", "dokimi.llm", "dokimi.llm_pricing",
"dokimi.process", "dokimi.records", "dokimi.registry", "dokimi.utils",
# Token/cost record the LLM chokepoint fills; it lives under metrics/ but
# depends on nothing and is read by every LLM caller.
Expand Down
Loading