diff --git a/.github/fork-ci-tests.txt b/.github/fork-ci-tests.txt index d7815e726cf73..400503c928a5f 100644 --- a/.github/fork-ci-tests.txt +++ b/.github/fork-ci-tests.txt @@ -63,6 +63,7 @@ fork-kit/heavy/tests/sweep.test.py # A real systemd user manager (the step starts one for the runner user). fork-kit/heavy/tests/heavy.test.py run by .github/workflows/fork-tests.yml fork-kit/pc-agents/tests/check.test.py +fork-kit/pc-agents/tests/setup.test.py # Upstream's tests, run where fork-tests.yml follows upstream's ci.yml. .github/scripts/check-nightly-release.test.cjs run by .github/workflows/fork-tests.yml diff --git a/fork-kit/README.md b/fork-kit/README.md index 0f3680cd3d5c3..508cf32308c36 100644 --- a/fork-kit/README.md +++ b/fork-kit/README.md @@ -13,12 +13,13 @@ Every kit follows the same pattern: - safe to run again, and safe to stop at any step; - an uninstall that leaves nothing. -| Kit | What | -| ---------------------------- | ------------------------------------------------------------------------------------------------------- | -| [desk](desk/README.md) | A desktop's side of a fork fleet desk pool: runner users, walls, desk agent; one folder (Linux, WSL2) | -| [gh-apps](gh-apps/README.md) | GitHub App tokens for agents through `gh-as`, without giving agents the keys (Linux) | -| [golive](golive/README.md) | Move a T3 boot service to a new build with a cold copy first; core and `check` so far (Linux) | -| [heavy](heavy/README.md) | `heavy `: run builds and tests in a capped slice, stop what they leave behind (Linux, systemd) | +| Kit | What | +| -------------------------------- | ------------------------------------------------------------------------------------------------------- | +| [desk](desk/README.md) | A desktop's side of a fork fleet desk pool: runner users, walls, desk agent; one folder (Linux, WSL2) | +| [gh-apps](gh-apps/README.md) | GitHub App tokens for agents through `gh-as`, without giving agents the keys (Linux) | +| [golive](golive/README.md) | Move a T3 boot service to a new build with a cold copy first; core and `check` so far (Linux) | +| [heavy](heavy/README.md) | `heavy `: run builds and tests in a capped slice, stop what they leave behind (Linux, systemd) | +| [pc-agents](pc-agents/README.md) | Run T3's agents on a Windows PC in a walled WSL2 distro, checked before every start (WSL2) | [common](common/README.md) is code the kits share, copied into each kit's version folder: the loader of the one private machine file (`/etc/fork-kit/machine.json`) where the facts of one machine live. diff --git a/fork-kit/common/machine.example.json b/fork-kit/common/machine.example.json index 6bd84bf8c35f9..629e8ed54aae6 100644 --- a/fork-kit/common/machine.example.json +++ b/fork-kit/common/machine.example.json @@ -31,6 +31,19 @@ "quietWaitSeconds": 7200, "stopLimitSeconds": 1800 }, + "pc-agents": { + "organization": "Example home lab", + "t3Version": "0.0.30", + "t3ReleaseBaseUrl": "https://github.com/example/t3code/releases/download", + "port": 3773, + "maxRuntimeMode": "full-access", + "lockedSettings": { "agentSlots.enabled": true, "agentSlots.maxPerMachine": 2 }, + "memoryMax": "8G", + "cpuQuota": "400%", + "nameservers": ["1.1.1.1", "9.9.9.9"], + "peers": ["100.101.102.103/32"], + "tailscaleTag": "tag:pc-agents" + }, "desk": { "pool": "pc", "host": "fleet-host.example", diff --git a/fork-kit/pc-agents/README.md b/fork-kit/pc-agents/README.md index 73213ec8991d4..3fb5f6048ed5b 100644 --- a/fork-kit/pc-agents/README.md +++ b/fork-kit/pc-agents/README.md @@ -32,14 +32,17 @@ so the agent cannot undo it: - no Windows drive is mounted; - the loaded firewall table is the one from the kit's `firewall.nft`: `pc-agents load-firewall` saves the file's hash and what the kernel then lists, and any later change, or an older file left loaded, differs; -- the default gateway (the Windows host) is in a blocked range, and no `allow` or `peers` entry lets it through. +- the default gateway (the Windows host) is in a blocked range, and no `allow` or `peers` entry lets it through; +- no T3 runs that started before setup's last change: while setup's marker `/etc/pc-agents/restart-pending` is there + (see "Runs again" below), a T3 process started before the time in it fails this wall, and so does a marker or a + start time it cannot read. So the 2-minute check stops a T3 that a stopped or failed setup left running with old + files, and `pc-agents as-agent` refuses meanwhile. Setup's own first check skips it. The fleet's `mounts`, `sockets` and `userns` walls are about the CI launcher's job units, so they are not reported here; the unit's own sandbox is checked instead (it must be the kit's file, byte for byte, with no drop-in). -Linux (WSL2, systemd, Debian or Ubuntu) only. Python 3 standard library only. This folder has the walls and their -check so far; `pc-agents setup`, which writes them into a fresh distro, and its owner steps are still to come. +Linux (WSL2, systemd, Debian or Ubuntu) only. Python 3 standard library only. ## Config @@ -90,11 +93,119 @@ refuses what is cheap to tell (a field of a provider instance, an object deeper two settings, a blank `organization`); T3 refuses the rest, like a field inside a setting that can be empty (lock the whole setting), says why in its log (`journalctl -u t3code-agents`) and does not start. T3 exits 1 on a policy it cannot apply, the same code as any other failure, so the unit cannot tell it apart (`RestartPreventExitStatus=`); -it gives up after 5 starts in 10 minutes instead (`StartLimitBurst=5`). +it gives up after 5 starts in 10 minutes instead (`StartLimitBurst=5`). Setup clears that count +(`systemctl reset-failed`) before it starts T3, so running it again within the 10 minutes still starts T3. A wall that fails stops T3, and nothing starts it again by itself: fix what `pc-agents check` names (or run `pc-agents setup` again), then start it as root: `systemctl start t3code-agents.service` (its own start runs the -check first and refuses on a failed wall). +check first and refuses on a failed wall). If it says `start request repeated too quickly`, run +`systemctl reset-failed t3code-agents.service` first. + +## Owner steps (once per PC) + +Commands are for Windows PowerShell unless a step says otherwise. `` is a name of your choice (for example +`t3-agents`); `` is the fork's `OWNER/NAME`, `` a commit of its `ci-release` branch. + +1. **Make a fresh distro** (not the CI one, not your own): `wsl --install -d Ubuntu-24.04 --name --no-launch`. +2. **Write the machine file** inside it as root: `wsl -d -u root`, then + `mkdir -p /etc/fork-kit && nano /etc/fork-kit/machine.json` (the example above, your values), and leave with `exit`. + Always open this distro with `-u root`. If a window ever asks you for a new user name and password (Ubuntu's + first-launch setup), that makes a login user, which setup refuses: remove it as root (`userdel -r `) or + start again from step 1. The kit makes `root` the default user, so this cannot happen after step 4. +3. **Run the setup** as root: + `wsl -d -u root -- sh -c 'curl -fsSL https://raw.githubusercontent.com//ci-release/fork-kit/pc-agents/bootstrap.sh -o /root/pc-agents-bootstrap.sh && sh /root/pc-agents-bootstrap.sh --repo --sha '`. + The bootstrap comes from the protected `ci-release` branch itself, not from ``, so it is a reviewed one; it + refuses a `` that is not on that branch before it downloads or runs anything of it. + The first run writes everything, then says a wall does not hold (Windows' drives are still mounted) and does + not start T3. +4. **Restart the distro** so `wsl.conf` applies: `wsl --terminate `. +5. **Run the setup again** (the same command as step 3). Now every wall holds and T3 starts. It prints the next step. +6. **Join Tailscale** with the tag (as root in the distro): install it + (`curl -fsSL https://tailscale.com/install.sh | sh`), then `tailscale up --advertise-tags= --accept-dns=false` + (keep the kit's public nameservers; MagicDNS is on the tailnet, which the agent cannot reach) and open the link it + prints in a browser on any machine. The linking T3's Tailscale address is already in `peers` (step 2's file needs + it; setup refuses a file without it); if that address changes, edit the file and run step 3 again. The ACL is + below. +7. **Log in to Claude as the agent:** never open a shell as `agent` (`wsl -u agent` would start the agent's own + session, outside T3's walls). Run each command through the kit instead, which runs it as `agent` under the same + walls as T3 (a transient system unit, `systemd-run`): + `wsl -d -u root -- /opt/pc-agents/current/sbin/pc-agents as-agent -- sh -c ''`, + then the same with `-- claude`; open the link it prints in your browser. The kit runs the command through `env` + with the agent's `PATH` (`/home/agent/.local/bin` first), so a bare `claude` is found there, and with only + `HOME`, `T3CODE_HOME`, `PATH` and `SHELL` set. Codex: its browser login waits on `localhost` port 1455, outside + `devPorts`, so the walls refuse it: use its device-code login instead, the same with `-- codex login --device-auth` + (`codex login --help` names it). +8. **Log in to GitHub as the agent:** install `gh` (as root, from GitHub CLI's docs), then + `wsl -d -u root -- /opt/pc-agents/current/sbin/pc-agents as-agent -- gh auth login --with-token` and paste + a fine-grained token for the agents' repos into that window only. +9. **Link from the other T3:** on this PC's T3 (`http://:`), Settings → Sharing & access → + make a pairing code. In the other T3: Settings → Environments → Linked environments → Link, the same address + and the code, **Full access**. +10. **Keep the distro running:** WSL stops a distro soon after its last window closes, and does not start one when + Windows starts. A logon task of your own (no administrator needed) keeps this one running while you are logged + in. In PowerShell, as yourself (the same task as the desk starter's, `fork-kit/desk/windows/start-desk.ps1`): + ```powershell + $a = New-ScheduledTaskAction -Execute 'conhost.exe' -Argument '--headless wsl.exe -d -u root --exec sleep infinity' + Register-ScheduledTask -TaskName 'pc-agents ' -Action $a -Trigger (New-ScheduledTaskTrigger -AtLogOn -User ([Security.Principal.WindowsIdentity]::GetCurrent().Name)) -Settings (New-ScheduledTaskSettingsSet -ExecutionTimeLimit 0 -AllowStartIfOnBatteries -DontStopIfGoingOnBatteries) -Force + Start-ScheduledTask -TaskName 'pc-agents ' + ``` + `conhost.exe --headless` shows no window. Running it again replaces the task. A distro set up before this step + only needs these lines; nothing in the distro changes. To remove it: + `Unregister-ScheduledTask -TaskName 'pc-agents ' -Confirm:$false`. + +Run step 3 again for a new release or a changed machine file (and steps 4 and 5 if it asks); T3 restarts only when +something changed. To remove everything: `wsl --unregister `. + +## Tailscale tag and ACL + +The PC's node is tagged, so it is not you: it gets no access of its own, only what the ACL grants. Grant one thing, +the linking T3 reaching T3's port here, and nothing from the PC: + +```json +{ + "tagOwners": { "tag:pc-agents": ["autogroup:admin"], "tag:orchestrator": ["autogroup:admin"] }, + "grants": [{ "src": ["tag:orchestrator"], "dst": ["tag:pc-agents"], "ip": ["tcp:3773"] }] +} +``` + +With no grant whose `src` is `tag:pc-agents`, the PC starts no connection into the tailnet; and the agent's uid cannot +reach any tailnet address anyway (100.64.0.0/10 and `fc00::/7` are dropped for it). Replies to the linking T3 are +allowed (connection tracking). + +## Runs again, stops half-way, meets old or empty state + +- **Again:** every file is compared before it is written; an unchanged file is not touched, and T3 restarts only + when a file, the release or the T3 version changed. The tests run the setup twice and find no change. +- **Stopped half-way:** each file is written whole (a new file, then one rename). Before and after every change + (and after a T3 install, and once more right before its restart), setup writes the root-owned marker + `/etc/pc-agents/restart-pending` with this boot's id and the time (`CLOCK_MONOTONIC`, the clock of systemd's + `ExecMainStartTimestampMonotonic`, which no clock change moves). It is removed only after T3 restarted and stayed + up for 15 seconds (the same process, active, no automatic restart). So a run stopped anywhere, or a failed T3 + install, leaves the marker, and the next good run restarts T3 even when it has nothing left to write. Meanwhile + the `restart` wall fails while the T3 that runs started before the marker's time, so the 2-minute check stops it + (an old T3 never keeps running under new files and a passing check). A run stopped during setup's own restart or + its 15 seconds leaves a T3 started after the marker: the wall passes, and the next run restarts it once more. A + marker of an earlier boot passes (every T3 now started after it); one an older kit wrote, with no time in it, + fails until setup runs again. A T3 that starts and then exits within the 15 seconds is reported (`T3 did not + start or stay up`), and the marker stays. The record + (`/etc/pc-agents/installed.tsv`) holds a file's old and new hash while it is replaced, so a run stopped right after + a write, followed by an edit of the machine file, is not mistaken for a hand edit. A new T3 version is installed by + root after the new firewall is loaded, so a slow or failed download never leaves the old firewall in place. +- **Timeouts and signals:** every program setup runs has a time limit (30 minutes for T3's download, 15 for + `apt-get`); past it the whole process group is stopped and setup says so. Run it again. Ctrl-C, `SIGTERM` or + `SIGHUP` (a closed window) stop setup the same way, the running download's whole group too. +- **Start limit:** a rerun within 10 minutes of five failed starts still starts T3 (`systemctl reset-failed` first). +- **An older kit's distro:** an `agent` made with a login shell gets `/usr/sbin/nologin` (`usermod`), once. +- **A distro that is not fresh:** refused before anything is written, with the reasons: another login user, an + `agent` with another uid or in another group, a file the kit would write that it did not write (or that was + edited since, for example a hand-written `wsl.conf` or `policy.json`), a cron, at or ssh unit where the kit masks + one. A fresh distro's own `wsl.conf` (systemd only) and WSL's generated `resolv.conf` are replaced. +- **Old releases:** each release has its own folder under `/opt/pc-agents/`, switched by the `current` link; old ones + stay (small) until the distro is removed. A T3 that an older kit installed in the agent's home is left there and + no longer run (the unit runs root's copy under `/opt/pc-agents/t3`). +- **Unsafe folders:** setup refuses a distro whose `/`, `/opt`, `/opt/pc-agents`, `/etc`, `/etc/systemd`, + `/etc/systemd/system`, `/etc/pc-agents` or `/etc/t3code` is not root's alone. +- **No or a broken machine file:** setup and check refuse, naming the key; the unit does not start. ## Limits @@ -128,19 +239,33 @@ check first and refuses on a failed wall). - **DNS** goes to the public `nameservers` only. A `systemd-resolved` stub (`127.0.0.53`), which may resolve LAN names through Windows, is out of reach: the agent's localhost traffic outside T3's port and `devPorts` is dropped. - **Terminals:** what the agent prints in a terminal can use escape codes, such as OSC 52, which some terminals turn - into a write to the Windows clipboard. Do not paste from the clipboard blindly after watching agent output. + into a write to the Windows clipboard. `pc-agents as-agent` runs with `--pty`, so the owner steps' output reaches + your terminal that way too. Do not paste from the clipboard blindly after watching agent output. +- **Runs only while you are logged in:** the logon task (step 10) keeps the distro running while you are logged in + and the PC is awake. Logging out, sleep, a Windows restart or `wsl --shutdown` stop T3 (the linking T3 sees this + PC offline); the task starts it again at your next logon. Without the task, T3 stops soon after the last WSL + window closes. - **No namespaces in the unit** (`RestrictNamespaces=yes`): provider sandboxes that need them (bubblewrap) do not run; the distro is the sandbox, so use full access. +- **Tailscale, Claude Code and `gh` are yours to install** (steps 6 to 8): their installers and logins need you. + Claude Code and logins go through `pc-agents as-agent`, never a shell as `agent`. +- **No uninstall** but removing the distro; the distro holds nothing else. - **No code hash manifest yet:** the kit's code is checked for root ownership and its written files byte for byte, - but the code files are not hashed (T3's `t3` is). + but the code files are not hashed (T3's `t3` is). The bootstrap duplicates the heavy kit's (a shared bootstrap is + a follow-up). ## Tests ``` +python3 -I -B fork-kit/pc-agents/tests/setup.test.py python3 -I -B fork-kit/pc-agents/tests/check.test.py ``` -A fake distro in a temporary folder (`PC_AGENTS_ROOT`, refused for root; `tests/fake_distro.py`), with fake `nft` and -`systemctl` (it keeps which units are enabled and running): every wall passes on a walled distro, and each one +A fake distro in a temporary folder (`PC_AGENTS_ROOT`, refused for root; `tests/fake_distro.py`), with fake +`useradd`, `usermod`, T3's installer, `systemctl` (it keeps which units are enabled and running, their main process +and start time, a start limit, and a T3 that dies a moment after it starts), `systemd-run` (a bare command is looked +up in the caller's `PATH`, as the real one does) and `nft`: every file and mode, a second run changes nothing, a run +stopped at every write finishes on the next and restarts T3, an upgrade stopped at every write fails `check` until +setup runs again, `SIGTERM` and `SIGHUP` end the download too, each kind of foreign distro is refused, and each wall fails on its own when its part is taken away. `apps/server/src/policy/managedSettings.forkKeys.test.ts` proves the managed policy locks the fork's settings. diff --git a/fork-kit/pc-agents/bootstrap.sh b/fork-kit/pc-agents/bootstrap.sh new file mode 100755 index 0000000000000..0531bb1e8348b --- /dev/null +++ b/fork-kit/pc-agents/bootstrap.sh @@ -0,0 +1,51 @@ +#!/bin/sh +# pc-agents bootstrap: runs `pc-agents setup` from one release of this fork. As root, in a fresh WSL2 distro: +# +# sh pc-agents-bootstrap.sh --repo FORK_OWNER/FORK --sha RELEASE_SHA +# +# It refuses a sha that is not a commit on the fork's protected release branch (ci-release), downloads exactly that +# commit, prints the sha and the hash of what it downloaded, and runs its setup (safe to run again). The setup reads +# the "pc-agents" section of /etc/fork-kit/machine.json (fork-kit/pc-agents/README.md). Needs curl, tar, python3. +set -eu +umask 022 +PATH=/usr/sbin:/usr/bin:/sbin:/bin +export PATH +unset TMPDIR CURL_HOME SSL_CERT_FILE SSL_CERT_DIR PC_AGENTS_ROOT || true + +die() { echo "pc-agents-bootstrap: $*" >&2; exit 1; } + +repo='' sha='' branch='ci-release' api='https://api.github.com' codeload='https://codeload.github.com' +while [ $# -gt 0 ]; do + case "$1" in + --repo) repo="${2-}"; shift 2 ;; + --sha) sha="${2-}"; shift 2 ;; + *) die "unknown option $1 (see the top of this file)" ;; + esac +done +printf '%s\n' "$repo" | grep -Eqx '[A-Za-z0-9][A-Za-z0-9-]{0,38}/[A-Za-z0-9._-]{1,100}' || die "--repo must be OWNER/NAME" +printf '%s\n' "$sha" | grep -Eqx '[0-9a-f]{40}' || die "--sha must be the full 40-character release commit" +[ "$(id -u)" = 0 ] || die "run this as root" + +work="$(mktemp -d /root/pc-agents-bootstrap.XXXXXXXX)" +trap 'rm -rf "${work:?}"' EXIT +trap 'exit 1' INT TERM HUP +fetch() { curl -fsSL --proto '=https' --tlsv1.2 --max-redirs 0 --connect-timeout 30 --max-time 900 -H 'Accept: application/vnd.github+json' -o "$2" "$1"; } + +fetch "$api/repos/$repo/branches/$branch" "$work/branch.json" || die "cannot read branch $branch of $repo" +fetch "$api/repos/$repo/compare/$sha...$branch" "$work/compare.json" || die "cannot compare $sha with $branch" +python3 -I -B - "$work/branch.json" "$work/compare.json" "$sha" <<'PY' || die "$sha is not a release: it must be a commit on the protected branch $branch" +import json, sys +branch, compare = (json.load(open(p, encoding='utf-8')) for p in sys.argv[1:3]) +ok = (branch.get('protected') is True and compare.get('status') in ('ahead', 'identical') + and (compare.get('merge_base_commit') or {}).get('sha') == sys.argv[3]) +sys.exit(0 if ok else 1) +PY + +fetch "$codeload/$repo/tar.gz/$sha" "$work/kit.tar.gz" || die "cannot download $repo at $sha" +mkdir "$work/src" +tar -xzf "$work/kit.tar.gz" -C "$work/src" --no-same-owner --no-same-permissions +top="$(find "$work/src" -mindepth 1 -maxdepth 1 -type d)" +[ "$(printf '%s\n' "$top" | wc -l)" = 1 ] && [ -n "$top" ] || die "the download does not have one top folder" +[ -f "$top/fork-kit/pc-agents/sbin/pc-agents" ] || die "release $sha has no fork-kit/pc-agents" +echo "pc-agents-bootstrap: release $sha of $repo, download sha256 $(sha256sum "$work/kit.tar.gz" | cut -d' ' -f1)" +python3 -I -B "$top/fork-kit/pc-agents/sbin/pc-agents" setup --source "$top" --sha "$sha" diff --git a/fork-kit/pc-agents/lib/pc_agents.py b/fork-kit/pc-agents/lib/pc_agents.py index 930ddf4e4c1e6..59b45718cc599 100644 --- a/fork-kit/pc-agents/lib/pc_agents.py +++ b/fork-kit/pc-agents/lib/pc_agents.py @@ -2,7 +2,7 @@ Every path is read and written under ROOT, which is '' (the real machine) except in the tests, where PC_AGENTS_ROOT names a fake one; root never gets a fake one. Standard library only.""" -import hashlib, ipaddress, json, os, re, signal, subprocess +import hashlib, ipaddress, json, os, re, signal, subprocess, time import machine_file @@ -15,6 +15,10 @@ NOLOGIN = '/usr/sbin/nologin' # the agent's login shell: no shell for `wsl -u agent` or su; T3 gets SHELL from its unit OPT, STATE = '/opt/pc-agents', '/etc/pc-agents' RECORD = STATE + '/installed.tsv' # sha256 path of every file this kit wrote (a path may have two mid-write) +PENDING = STATE + '/restart-pending' # setup changed something T3 runs with: restart T3 before removing this +BOOT_ID = '/proc/sys/kernel/random/boot_id' # the marker holds this boot's id and CLOCK_MONOTONIC, like systemd's +AS_AGENT = 'pc-agents-as-agent' # name prefix of `pc-agents as-agent`'s transient units (check --enforce stops them) +SETTLE, SETTLE_STEP = 15, 1 # after a (re)start, setup watches T3 this many seconds, every SETTLE_STEP, before done LOADED = '/run/pc-agents/firewall.loaded' # sha256 of the firewall file and `nft list` of it, saved when loaded T3 = OPT + '/t3' # T3's runtime, installed by root (T3CODE_HOME of scripts/install.sh): the agent cannot change it UNIT, WALLS = 't3code-agents.service', 'pc-agents-walls' @@ -135,6 +139,41 @@ def dropin_dirs(unit): return [unit + '.d', kind + '.d'] + ['-'.join(parts[:i]) + f'-.{kind}.d' for i in range(1, len(parts))] +def sandbox(c): + """The unit's walls, one `Key=value` each: T3's unit runs under them, and so does `pc-agents as-agent`.""" + return [ + 'UMask=0077', + f'MemoryMax={c["memoryMax"]}', + f'CPUQuota={c["cpuQuota"]}', + 'NoNewPrivileges=yes', + 'ProtectSystem=strict', + 'ProtectHome=read-only', + f'ReadWritePaths={HOME}', + 'PrivateTmp=yes', + 'PrivateDevices=yes', + 'PrivateIPC=yes', + 'ProtectProc=invisible', + 'InaccessiblePaths=-/mnt -/run/WSL -/tmp/.X11-unix -/usr/lib/wsl -/run/tailscale', + f'IPAddressDeny={" ".join(BLOCKED4 + BLOCKED6)}', + f'IPAddressAllow={" ".join(c["peers"] + c["allow"])}', + 'SocketBindDeny=any', + *('SocketBindAllow=tcp:' + p for p in loopback_ports(c)), + 'RestrictAddressFamilies=AF_UNIX AF_INET AF_INET6 AF_NETLINK', + 'SystemCallFilter=~io_uring_setup:ENOSYS io_uring_enter:ENOSYS io_uring_register:ENOSYS', + 'SystemCallArchitectures=native', + 'RestrictNamespaces=yes', + 'RestrictSUIDSGID=yes', + 'RestrictRealtime=yes', + 'LockPersonality=yes', + 'ProtectKernelTunables=yes', + 'ProtectKernelModules=yes', + 'ProtectKernelLogs=yes', + 'ProtectControlGroups=yes', + 'ProtectClock=yes', + 'ProtectHostname=yes', + ] + + def render(c): """{path: (text, mode)} of every file the kit owns, from the config alone (the same every run).""" settings = {**KIT_SETTINGS, **c['lockedSettings']} @@ -151,6 +190,7 @@ def render(c): (f'\t\tip daddr {{ {", ".join(allow4)} }} accept\n' if allow4 else '') + \ (f'\t\tip6 daddr {{ {", ".join(allow6)} }} accept\n' if allow6 else '') + \ f'\t\tip daddr {{ {", ".join(BLOCKED4)} }} drop\n\t\tip6 daddr {{ {", ".join(BLOCKED6)} }} drop\n\t}}\n}}\n' + walls = '\n'.join(sandbox(c)) unit = HEAD + f'''[Unit] Description=T3 Code for agents, walled (pc-agents kit) Wants=network-online.target @@ -169,35 +209,7 @@ def render(c): ExecStart={runtime(c)}/t3 serve Restart=on-failure RestartSec=30 -UMask=0077 -MemoryMax={c["memoryMax"]} -CPUQuota={c["cpuQuota"]} -NoNewPrivileges=yes -ProtectSystem=strict -ProtectHome=read-only -ReadWritePaths={HOME} -PrivateTmp=yes -PrivateDevices=yes -PrivateIPC=yes -ProtectProc=invisible -InaccessiblePaths=-/mnt -/run/WSL -/tmp/.X11-unix -/usr/lib/wsl -/run/tailscale -IPAddressDeny={" ".join(BLOCKED4 + BLOCKED6)} -IPAddressAllow={" ".join(c["peers"] + c["allow"])} -SocketBindDeny=any -{chr(10).join("SocketBindAllow=tcp:" + p for p in loopback_ports(c))} -RestrictAddressFamilies=AF_UNIX AF_INET AF_INET6 AF_NETLINK -SystemCallFilter=~io_uring_setup:ENOSYS io_uring_enter:ENOSYS io_uring_register:ENOSYS -SystemCallArchitectures=native -RestrictNamespaces=yes -RestrictSUIDSGID=yes -RestrictRealtime=yes -LockPersonality=yes -ProtectKernelTunables=yes -ProtectKernelModules=yes -ProtectKernelLogs=yes -ProtectControlGroups=yes -ProtectClock=yes -ProtectHostname=yes +{walls} [Install] WantedBy=multi-user.target @@ -217,7 +229,7 @@ def render(c): sd + WALLS + '.service': (HEAD + f'[Unit]\nDescription=pc-agents walls check\nAfter={UNIT}\n\n[Service]\n' f'Type=oneshot\nTimeoutStartSec=5min\nExecStart={CHECK} check --enforce\n' 'ExecStopPost=/bin/sh -c \'[ "$$SERVICE_RESULT" = success ] || ' - f'exec /usr/bin/systemctl stop {UNIT}\'\n', 0o644), + f'exec /usr/bin/systemctl stop {UNIT} "{AS_AGENT}-*.service"\'\n', 0o644), sd + WALLS + '.timer': (HEAD + '[Unit]\nDescription=pc-agents walls check every 2 minutes\n\n[Timer]\n' 'OnActiveSec=1min\nOnUnitActiveSec=2min\nAccuracySec=10s\n\n[Install]\n' 'WantedBy=timers.target\n', 0o644), @@ -320,6 +332,58 @@ def login_shell(name): return None +def clock(): + """(this boot's id, CLOCK_MONOTONIC in microseconds): systemd's *TimestampMonotonic clock, which no clock change + or WSL time sync moves.""" + with open(ROOT + BOOT_ID, encoding='ascii') as f: + return f.read().strip(), time.monotonic_ns() // 1000 + + +def marker(): + """The restart marker's text: when setup last changed something T3 runs with (boot id, monotonic microseconds).""" + boot, now = clock() + return f'boot_id={boot} monotonic_us={now}\nsetup changed what T3 runs with; the next good run restarts T3 and ' \ + 'removes this. Until then `pc-agents check` fails while a T3 started before this time runs.\n' + + +def props(text): + """`systemctl show` output as a dict.""" + return dict(line.split('=', 1) for line in text.splitlines() if '=' in line) + + +def restart_problem(): + """Why a T3 runs that started before setup's last change (a restart is pending), or None. No marker, or no T3 + process (stopped, or still in its ExecStartPre check): nothing stale runs. Anything unreadable fails.""" + if lstat(PENDING) is None: + return None + r = run(['/usr/bin/systemctl', 'show', '-p', 'MainPID,ExecMainStartTimestampMonotonic', UNIT]) + got = props(r.stdout) + if r.returncode or not re.fullmatch(r'\d+', got.get('MainPID', '')): + return f'cannot tell whether T3 runs ({(r.stderr or r.stdout).strip()[-200:]})' + if got['MainPID'] == '0': + return None + try: + with open(ROOT + PENDING, encoding='utf-8', errors='replace') as f: + m = re.match(r'boot_id=(\S+) monotonic_us=(\d+)\n', f.read()) + boot = clock()[0] + except OSError as e: + return f'cannot read {PENDING} or the boot id ({e})' + started = got.get('ExecMainStartTimestampMonotonic', '') + if not m or not re.fullmatch(r'[1-9]\d*', started): + return f'a restart is pending ({PENDING}) and T3\'s start time or the marker\'s cannot be read' + if m[1] == boot and int(started) <= int(m[2]): # a marker of an earlier boot is older than any T3 now + return f'T3 runs as started before setup\'s last change ({PENDING}): run pc-agents setup again' + return None + + +def stop_on_signals(): + """SIGTERM and SIGHUP end this process like Ctrl-C does, through run()'s clean-up (its child's group is killed).""" + def stop(signum, frame): + raise SystemExit(128 + signum) + for s in (signal.SIGTERM, signal.SIGHUP): + signal.signal(s, stop) + + def sudo_problem(c): """Why sudo could give the agent more (a sudoers line naming it, its group or uid, or every user), or None. Harmless inside the unit (NoNewPrivileges), but a way out from anywhere else.""" @@ -359,8 +423,9 @@ def user_problem(c, need_agent): def run(argv, timeout=120): - """Run argv (argv[0] under ROOT) in a process group of its own. Past timeout, or when this process stops, the - whole group is killed (a download's curl too); a timeout returns exit 124 and says so in stderr.""" + """Run argv (argv[0] under ROOT) in a process group of its own. Past timeout, or when this process stops (Ctrl-C, + or SIGTERM and SIGHUP once stop_on_signals() ran), the whole group is killed (a download's curl too); a timeout + returns exit 124 and says so in stderr.""" p = subprocess.Popen([ROOT + argv[0]] + argv[1:], stdin=subprocess.DEVNULL, stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True, start_new_session=True) try: diff --git a/fork-kit/pc-agents/sbin/pc-agents b/fork-kit/pc-agents/sbin/pc-agents index 188d680d13a44..d30404ed22481 100755 --- a/fork-kit/pc-agents/sbin/pc-agents +++ b/fork-kit/pc-agents/sbin/pc-agents @@ -1,18 +1,22 @@ #!/usr/bin/python3 -I """pc-agents: wall a WSL2 distro in, then run T3 Code in it as an unprivileged user (fork-kit/pc-agents/README.md). + pc-agents setup --source DIR --sha SHA as root, in a fresh distro: make or repair everything; safe to run again pc-agents check [--enforce] every wall, PASS or FAIL; exit 0 only when all hold. --enforce also - stops T3 when one fails (the timer, every 2 minutes). T3's unit runs - it before every start, so T3 never starts on a failed wall. + stops T3 and every as-agent unit when one fails (the timer, every 2 + minutes). T3's unit runs it before every start, so T3 never starts on + a failed wall. pc-agents load-firewall as root: load the agent firewall and note what the kernel lists + pc-agents as-agent -- CMD... as root: run CMD as the agent under T3's unit walls (the owner steps) Standard library only.""" -import os, re, stat, sys +import os, re, stat, sys, time HERE = os.path.dirname(os.path.dirname(os.path.realpath(__file__))) LIB = os.path.join(HERE, 'lib') COMMON = os.path.join(os.path.dirname(HERE), 'common') # the repository's; an installed kit has its own copy in LIB sys.path[:0] = [LIB] + ([] if os.path.exists(os.path.join(LIB, 'machine_file.py')) else [COMMON]) sys.dont_write_bytecode = True +import machine_file # noqa: E402 import pc_agents as k # noqa: E402 @@ -20,6 +24,238 @@ def say(text): print(f'pc-agents: {text}', flush=True) +def same(path, data, mode, link=False): + """Whether path already is data (a root-owned file of mode, or a link to data).""" + st = k.lstat(path) + if link: + return bool(st and stat.S_ISLNK(st.st_mode) and os.readlink(k.ROOT + path) == data) + if not (st and stat.S_ISREG(st.st_mode) and stat.S_IMODE(st.st_mode) == mode and st.st_uid == k.OWNER): + return False + with open(k.ROOT + path, 'rb') as f: + return f.read() == (data.encode() if isinstance(data, str) else data) + + +def put(path, data, mode, link=False): + """Write a file (or a link) whole: a new file beside it, then one rename. Returns True when it changed. Every + change but to the record marks T3 for a restart before and after it, so a run stopped after it still restarts T3 + next time, and `check` fails on a T3 that started before it.""" + if same(path, data, mode, link): + return False + marks = path not in (k.RECORD, k.PENDING) + if marks: + mark() + real = k.ROOT + path + os.makedirs(os.path.dirname(real), mode=0o755, exist_ok=True) + tmp = f'{real}.pc-agents-new' + if os.path.lexists(tmp): + os.unlink(tmp) + if link: + os.symlink(data, tmp) + else: + fd = os.open(tmp, os.O_WRONLY | os.O_CREAT | os.O_EXCL | os.O_NOFOLLOW, 0o600) + with os.fdopen(fd, 'wb') as f: + f.write(data.encode() if isinstance(data, str) else data) + os.fchmod(f.fileno(), mode) + if not k.ROOT: + os.fchown(f.fileno(), 0, 0) + os.fsync(f.fileno()) + os.replace(tmp, real) + if marks: + mark() + return True + + +def mark(): + """Note that T3 must restart, and when setup last changed something (root's marker, rewritten on every change and + removed only once T3 restarted and stayed up). A T3 started before that time is stale: `check` fails on it.""" + put(k.PENDING, k.marker(), 0o644) + + +def keep(done): + """Write the record: {path: {sha256, ...}}.""" + put(k.RECORD, ''.join(f'{h}\t{p}\n' for p, hs in sorted(done.items()) for h in sorted(hs)), 0o644) + + +def fresh(path, st): + """A file a fresh distro has where the kit writes its own: replaced without asking.""" + def text(): + with open(k.ROOT + path, errors='replace') as f: + return f.read() + if path == '/etc/resolv.conf': + return stat.S_ISLNK(st.st_mode) or 'generated by WSL' in text() + if path == '/etc/wsl.conf' and stat.S_ISREG(st.st_mode): + return re.sub(r'(?m)^\s*[#;].*\n?|\s+', '', text()).lower() in ('', '[boot]systemd=true') + return False + + +def preflight(c, files): + """Every reason not to touch this distro, before anything is written.""" + bad = [] + if why := k.user_problem(c, False): + bad.append(why + '. A first launch that asked for a user name made one: remove it as root (userdel -r ' + 'NAME) or start again from a fresh distro (README step 2)' if 'login users' in why else why) + for d in ('/', '/opt', k.OPT, '/etc', '/etc/systemd', '/etc/systemd/system', k.STATE, '/etc/t3code'): + st = k.lstat(d) + if st and (why := k.safe(d, st, stat.S_ISDIR)) or st is None and d in ('/', '/opt', '/etc', '/etc/systemd'): + bad.append(why or f'{d} is missing') + done = k.record() + for path, (text, _) in files.items(): + st = k.lstat(path) + if st is None or fresh(path, st): + continue + if not stat.S_ISREG(st.st_mode): + bad.append(f'{path} is not a plain file') + continue + now = k.file_sha(path) + if now != k.sha(text) and now not in done.get(path, ()): + bad.append(f'{path} was not written by this kit, or was edited since (move it away to go on)') + for path in k.masks(c): + st = k.lstat(path) + if st and not (stat.S_ISLNK(st.st_mode) and os.readlink(k.ROOT + path) == '/dev/null'): + bad.append(f'{path} exists and is not a mask (a link to /dev/null)') + return bad + + +def install_t3(c, version, done): + """Install T3 as root under k.T3 with upstream's scripts/install.sh (it checks SHA256SUMS and runs the new t3), + and record its t3's hash. Returns why not, or None.""" + folder = k.runtime(c) + if k.lstat(folder) and k.lstat(folder + '/t3') is None: + return f'{folder} is there but has no t3 (not made by setup): remove that folder' + if k.lstat(folder) is None: + say(f'installing T3 {c["t3Version"]} under {k.T3}') + mark() + r = k.run(['/usr/bin/env', '-i', 'HOME=/root', 'PATH=/usr/sbin:/usr/bin:/sbin:/bin', f'T3CODE_HOME={k.T3}', + f'T3CODE_INSTALL_BIN_DIR={k.T3}/bin', f'T3CODE_VERSION={c["t3Version"]}', + f'T3CODE_RELEASE_BASE_URL={c["t3ReleaseBaseUrl"]}', '/bin/sh', f'{version}/share/install.sh'], + timeout=1800) + if r.returncode or k.lstat(folder + '/t3') is None: + return f'T3 install failed: {(r.stderr or r.stdout).strip()[-400:]}' + mark() + if not done.get(folder + '/t3'): # a mismatch is left to the runtime wall; only a missing hash is added + done[folder + '/t3'] = {k.file_sha(folder + '/t3')} + keep(done) + return None + + +def setup(source, sha): + if not re.fullmatch(r'[0-9a-f]{40}', sha) or not os.path.isdir(source): + return say('usage: pc-agents setup --source DIR --sha 40-HEX-SHA') or 2 + if not k.ROOT and os.geteuid() != 0: + return say('run this as root') or 2 + os.umask(0o022) + try: + c = k.config() + except machine_file.MachineFileError as e: + return say(f'{e} (fork-kit/pc-agents/README.md has an example)') or 2 + files = k.render(c) + bad = preflight(c, files) + if bad: + return say('refused, nothing changed:\n ' + '\n '.join(bad)) or 1 + if k.USER not in k.users()[0]: + r = k.run(['/usr/sbin/useradd', '--create-home', '--home-dir', k.HOME, '--shell', k.NOLOGIN, + '--uid', str(c['agentUid']), '--user-group', k.USER]) + if r.returncode: + return say(f'useradd failed: {r.stderr.strip()[-300:]}') or 1 + elif k.login_shell(k.USER) != k.NOLOGIN: # an agent an older kit made with a login shell + r = k.run(['/usr/sbin/usermod', '--shell', k.NOLOGIN, k.USER]) + if r.returncode: + return say(f'usermod failed: {r.stderr.strip()[-300:]}') or 1 + if k.lstat('/usr/sbin/nft') is None: + for argv in (['/usr/bin/apt-get', 'update'], ['/usr/bin/apt-get', 'install', '-y', 'nftables']): + r = k.run(argv, timeout=900) + if r.returncode: + return say(f'cannot install nftables: {" ".join(argv)} failed ({r.stderr.strip()[-300:]}). Fix the ' + 'distro\'s network or package sources, run `apt-get update && apt-get install -y ' + 'nftables` as root, then setup again') or 1 + version = f'{k.OPT}/{sha}' + for src, (dst, mode) in k.CODE.items(): + with open(os.path.join(source, src), 'rb') as f: + put(f'{version}/{dst}', f.read(), mode) + put(f'{k.OPT}/current', sha, None, link=True) + done = k.record() + for path, (text, mode) in files.items(): # recorded old and new before the write, new alone after it + if same(path, text, mode): + continue + if k.lstat(path) and stat.S_ISREG(k.lstat(path).st_mode) and k.file_sha(path) in done.get(path, ()): + done[path] = {k.file_sha(path), k.sha(text)} + else: + done[path] = {k.sha(text)} + keep(done) + put(path, text, mode) + done[path] = {k.sha(text)} + keep(done) + why = k.load_firewall() # now, not after the install: a tightened firewall must not wait for a download + if why: + return say(why) or 1 + k.run(['/usr/bin/systemctl', 'daemon-reload']) # systemd reads the units as written, even if the install fails + for path in k.masks(c): + put(path, '/dev/null', None, link=True) + why = install_t3(c, version, done) + if why: + return say(why + ' (run setup again; T3 restarts once it succeeds)') or 1 + units = [k.UNIT, k.WALLS + '.timer'] + if check(False, before_enable=True) != 0: + k.run(['/usr/bin/systemctl', 'disable', '--now'] + units) + return say('a wall does not hold, so T3 is not enabled. After a first run: wsl --terminate ' + 'from Windows, then run setup again') or 3 + k.run(['/usr/bin/systemctl', 'reset-failed', k.UNIT]) # a rerun within the start limit's 10 minutes still starts + ran = k.run(['/usr/bin/systemctl', 'is-active', k.UNIT]).returncode == 0 + r = k.run(['/usr/bin/systemctl', 'enable', '--now'] + units) + if r.returncode == 0 and k.lstat(k.PENDING): + mark() # every change of this and earlier runs is before now; the T3 started next is after it + r = k.run(['/usr/bin/systemctl', 'restart', k.UNIT]) + if r.returncode == 0: + r = k.run(['/usr/bin/systemctl', 'is-active', k.UNIT]) + why = (r.stderr or r.stdout).strip()[-300:] if r.returncode else \ + stays_up() if k.lstat(k.PENDING) or not ran else None + if why: + return say(f'T3 did not start or stay up ({why}): see journalctl -u {k.UNIT}; run setup again once fixed') or 1 + if k.lstat(k.PENDING): + os.unlink(k.ROOT + k.PENDING) + say(f'done: T3 serves on port {c["port"]}. Next: tailscale up --advertise-tags={c["tailscaleTag"]} ' + '--accept-dns=false') + return 0 + + +def stays_up(): + """Why T3 did not stay up for k.SETTLE seconds after a start (it can exit on a policy it cannot apply a moment + after it started), or None: active all along, the same main process, no automatic restart.""" + first, end = None, time.monotonic() + k.SETTLE + while True: + r = k.run(['/usr/bin/systemctl', 'show', '-p', 'ActiveState,MainPID,NRestarts', k.UNIT]) + got = k.props(r.stdout) + now = (got.get('ActiveState'), got.get('MainPID'), got.get('NRestarts')) + if r.returncode or now[0] != 'active' or now[1] in (None, '', '0'): + return f'T3 is {now[0] or "unknown"}' + if first and now != first: + return f'T3 restarted (main process {first[1]} then {now[1]}, restarts {first[2]} then {now[2]})' + first = first or now + if time.monotonic() >= end: + return None + time.sleep(k.SETTLE_STEP) + + +def as_agent(cmd): + """Run cmd as the agent under the unit's walls (a transient system unit): never a login shell, never the + agent's user manager. For the owner steps (a provider's login), as root.""" + if not k.ROOT and os.geteuid() != 0: + return say('run this as root') or 2 + try: + c = k.config() + except machine_file.MachineFileError as e: + return say(str(e)) or 2 + if check(False) != 0: + return say('a wall does not hold: not running anything as the agent') or 1 + argv = [k.ROOT + '/usr/bin/systemd-run', '--quiet', '--collect', '--pty', f'--unit={k.AS_AGENT}-{os.getpid()}', + f'--uid={k.USER}', f'--gid={k.USER}', f'--working-directory={k.HOME}', f'--setenv=HOME={k.HOME}', + f'--setenv=T3CODE_HOME={k.HOME}/.t3', f'--setenv=PATH={k.HOME}/.local/bin:/usr/local/bin:/usr/bin:/bin', + '--setenv=SHELL=/bin/bash'] + # systemd-run looks a bare name up in root's PATH, not the unit's: env looks it up in the agent's (~/.local/bin) + argv += [f'--property={line}' for line in k.sandbox(c)] + ['--', '/usr/bin/env', '--'] + cmd + return k.subprocess.call(argv) + + def tree(top, inside=None): """Why something in the folder top (itself included) is not root's alone, or a link out of inside; or None.""" bad = [why] if (why := k.safe(top, k.lstat(top), stat.S_ISDIR)) else [] @@ -78,6 +314,8 @@ def own_walls(c, before_enable=False): k.user_problem(c, True) or (None if shell == k.NOLOGIN else f'{k.USER}\'s login shell is {shell}, not {k.NOLOGIN}'))) out.append(('sudoers', f'no sudo rule covers {k.USER}', k.sudo_problem(c))) + if not before_enable: # setup asks before its own restart, while the marker is there by design + out.append(('restart', 'no T3 runs that started before setup\'s last change', k.restart_problem())) current = k.lstat(k.OPT + '/current') target = os.readlink(k.ROOT + k.OPT + '/current') if current and stat.S_ISLNK(current.st_mode) else '' code = [] if current and current.st_uid == k.OWNER and re.fullmatch(r'[0-9a-f]{40}', target) else \ @@ -155,21 +393,27 @@ def check(enforce, before_enable=False): print(f'FAIL {wall} {what}: {why}'[:500] if why else f'PASS {wall} {what}', flush=True) failed = any(why for _, _, why in results) if failed and enforce: - r = k.run(['/usr/bin/systemctl', 'stop', k.UNIT]) - say(f'a wall does not hold: stopped {k.UNIT}' if r.returncode == 0 else - f'a wall does not hold, and stopping {k.UNIT} FAILED: {r.stderr.strip()[-300:]}') + for unit in (k.UNIT, f'{k.AS_AGENT}-*.service'): # T3, then every running `pc-agents as-agent` + r = k.run(['/usr/bin/systemctl', 'stop', unit]) + say(f'a wall does not hold: stopped {unit}' if r.returncode == 0 else + f'a wall does not hold, and stopping {unit} FAILED: {r.stderr.strip()[-300:]}') return 1 if failed else 0 def main(argv): if argv[:1] == ['check'] and argv[1:] in ([], ['--enforce']): return check(argv[1:] == ['--enforce']) + if argv[:1] == ['setup'] and len(argv) == 5 and argv[1] == '--source' and argv[3] == '--sha': + return setup(argv[2], argv[4]) if argv == ['load-firewall']: why = k.load_firewall() return say(why) or 1 if why else 0 + if argv[:2] == ['as-agent', '--'] and len(argv) > 2: + return as_agent(argv[2:]) print(__doc__, file=sys.stderr) return 2 if __name__ == '__main__': + k.stop_on_signals() sys.exit(main(sys.argv[1:])) diff --git a/fork-kit/pc-agents/tests/check.test.py b/fork-kit/pc-agents/tests/check.test.py index de9717f11eec8..155974180e039 100644 --- a/fork-kit/pc-agents/tests/check.test.py +++ b/fork-kit/pc-agents/tests/check.test.py @@ -6,7 +6,7 @@ import os, shutil, subprocess, sys, time, unittest sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) -from fake_distro import CONFIG, Case, DRVFS, SHA1, k # noqa: E402 +from fake_distro import CONFIG, Case, DRVFS, SHA1, cli, k # noqa: E402 class Walls(Case): @@ -25,7 +25,8 @@ def test_all_hold(self): self.assertEqual(code, 0, out) walls = [line.split()[1] for line in out.splitlines()] for wall in ('wsl2:interop', 'wsl2:resolv', 'wsl2:windows-path', 'wsl2:networking', 'wsl2:os', 'masks', - 'record', 'linger', 'user-units', 'drop-ins', 'units', 'agent', 'sudoers', 'kit-code', 'runtime', + 'record', 'linger', 'user-units', 'drop-ins', 'units', 'agent', 'sudoers', 'restart', 'kit-code', + 'runtime', 'automount', 'firewall', 'gateway', 'file:policy.json', 'file:t3code-agents.service', 'file:firewall.nft', 'file:wsl.conf', 'file:pc-agents-walls.timer'): self.assertIn(wall, walls) @@ -37,6 +38,7 @@ def test_enforce_stops_t3(self): code, out = self.check('--enforce') self.assertEqual(code, 1, out) self.assertIn('stop t3code-agents.service', self.log()) + self.assertIn('stop pc-agents-as-agent-*.service', self.log()) # and every `pc-agents as-agent` run self.assertFalse(os.path.exists(self.path('/log/units/t3code-agents.service.active'))) self.assertIn('stopped t3code-agents.service', out) @@ -229,7 +231,7 @@ def test_missing_lib_stops_t3(self): self.assertNotEqual(p.returncode, 0, p.stdout + p.stderr) self.assertNotIn('stop t3code-agents.service', self.log()) # the check itself could not stop T3 self.stop_post('exit-code') - self.assertIn('stop t3code-agents.service', self.log()) + self.assertIn('stop t3code-agents.service pc-agents-as-agent-*.service', self.log()) self.assertFalse(os.path.exists(self.path('/log/units/t3code-agents.service.active'))) def test_success_does_not_stop_t3(self): @@ -237,6 +239,70 @@ def test_success_does_not_stop_t3(self): self.assertEqual(self.log(), []) +class Restart(Case): + """The restart wall: while setup's marker is there, a T3 that started before it runs stale files.""" + def setUp(self): + super().setUp() + self.walled() + + def mark(self, text=None): + self.write(k.PENDING, k.marker() if text is None else text) + + def restart_fails(self, why): + code, out = self.check() + self.assertEqual(code, 1, out) + self.assertEqual({line.split()[1] for line in out.splitlines() if line.startswith('FAIL')}, {'restart'}, out) + self.assertIn(why, out) + + def test_t3_started_before_the_marker_fails_and_enforce_stops_it(self): + self.mark() + self.restart_fails('T3 runs as started before setup\'s last change') + code, out = self.check('--enforce') + self.assertEqual(code, 1, out) + self.assertIn('stop t3code-agents.service', self.log()) + self.assertIn('stop pc-agents-as-agent-*.service', self.log()) + self.assertEqual(self.check()[0], 0) # T3 stopped: nothing stale runs; the next setup starts it + + def test_t3_started_after_the_marker_passes(self): # setup's own restart, before it removes the marker + self.mark() + k.run(['/usr/bin/systemctl', 'restart', k.UNIT]) + code, out = self.check() + self.assertEqual(code, 0, out) + + def test_marker_of_an_earlier_boot_passes(self): # every T3 now started after it + self.mark('boot_id=00000000-0000-0000-0000-000000000000 monotonic_us=999999999999999\n') + code, out = self.check() + self.assertEqual(code, 0, out) + + def test_marker_of_an_older_kit_fails(self): # it had no time in it + self.mark('setup changed what T3 runs with; the next good run restarts T3 and removes this\n') + self.restart_fails('cannot be read') + + def test_no_start_time_fails(self): + self.mark() + os.unlink(self.path('/log/units/t3code-agents.service.start')) + self.restart_fails('cannot be read') + + def test_unreadable_marker_fails(self): + if os.geteuid() == 0: + self.skipTest('root reads any file') + self.mark() + os.chmod(self.path(k.PENDING), 0) + self.restart_fails('cannot read') + + def test_systemctl_show_fails(self): + self.mark() + self.write('/log/systemctl-fail', '') + code, out = self.check() + self.assertEqual(code, 1, out) + self.assertIn('FAIL restart', out) + + def test_skipped_before_the_first_enable(self): # setup's own check, while its marker is there by design + self.mark() + code, out = self.call(cli.check, False, True) + self.assertNotIn('restart', [line.split()[1] for line in out.splitlines() if line.startswith(('PASS', 'FAIL'))]) + + class Unit(Case): """What the kit's files say: things no wall can see from outside the unit.""" def test_unit(self): diff --git a/fork-kit/pc-agents/tests/fake_distro.py b/fork-kit/pc-agents/tests/fake_distro.py index eab2f08c38cc6..1831a88a036f8 100644 --- a/fork-kit/pc-agents/tests/fake_distro.py +++ b/fork-kit/pc-agents/tests/fake_distro.py @@ -1,7 +1,8 @@ """A fake WSL2 distro in a temporary folder for the pc-agents tests: every fixed path under PC_AGENTS_ROOT, the -current user as root, fake useradd, env (T3's installer), systemctl, systemd-run and nft (each logs to /log in the -fake root; systemctl keeps which units are enabled and which ExecStart runs in /log/units). No root, no systemd, no -network, no real /etc. Fresh as `wsl --install` leaves one, from the fleet's wsl2 walls fixture.""" +current user as root, fake useradd, usermod, env (T3's installer), apt-get, systemctl, systemd-run and nft (each logs +to /log in the fake root; systemctl keeps which units are enabled, and the ExecStart and policy a started one runs +with, its main PID and its start time (CLOCK_MONOTONIC, as systemd's) in /log/units). No root, no systemd, no network, +no real /etc. Fresh as `wsl --install` leaves one, from the fleet's wsl2 walls fixture.""" import contextlib, importlib.machinery, importlib.util, io, json, os, shutil, stat, sys, tempfile, unittest KIT = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) @@ -17,31 +18,57 @@ '[ "$prev" = --shell ] && sh=$a; prev=$a; done\n' 'echo "$a:x:$uid:$uid::/home/$a:$sh" >>"$R/etc/passwd"\n' 'echo "$a:x:$uid:" >>"$R/etc/group"\nmkdir -p "$R/home/$a"\n', + 'usr/sbin/usermod': 'R="${0%/usr/sbin/usermod}"; echo "$*" >>"$R/log/usermod"\n' + '[ "$1" = --shell ] && sed -i "s#^$3:\\(\\([^:]*:\\)\\{5\\}\\)[^:]*\\$#$3:\\1$2#" "$R/etc/passwd"\n', # T3's installer, run by root through env -i (scripts/install.sh with T3CODE_HOME=/opt/pc-agents/t3) 'usr/bin/env': 'R="${0%/usr/bin/env}"; echo "$*" >>"$R/log/install"\n[ -e "$R/log/install-fail" ] && exit 1\n' + # install-hang: a download that never ends, its pid noted (the group kill must end it) + '[ -e "$R/log/install-hang" ] && { sleep 60 & echo $! >"$R/log/hang.pid"; wait; }\n' 'for a; do case "$a" in T3CODE_VERSION=*) v=${a#*=};; esac; done\n' 'd="$R/opt/pc-agents/t3/runtime/versions/$v"; mkdir -p "$d"; printf "#!/bin/sh\\n# t3 $v\\n" >"$d/t3"\n' 'chmod 755 "$d/t3"\n', - 'usr/bin/systemd-run': 'R="${0%/usr/bin/systemd-run}"; printf "%s\\n" "$@" >"$R/log/systemd-run"\n', + # systemd-run: like the real one, a bare command name is looked up in the CALLER's PATH, before anything runs + 'usr/bin/systemd-run': 'R="${0%/usr/bin/systemd-run}"; printf "%s\\n" "$@" >"$R/log/systemd-run"\n' + 'while [ $# -gt 0 ] && [ "$1" != -- ]; do shift; done; cmd=$2\n' + 'case "$cmd" in */*) [ -x "$R$cmd" ] || [ -x "$cmd" ];; *) command -v "$cmd" >/dev/null;; esac' + ' || { echo "Failed to find executable $cmd: No such file or directory" >&2; exit 1; }\n', 'usr/bin/systemctl': 'R="${0%/usr/bin/systemctl}"; echo "$*" >>"$R/log/systemctl"; S="$R/log/units"; mkdir -p "$S"\n' '[ -e "$R/log/systemctl-fail" ] && { echo "systemctl: failed (test)" >&2; exit 1; }\n' - 'run() { grep "^ExecStart=" "$R/etc/systemd/system/$1" >"$S/$1.active" 2>/dev/null || :; }\n' + # what a started unit runs with: its ExecStart and T3's policy + # a start: refused while the start limit is hit ($S/U.limit, until reset-failed); with + # /log/t3-crash T3 exits at once, with /log/t3-dies after the first look (show) at it + 'run() { [ -e "$S/$1.limit" ] && { echo "$1: start request repeated too quickly" >&2; ' + 'return 1; }\nrm -f "$S/$1.seen"\n' + '[ "$1" = t3code-agents.service ] && [ -e "$R/log/t3-crash" ] && { rm -f "$S/$1.active"; return; }\n' + '{ grep "^ExecStart=" "$R/etc/systemd/system/$1"; ' + 'cat "$R/etc/t3code/policy.json"; } >"$S/$1.active" 2>/dev/null || :\n' + 'echo $$ >"$S/$1.pid"; python3 -I -c "import time; print(time.monotonic_ns() // 1000)" ' + '>"$S/$1.start"; }\n' 'case "$1" in\n' 'enable|disable) v=$1; shift; now=; [ "$1" = --now ] && { now=1; shift; }\n' ' for u; do if [ $v = enable ]; then : >"$S/$u.enabled"; [ -n "$now" ] && ' - '{ [ -e "$S/$u.active" ] || run "$u"; }\n' + '{ [ -e "$S/$u.active" ] || run "$u" || exit 1; }\n' ' else rm -f "$S/$u.enabled"; [ -z "$now" ] || rm -f "$S/$u.active"; fi; done;;\n' - 'restart) shift; for u; do run "$u"; done;;\n' + 'restart) shift; for u; do run "$u" || exit 1; done;;\n' + 'reset-failed) shift; for u; do rm -f "$S/$u.limit"; done;;\n' 'stop) shift; for u; do rm -f "$S/$u.active"; done;;\n' 'is-active) [ -e "$S/$2.active" ] && echo active || { echo inactive; exit 3; };;\n' 'show) u=$4; f=; [ -e "$R/etc/systemd/system/$u" ] && f=/etc/systemd/system/$u\n' + ' [ -e "$R/log/t3-dies" ] && [ -e "$S/$u.active" ] && { [ -e "$S/$u.seen" ] && ' + 'rm -f "$S/$u.active"; : >"$S/$u.seen"; }\n' ' st=disabled; [ -e "$S/$u.enabled" ] && st=enabled\n' ' [ "$u" = pc-agents-walls.service ] && st=static\n' ' echo "LoadState=${f:+loaded}"; echo "FragmentPath=$f"\n' ' echo "DropInPaths=$(cat "$S/$u.dropins" 2>/dev/null)"; echo "UnitFileState=$st"\n' ' [ -e "$S/reload" ] && echo NeedDaemonReload=yes || echo NeedDaemonReload=no\n' - ' [ -e "$S/$u.active" ] && echo ActiveState=active || echo ActiveState=inactive;;\n' + ' if [ -e "$S/$u.active" ]; then echo ActiveState=active; echo "MainPID=$(cat "$S/$u.pid")"\n' + ' else echo ActiveState=inactive; echo MainPID=0; fi; echo NRestarts=0\n' + ' echo "ExecMainStartTimestampMonotonic=$(cat "$S/$u.start" 2>/dev/null || echo 0)";;\n' 'esac\n', + # apt-get: update, or install nftables (the fake nft kept in /log/nft-package); /log/apt-fail makes it fail + 'usr/bin/apt-get': 'R="${0%/usr/bin/apt-get}"; echo "$*" >>"$R/log/apt-get"\n' + '[ -e "$R/log/apt-fail" ] && { echo "E: no network (test)" >&2; exit 100; }\n' + '[ "$1" = install ] && cp "$R/log/nft-package" "$R/usr/sbin/nft"; exit 0\n', 'usr/sbin/nft': 'R="${0%/usr/sbin/nft}"\ncase "$1" in -f) cp "$R$2" "$R/log/nft-loaded";;\n' 'list) cat "$R/log/nft-loaded" 2>/dev/null || exit 1;; esac\n', } @@ -67,6 +94,7 @@ def load(): class Case(unittest.TestCase): def setUp(self): + k.SETTLE, k.SETTLE_STEP = 0.3, 0.05 # setup watches T3 0.3 s here, not 15 self.umask = os.umask(0o022) self.root = os.path.realpath(tempfile.mkdtemp(prefix='pc-agents-test-')) os.chmod(self.root, 0o755) @@ -79,8 +107,9 @@ def setUp(self): files['etc/wsl.conf'] = '[boot]\nsystemd=true\n' files['etc/passwd'] = 'root:x:0:0:root:/root:/bin/bash\nnobody:x:65534:65534::/nonexistent:/usr/sbin/nologin\n' files['etc/group'] = 'root:x:0:\nsudo:x:27:\nnogroup:x:65534:\n' + files['proc/sys/kernel/random/boot_id'] = '0f8e4f9c-1b6a-4d2e-9a57-3c1d2e4f5a6b\n' files['proc/net/route'] = 'Iface\tDestination\tGateway\tFlags\nboth\t00000000\t011014AC\t0003\n' - files['log/'] = '' + files['log/'] = files['opt/'] = files['etc/systemd/system/'] = '' # as in a fresh Ubuntu files.update({rel: {'text': '#!/bin/sh\n' + text, 'mode': '755'} for rel, text in TOOLS.items()}) for rel, value in files.items(): self.write('/' + rel, value) diff --git a/fork-kit/pc-agents/tests/setup.test.py b/fork-kit/pc-agents/tests/setup.test.py new file mode 100644 index 0000000000000..dbd4e25024801 --- /dev/null +++ b/fork-kit/pc-agents/tests/setup.test.py @@ -0,0 +1,552 @@ +"""pc-agents setup on a fake WSL2 distro (fake_distro.py): every file and mode, T3 enabled only after every wall +holds, a second run changes nothing, a run stopped at any write or a failed T3 install finishes on the next and +restarts T3, the firewall is loaded before T3 is downloaded, the owner steps run as the agent only under the unit's +walls, and a distro that is not fresh is refused with nothing changed. + + python3 -I -B fork-kit/pc-agents/tests/setup.test.py +""" +import json, os, re, shutil, signal, stat, subprocess, sys, time, unittest + +sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) +from fake_distro import CONFIG, KIT, REPO, SHA1, SHA2, Case, blob, cli, k # noqa: E402 + + +class Setup(Case): + def test_fresh_distro_serves_only_after_the_walls_hold(self): + code, out = self.setup_run() + self.assertEqual(code, 3, out) + self.assertIn('FAIL automount', out) + self.assertIn('wsl --terminate', out) + self.assertFalse([line for line in self.log() if line.startswith('enable')]) + self.assertIn('disable --now t3code-agents.service pc-agents-walls.timer', self.log()) + self.terminate() + code, out = self.setup_run() + self.assertEqual(code, 0, out) + self.assertNotIn('FAIL', out) + self.assertIn('enable --now t3code-agents.service pc-agents-walls.timer', self.log()) + self.assertIn('tailscale up --advertise-tags=tag:pc-agents --accept-dns=false', out) + self.assertFalse(os.path.lexists(self.path(k.PENDING))) + self.assertIn('versions/1.2.3/t3 serve', self.read('/log/units/t3code-agents.service.active')) + + def test_every_file_and_permission(self): + self.ready() + c = k.config() + record = k.record() + for p, (text, mode) in k.render(c).items(): + st = os.lstat(self.path(p)) + self.assertTrue(stat.S_ISREG(st.st_mode), p) + self.assertEqual((stat.S_IMODE(st.st_mode), st.st_uid), (mode, os.geteuid()), p) + self.assertEqual(self.read(p), text, p) + self.assertEqual(record[p], {k.sha(text)}, p) + for p in k.masks(c): + self.assertEqual(os.readlink(self.path(p)), '/dev/null', p) + self.assertEqual(os.readlink(self.path('/opt/pc-agents/current')), SHA1) + for src, (dst, mode) in k.CODE.items(): + p = self.path(f'/opt/pc-agents/{SHA1}/{dst}') + self.assertEqual(stat.S_IMODE(os.stat(p).st_mode), mode, dst) + self.assertEqual(blob(p), blob(os.path.join(REPO, src)), dst) + self.assertIn('agent:x:1100:1100::/home/agent:/usr/sbin/nologin', self.read('/etc/passwd')) + self.assertIn('/etc/systemd/system/user@1100.service', k.masks(c)) + t3 = self.path('/opt/pc-agents/t3/runtime/versions/1.2.3/t3') + self.assertEqual(os.stat(t3).st_uid, os.geteuid()) + self.assertEqual(record['/opt/pc-agents/t3/runtime/versions/1.2.3/t3'], {k.file_sha(t3[len(self.root):])}) + self.assertFalse(os.path.lexists(self.path('/home/agent/.t3'))) + install = self.log('install')[0].split() # root runs upstream's install.sh, in a clean environment + for arg in ('-i', 'HOME=/root', 'T3CODE_HOME=/opt/pc-agents/t3', 'T3CODE_INSTALL_BIN_DIR=/opt/pc-agents/t3/bin', + 'T3CODE_VERSION=1.2.3', 'T3CODE_RELEASE_BASE_URL=https://example.com/releases', + f'/opt/pc-agents/{SHA1}/share/install.sh'): + self.assertIn(arg, install) + + def test_what_the_files_say(self): + self.ready() + policy = json.loads(self.read('/etc/t3code/policy.json')) + self.assertEqual(policy['settings'], {'agentSlots.maxPerMachine': {'value': 2, 'locked': True}, + 'automaticUpdates': {'value': False, 'locked': True}, + 'linkedEnvironmentsEnabled': {'value': True, 'locked': True}}) + self.assertEqual(policy['limits']['maxRuntimeMode'], 'full-access') + self.assertEqual((policy['limits']['connect'], policy['limits']['webhookTasks']), (False, False)) + wsl = self.read('/etc/wsl.conf') + for line in ('[automount]\nenabled=false\nmountFsTab=false', '[interop]\nenabled=false\nappendWindowsPath=false', + '[boot]\nsystemd=true', 'generateResolvConf=false', '[user]\ndefault=root'): + self.assertIn(line, wsl) + self.assertNotIn('default=agent', wsl) + unit = self.read('/etc/systemd/system/t3code-agents.service') + for line in ('User=agent', 'ExecStartPre=+/opt/pc-agents/current/sbin/pc-agents check', 'ProtectSystem=strict', + 'InaccessiblePaths=-/mnt -/run/WSL -/tmp/.X11-unix', 'MemoryMax=8G', 'CPUQuota=400%', + '-/run/tailscale', 'IPAddressAllow=100.101.102.103/32', + 'ExecStart=/opt/pc-agents/t3/runtime/versions/1.2.3/t3 serve', 'T3CODE_PORT=3800', + 'RestrictAddressFamilies=AF_UNIX AF_INET AF_INET6 AF_NETLINK', 'SystemCallFilter=~io_uring_setup'): + self.assertIn(line, unit) + self.assertNotIn('AF_VSOCK', unit) + nft = self.read('/etc/pc-agents/firewall.nft') + for line in ('meta skuid != 1100 accept', '172.16.0.0/12', '10.0.0.0/8', '192.168.0.0/16', '100.64.0.0/10', + 'fc00::/7'): + self.assertIn(line, nft) + self.assertEqual(self.read('/etc/resolv.conf').splitlines()[1:], ['nameserver 1.1.1.1', 'nameserver 9.9.9.9']) + + def test_second_run_changes_nothing(self): + self.ready() + before = self.snapshot() + os.unlink(self.path('/log/systemctl')) + code, out = self.setup_run() + self.assertEqual(code, 0, out) + self.assertEqual(self.snapshot(), before) + self.assertFalse([line for line in self.log() if line.startswith('restart')]) + self.assertEqual(len(self.log('install')), 1) # T3 downloaded once + + def test_new_release_or_config_restarts_t3(self): + self.ready() + os.unlink(self.path('/log/systemctl')) + self.assertEqual(self.setup_run(SHA2)[0], 0) + self.assertEqual(os.readlink(self.path('/opt/pc-agents/current')), SHA2) + self.assertIn('restart t3code-agents.service', self.log()) + os.unlink(self.path('/log/systemctl')) + self.machine({**CONFIG, 'memoryMax': '4G'}) + self.assertEqual(self.setup_run(SHA2)[0], 0) + self.assertIn('MemoryMax=4G', self.read('/etc/systemd/system/t3code-agents.service')) + self.assertIn('restart t3code-agents.service', self.log()) + + def test_stopped_half_way_then_run_again(self): + self.terminate() + self.assertEqual(self.setup_run()[0], 0) + whole = {p: v[4] for p, v in self.snapshot().items()} + real_put, writes = cli.put, [] + + def counting(*a, **kw): + writes.append(a[0]) + return real_put(*a, **kw) + cli.put = counting + self.addCleanup(setattr, cli, 'put', real_put) + shutil.rmtree(self.root) + self.setUp() + self.terminate() + self.setup_run() + total = len(writes) + self.assertGreater(total, 20) + for stop in range(1, total + 1): + shutil.rmtree(self.root) + self.setUp() + self.terminate() + + def stopping(*a, n=[0], **kw): + n[0] += 1 + if n[0] == stop: + raise KeyboardInterrupt + return real_put(*a, **kw) + cli.put = stopping + with self.assertRaises(KeyboardInterrupt): + self.setup_run() + cli.put = real_put + if stop < total: # the last write is the marker right before the restart, after the enable + self.assertFalse([line for line in self.log() if line.startswith('enable')], f'stopped at write {stop}') + code, out = self.setup_run() + self.assertEqual(code, 0, f'stopped at write {stop} ({writes[stop - 1]}): {out}') + self.assertEqual({p: v[4] for p, v in self.snapshot().items()}, whole, f'stopped at write {stop}') + self.assertIn('enable --now t3code-agents.service pc-agents-walls.timer', self.log()) + + def test_t3_install_fails_then_works(self): + self.terminate() + self.write('/log/install-fail', '') + code, out = self.setup_run() + self.assertEqual(code, 1, out) + self.assertIn('T3 install failed', out) + self.assertFalse([line for line in self.log() if line.startswith('enable')]) + os.unlink(self.path('/log/install-fail')) + self.assertEqual(self.setup_run()[0], 0) + + def test_failed_t3_install_on_upgrade_still_restarts_t3_on_the_next_run(self): + self.ready() + os.unlink(self.path('/log/systemctl')) + self.machine({**CONFIG, 't3Version': '1.2.4'}) + self.write('/log/install-fail', '') + code, out = self.setup_run() + self.assertEqual(code, 1, out) + self.assertIn('T3 restarts once it succeeds', out) + self.assertTrue(os.path.isfile(self.path(k.PENDING))) + os.unlink(self.path('/log/install-fail')) + self.assertEqual(self.setup_run()[0], 0) + self.assertIn('restart t3code-agents.service', self.log()) + self.assertIn('versions/1.2.4/t3 serve', self.read('/log/units/t3code-agents.service.active')) + self.assertFalse(os.path.lexists(self.path(k.PENDING))) + + def stop_after(self, path, before=False): + """Run setup, stopped (as by a kill) right after path is written, or right before.""" + real_put = cli.put + + def stopping(p, *a, **kw): + if p == path and before: + raise KeyboardInterrupt + changed = real_put(p, *a, **kw) + if p == path: + raise KeyboardInterrupt + return changed + cli.put = stopping + try: + with self.assertRaises(KeyboardInterrupt): + self.setup_run() + finally: + cli.put = real_put + + def test_killed_after_the_policy_is_written_still_restarts_t3(self): + self.ready() + os.unlink(self.path('/log/systemctl')) + self.machine({**CONFIG, 'maxRuntimeMode': 'approval-required'}) + self.stop_after('/etc/t3code/policy.json') + running = '/log/units/t3code-agents.service.active' + self.assertNotIn('approval-required', self.read(running)) # the old T3 still runs with the old policy + self.assertIn('approval-required', self.read('/etc/t3code/policy.json')) + code, out = self.setup_run() + self.assertEqual(code, 0, out) + self.assertIn('restart t3code-agents.service', self.log()) + self.assertIn('approval-required', self.read(running)) + os.unlink(self.path('/log/systemctl')) + self.assertEqual(self.setup_run()[0], 0) # and only once + self.assertFalse([line for line in self.log() if line.startswith('restart')]) + + def test_killed_at_a_write_then_the_machine_file_edited(self): + unit = '/etc/systemd/system/t3code-agents.service' + for before in (True, False): # the record already names the new file, which is or is not written yet + with self.subTest(before=before): + self.ready() + self.machine({**CONFIG, 'memoryMax': '4G'}) + self.stop_after(unit, before) + self.machine({**CONFIG, 'memoryMax': '2G'}) # the owner changes their mind before running again + code, out = self.setup_run() + self.assertEqual(code, 0, out) + self.assertIn('MemoryMax=2G', self.read(unit)) + self.assertEqual(k.record()[unit], {k.file_sha(unit)}) + shutil.rmtree(self.root) + self.setUp() + + def test_nftables_installed_after_a_package_list_update(self): + os.replace(self.path('/usr/sbin/nft'), self.path('/log/nft-package')) + self.terminate() + self.write('/log/apt-fail', '') + code, out = self.setup_run() + self.assertEqual(code, 1, out) + self.assertIn('apt-get update && apt-get install -y nftables', out) + os.unlink(self.path('/log/apt-fail')) + code, out = self.setup_run() + self.assertEqual(code, 0, out) + self.assertEqual(self.log('apt-get'), ['update', 'update', 'install -y nftables']) + + def test_t3_that_does_not_start_is_reported(self): + self.ready() + self.write('/log/t3-crash', '') + self.machine({**CONFIG, 'memoryMax': '4G'}) + code, out = self.setup_run() + self.assertEqual(code, 1, out) + self.assertIn('journalctl -u t3code-agents.service', out) + self.assertTrue(os.path.isfile(self.path(k.PENDING))) + os.unlink(self.path('/log/t3-crash')) + self.assertEqual(self.setup_run()[0], 0) + self.assertFalse(os.path.lexists(self.path(k.PENDING))) + + + def test_t3_that_dies_a_moment_after_the_restart_is_reported(self): + self.ready() + self.write('/log/t3-dies', '') # T3 starts, then exits on something it cannot apply + self.machine({**CONFIG, 'memoryMax': '4G'}) + code, out = self.setup_run() + self.assertEqual(code, 1, out) + self.assertIn('T3 did not start or stay up (T3 is inactive)', out) + self.assertNotIn('done:', out) + self.assertTrue(os.path.isfile(self.path(k.PENDING))) + os.unlink(self.path('/log/t3-dies')) + code, out = self.setup_run() + self.assertEqual(code, 0, out) + self.assertFalse(os.path.lexists(self.path(k.PENDING))) + + def test_rerun_within_the_start_limit_still_restarts_t3(self): + self.ready() + self.write('/log/units/t3code-agents.service.limit', '') # systemd: start request repeated too quickly + self.machine({**CONFIG, 'memoryMax': '4G'}) + os.unlink(self.path('/log/systemctl')) + code, out = self.setup_run() + self.assertEqual(code, 0, out) + log = self.log() + self.assertLess(log.index('reset-failed t3code-agents.service'), log.index('restart t3code-agents.service')) + + def test_agent_of_an_older_kit_gets_no_login_shell(self): + self.ready() + self.write('/etc/passwd', self.read('/etc/passwd').replace('/home/agent:/usr/sbin/nologin', '/home/agent:/bin/bash')) + code, out = self.setup_run() + self.assertEqual(code, 0, out) + self.assertIn('agent:x:1100:1100::/home/agent:/usr/sbin/nologin', self.read('/etc/passwd')) + self.assertEqual(self.log('usermod'), ['--shell /usr/sbin/nologin agent']) + os.unlink(self.path('/log/usermod')) + self.assertEqual(self.setup_run()[0], 0) + self.assertEqual(self.log('usermod'), []) # once + + +class RestartPending(Case): + """The restart wall with setup: a T3 that started before setup's last change fails `check`.""" + stop_after = Setup.stop_after + + def test_interrupted_upgrade_keeps_old_t3_and_check_fails(self): # lenspcas1 #1 + self.ready() + self.machine({**CONFIG, 't3Version': '1.2.4', 'maxRuntimeMode': 'approval-required'}) + self.write('/log/install-fail', '') + self.assertEqual(self.setup_run()[0], 1) + running = self.read('/log/units/t3code-agents.service.active') + self.assertIn('versions/1.2.3/t3 serve', running) + self.assertIn('full-access', running) + code, out = self.call(cli.main, ['as-agent', '--', 'sh']) + self.assertEqual(code, 1, out) + self.assertEqual(self.log('systemd-run'), []) + os.unlink(self.path('/log/systemctl')) + code, out = self.check('--enforce') + self.assertNotEqual(code, 0, out) + self.assertIn('FAIL restart', out) + self.assertIn('stop t3code-agents.service', self.log()) + os.unlink(self.path('/log/install-fail')) + self.assertEqual(self.setup_run()[0], 0) + self.assertIn('versions/1.2.4/t3 serve', self.read('/log/units/t3code-agents.service.active')) + self.assertEqual(self.check('--enforce')[0], 0) + + def test_killed_after_the_policy_is_written_check_fails(self): # lenspcac1 #3 + self.ready() + self.machine({**CONFIG, 'maxRuntimeMode': 'approval-required'}) + self.stop_after('/etc/t3code/policy.json') + self.assertNotIn('approval-required', self.read('/log/units/t3code-agents.service.active')) + self.assertNotEqual(self.check('--enforce')[0], 0) + + def test_upgrade_killed_at_any_write_fails_check_until_run_again(self): + def files(): + return {p: v[4] for p, v in self.snapshot().items() if p not in (k.PENDING, k.RECORD)} + upgrade = {**CONFIG, 'memoryMax': '4G', 'maxRuntimeMode': 'approval-required'} + real_put, writes = cli.put, [] + self.addCleanup(setattr, cli, 'put', real_put) + self.ready() + self.machine(upgrade) + cli.put = lambda *a, **kw: writes.append(a[0]) or real_put(*a, **kw) + self.assertEqual(self.setup_run(SHA2)[0], 0) + cli.put = real_put + self.assertGreater(len(writes), 10) + for stop in range(1, len(writes) + 1): + shutil.rmtree(self.root) + self.setUp() + self.ready() + self.machine(upgrade) + before = files() + + def stopping(*a, n=[0], **kw): + n[0] += 1 + if n[0] == stop: + raise KeyboardInterrupt + return real_put(*a, **kw) + cli.put = stopping + with self.assertRaises(KeyboardInterrupt): + self.setup_run(SHA2) + cli.put = real_put + code, out = self.check() + if files() != before: # something T3 runs with changed + self.assertIn('FAIL restart', out, f'stopped at write {stop} ({writes[stop - 1]})') + code, out = self.setup_run(SHA2) + self.assertEqual(code, 0, f'stopped at write {stop}: {out}') + code, out = self.check() + self.assertEqual(code, 0, f'stopped at write {stop}: {out}') + + def test_check_passes_on_its_own_restart_and_a_kill_while_watching_restarts_again(self): + self.ready() + self.machine({**CONFIG, 'memoryMax': '4G'}) + real, seen = cli.stays_up, [] + + def watching(): # killed while setup watches the restarted T3 + seen.append(self.check()) + raise KeyboardInterrupt + cli.stays_up = watching + try: + with self.assertRaises(KeyboardInterrupt): + self.setup_run() + finally: + cli.stays_up = real + self.assertEqual(seen[0][0], 0, seen[0][1]) # the T3 setup just restarted is not stale + self.assertTrue(os.path.isfile(self.path(k.PENDING))) + self.assertEqual(self.check()[0], 0) + os.unlink(self.path('/log/systemctl')) + self.assertEqual(self.setup_run()[0], 0) + self.assertIn('restart t3code-agents.service', self.log()) # once more, then the marker goes + self.assertFalse(os.path.lexists(self.path(k.PENDING))) + + def test_marker_of_an_older_kit_fails_until_setup_runs_again(self): + self.ready() + self.write(k.PENDING, 'setup changed what T3 runs with; the next good run restarts T3 and removes this\n') + self.assertIn('FAIL restart', self.check()[1]) + os.unlink(self.path('/log/systemctl')) + self.assertEqual(self.setup_run()[0], 0) + self.assertIn('restart t3code-agents.service', self.log()) + self.assertFalse(os.path.lexists(self.path(k.PENDING))) + self.assertEqual(self.check()[0], 0) + + +class Signals(Case): + def test_sigterm_or_sighup_kills_the_running_download_too(self): + for sig in (signal.SIGTERM, signal.SIGHUP): + with self.subTest(signal=sig.name): + self.write('/log/install-hang', '') + p = subprocess.Popen([sys.executable, '-I', os.path.join(KIT, 'sbin', 'pc-agents'), 'setup', + '--source', REPO, '--sha', SHA1], env={'PC_AGENTS_ROOT': self.root, + 'PATH': '/usr/bin:/bin'}, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL) + hang = self.path('/log/hang.pid') + try: + end = time.monotonic() + 60 + while not os.path.exists(hang) and time.monotonic() < end and p.poll() is None: + time.sleep(0.05) + self.assertTrue(os.path.exists(hang), 'the fake download did not start') + time.sleep(0.2) + pid = int(self.read('/log/hang.pid')) + p.send_signal(sig) + self.assertEqual(p.wait(timeout=30), 128 + sig) + end = time.monotonic() + 10 + while time.monotonic() < end and alive(pid): + time.sleep(0.05) + self.assertFalse(alive(pid), 'the download outlived pc-agents') + finally: + if p.poll() is None: + p.kill() + p.wait() + if os.path.exists(hang): + try: + os.kill(int(self.read('/log/hang.pid')), signal.SIGKILL) + except (ProcessLookupError, ValueError): + pass + os.unlink(hang) + + +def alive(pid): + """pid runs (a zombie, ended but not yet reaped, does not count).""" + try: + with open(f'/proc/{pid}/stat', encoding='ascii') as f: + return f.read().rsplit(')', 1)[1].split()[0] != 'Z' + except FileNotFoundError: + return False + + +class StaleFirewall(Case): + def test_tightened_allow_with_failed_install(self): + self.machine({**CONFIG, 'allow': ['192.168.1.0/24']}) + self.ready() + os.unlink(self.path('/log/systemctl')) + self.machine({**CONFIG, 't3Version': '1.2.4'}) + self.write('/log/install-fail', '') + self.assertEqual(self.setup_run()[0], 1) + self.assertNotIn('192.168.1.0/24', self.read('/log/nft-loaded')) # new table already loaded + status, out = self.check() + fails = [line for line in out.splitlines() if line.startswith('FAIL')] + self.assertEqual(status, 1) # the old T3 still runs: only the restart wall fails, + self.assertEqual([line.split()[1] for line in fails], ['restart'], out) # the firewall agrees with the file + + def test_a_loaded_table_that_is_not_the_files_fails(self): + self.ready() + self.write('/log/nft-loaded', self.read('/log/nft-loaded').replace('10.0.0.0/8', '10.0.0.0/9')) + os.unlink(self.path('/log/systemctl')) + code, out = self.check('--enforce') + self.assertEqual(code, 1, out) + self.assertIn('FAIL firewall', out) + self.assertIn('stop t3code-agents.service', self.log()) + + +class AsAgent(Case): + def test_runs_as_the_agent_under_the_units_walls(self): + self.ready() + tool = 'pc-agents-test-tool-only-in-the-agents-path' # like claude in ~/.local/bin: not on root's PATH + code, out = self.call(cli.main, ['as-agent', '--', tool, 'login']) + self.assertEqual(code, 0, out) + argv = self.log('systemd-run') + self.assertEqual(argv[-5:], ['--', '/usr/bin/env', '--', tool, 'login']) # env looks it up in PATH below + self.assertTrue([a for a in argv if re.fullmatch(r'--unit=pc-agents-as-agent-\d+', a)], argv) + for arg in ('--pty', '--uid=agent', '--gid=agent', '--setenv=HOME=/home/agent', '--setenv=SHELL=/bin/bash', + '--setenv=PATH=/home/agent/.local/bin:/usr/local/bin:/usr/bin:/bin', + '--property=InaccessiblePaths=-/mnt -/run/WSL -/tmp/.X11-unix -/usr/lib/wsl -/run/tailscale'): + self.assertIn(arg, argv) + for line in k.sandbox(k.config()): + self.assertIn('--property=' + line, argv) + + def test_refused_when_a_wall_does_not_hold(self): + self.ready() + self.write('/var/lib/systemd/linger/agent', '') + code, out = self.call(cli.main, ['as-agent', '--', 'claude']) + self.assertEqual(code, 1, out) + self.assertIn('not running anything as the agent', out) + self.assertEqual(self.log('systemd-run'), []) + + +class Refused(Case): + def refused(self, why): + before = self.snapshot() + code, out = self.setup_run() + self.assertEqual(code, 1, out) + self.assertIn('refused, nothing changed', out) + self.assertIn(why, out) + self.assertEqual(self.snapshot(), before) + self.assertEqual(self.log(), []) + + def test_a_distro_with_its_own_user(self): + with open(self.path('/etc/passwd'), 'a') as f: + f.write('someone:x:1000:1000::/home/someone:/bin/bash\n') + self.refused('already has login users (someone)') + self.refused('userdel -r') + + def test_folders_that_are_not_roots_alone(self): + for d in ('/opt/pc-agents', '/etc/systemd/system'): + os.makedirs(self.path(d), exist_ok=True) + os.chmod(self.path(d), 0o775) + self.refused(f'{d} is not owned by uid') + os.chmod(self.path(d), 0o755) + + def test_a_wsl_conf_of_its_own(self): + self.write('/etc/wsl.conf', '[boot]\nsystemd=true\n[user]\ndefault=someone\n') + self.refused('/etc/wsl.conf was not written by this kit') + + def test_a_policy_of_its_own(self): + self.write('/etc/t3code/policy.json', '{"schemaVersion": 1}\n') + self.refused('/etc/t3code/policy.json was not written by this kit') + + def test_an_agent_that_is_not_the_kits(self): + with open(self.path('/etc/passwd'), 'a') as f: + f.write('agent:x:1200:1200::/home/agent:/bin/bash\n') + self.refused('user agent has uid 1200, not 1100') + + def test_an_agent_with_sudo(self): + self.ready() + self.write('/etc/group', self.read('/etc/group').replace('sudo:x:27:', 'sudo:x:27:agent')) + os.unlink(self.path('/log/systemctl')) + self.refused('user agent is in other groups (sudo)') + + def test_an_unmasked_ssh(self): + self.write('/etc/systemd/system/ssh.service', '[Service]\nExecStart=/usr/sbin/sshd\n') + self.refused('ssh.service exists and is not a mask') + + def test_a_file_edited_after_setup(self): + self.ready() + with open(self.path('/etc/systemd/system/t3code-agents.service'), 'a') as f: + f.write('[Service]\nUser=root\n') + os.unlink(self.path('/log/systemctl')) + self.refused('t3code-agents.service was not written by this kit, or was edited since') + + def test_the_examples_are_valid_and_the_same(self): + with open(os.path.join(REPO, 'fork-kit', 'common', 'machine.example.json'), encoding='utf-8') as f: + example = json.load(f)['pc-agents'] + with open(os.path.join(KIT, 'README.md'), encoding='utf-8') as f: + readme = re.search(r'```json\n(\{\n "version".*?)\n```', f.read(), re.S).group(1) + self.assertEqual(json.loads(readme)['pc-agents'], example) + self.machine(example) + self.assertEqual(k.config()['agentUid'], 1100) + self.assertEqual(k.config()['lockedSettings'], {'agentSlots.enabled': True, 'agentSlots.maxPerMachine': 2}) + + def test_bad_config(self): + for change, why in (({'lockedSettings': {'linkedEnvironmentsEnabled': False}}, 'set by the kit itself'), + ({'lockedSettings': {'agentSlots.maxPerMachine.x': {'y': 2}}}, 'one field per dotted path'), + ({'nameservers': ['192.168.1.1']}, 'only public addresses'), + ({'t3Version': 'latest'}, 'an exact version'), + ({'maxRuntimeMode': 'root'}, 'one of')): + self.machine({**CONFIG, **change}) + code, out = self.setup_run() + self.assertEqual(code, 2, out) + self.assertIn(why, out) + self.assertFalse(os.path.lexists(self.path('/etc/t3code/policy.json'))) + + +if __name__ == '__main__': + unittest.main()