diff --git a/fork-kit/desk/README.md b/fork-kit/desk/README.md index 575d23b507ea0..8836e147557cb 100644 --- a/fork-kit/desk/README.md +++ b/fork-kit/desk/README.md @@ -115,7 +115,8 @@ distro (`-Distro`, default `fork-desk`, Ubuntu 24.04, installed with `--version leaves a mark in `%LOCALAPPDATA%\fork-desk`, so the next run asks no takeover question, and a broken one names `wsl --unregister DISTRO`); `.wslconfig`'s `[wsl2]` memory cap (`-MemoryGB`, default three quarters of the RAM, at most the RAM less 6 GB, never under 4 GB; one that leaves Windows under 2 GB is -refused; it is the least cap: a higher one already there, raised by hand, is kept), swap (`-SwapGB`, 4), NAT and DNS tunnelling, for every WSL distro (read and written as UTF-8 without a +refused; without `-MemoryGB`, a higher one already there, raised by hand, is kept up to the RAM less 2 GB; one +WSL cannot read, such as `26 GB` or `26gb`, is rewritten, since WSL would run on half the RAM), swap (`-SwapGB`, 4), NAT and DNS tunnelling, for every WSL distro (read and written as UTF-8 without a BOM; a dated copy of the old file is kept, other keys and sections stay; an old `networkingMode` or `dnsTunneling` line under `[experimental]`, the form before WSL 2.0 that can still win over `[wsl2]`, is removed, and the check FAILs while one is there; the question lists each change, old -> new, and the file is written only when it lists one, diff --git a/fork-kit/desk/tests/windows.test.py b/fork-kit/desk/tests/windows.test.py index 30ec03eec48da..ca65aa9765188 100644 --- a/fork-kit/desk/tests/windows.test.py +++ b/fork-kit/desk/tests/windows.test.py @@ -363,31 +363,42 @@ def test_wslconfig_merge_removes_old_experimental_networking_keys(self): self.assertIn(line, r.stdout) @unittest.skipUnless(shutil.which('pwsh'), 'pwsh is not installed') - def test_a_memory_cap_at_or_above_the_minimum_is_kept(self): - """-MemoryGB is the least cap: one a person raised (more agents or jobs in WSL) is neither lowered nor a FAIL, - nor a `wsl --shutdown` on every run (the live check's cap is the one kept).""" + def test_the_memory_cap_kept_is_one_wsl_reads_within_its_bounds(self): + """Without -MemoryGB, a higher cap a person set (more agents or jobs in WSL) is neither lowered nor a FAIL, nor + a `wsl --shutdown` on every run (the live check's cap is the one kept), up to the RAM less 2 GB. With + -MemoryGB, that is the cap. A value WSL itself cannot read (it then runs on half the RAM) is rewritten: its + parser takes a number and B, K, KB, M, MB, G, GB, T or TB exactly, no space, upper case (#296).""" d = tempfile.mkdtemp(prefix='desk-wslconfig-test-') self.addCleanup(shutil.rmtree, d, True) path = os.path.join(d, '.wslconfig') script = os.path.join(d, 'memory.ps1') with open(script, 'w', encoding='ascii') as f: - f.write(wslconfig_part() + '\n$want = [ordered]@{ memory = (WantMemory $args[0] 8); swap = "4GB" }\n' + f.write(wslconfig_part() + '\n$want = [ordered]@{ memory = (WantMemory $args[0] 8 $args[1]); swap = "4GB" }\n' '"want=$($want.memory) kb=$(CapKb $want.memory)"\n' '"notes=$(@(Changes (Read-WslConfig $args[0]) $want) -join \'|\')"\n' '$null = Merge-WslConfig $args[0] $want\n$cfg = Wsl2Keys (Read-WslConfig $args[0])\n' '"check=$(@($want.Keys | Where-Object { $cfg[$_] -ne $want[$_] }).Count)"\n') - for old, want, kb, note in (('memory=26GB', '26GB', 27262976, ''), # raised: kept, no change, no FAIL - ('memory = 30720mb', '30720mb', 31457280, ''), - ('memory=8GB', '8GB', 8388608, ''), - ('memory=9G', '9G', 9437184, ''), - ('memory=4GB', '8GB', 8388608, '[wsl2] memory: 4GB -> 8GB'), # below: raised - ('memory=8191MB', '8GB', 8388608, '[wsl2] memory: 8191MB -> 8GB'), - ('memory=lots', '8GB', 8388608, '[wsl2] memory: lots -> 8GB'), - ('', '8GB', 8388608, '[wsl2] memory: (not set) -> 8GB')): - with self.subTest(old): + for old, most, want, kb, note in ( + ('memory=26GB', 30, '26GB', 27262976, ''), # raised: kept, no change, no FAIL + ('memory=30GB', 30, '30GB', 31457280, ''), # the RAM less 2 GB + ('memory = 30720MB', 30, '30720MB', 31457280, ''), + ('memory=8GB', 30, '8GB', 8388608, ''), + ('memory=9G', 30, '9G', 9437184, ''), + ('memory=31GB', 30, '8GB', 8388608, '[wsl2] memory: 31GB -> 8GB'), # Windows would keep under 2 GB + ('memory=64GB', 30, '8GB', 8388608, '[wsl2] memory: 64GB -> 8GB'), # from a bigger PC + ('memory=16777216TB', 30, '8GB', 8388608, '[wsl2] memory: 16777216TB -> 8GB'), # WSL: 0, its default + ('memory=26 GB', 30, '8GB', 8388608, '[wsl2] memory: 26 GB -> 8GB'), # WSL rejects these two + ('memory=30720mb', 30, '8GB', 8388608, '[wsl2] memory: 30720mb -> 8GB'), + ('memory=4GB', 30, '8GB', 8388608, '[wsl2] memory: 4GB -> 8GB'), # below: raised + ('memory=8191MB', 30, '8GB', 8388608, '[wsl2] memory: 8191MB -> 8GB'), + ('memory=lots', 30, '8GB', 8388608, '[wsl2] memory: lots -> 8GB'), + ('', 30, '8GB', 8388608, '[wsl2] memory: (not set) -> 8GB'), + ('memory=26GB', 8, '8GB', 8388608, '[wsl2] memory: 26GB -> 8GB'), # -MemoryGB 8 lowers it + ('memory=8192MB', 8, '8192MB', 8388608, '')): # -MemoryGB 8, the same size: left as written + with self.subTest(old=old, most=most): with open(path, 'w', encoding='utf-8', newline='\n') as f: f.write(f'[wsl2]\n{old}\nswap=4GB\n') - r = subprocess.run(['pwsh', '-NoProfile', '-NonInteractive', '-File', script, path], + r = subprocess.run(['pwsh', '-NoProfile', '-NonInteractive', '-File', script, path, str(most)], capture_output=True, text=True, timeout=120, stdin=subprocess.DEVNULL) self.assertEqual(r.returncode, 0, r.stdout + r.stderr) self.assertIn(f'want={want} kb={kb}\n', r.stdout) @@ -396,10 +407,12 @@ def test_a_memory_cap_at_or_above_the_minimum_is_kept(self): if not note: self.assertEqual(slurp(path), f'[wsl2]\n{old}\nswap=4GB\n') # left alone - def test_the_minimum_memory_cap_decides_through_wantmemory(self): + def test_the_memory_cap_decides_through_wantmemory(self): src = text().decode('ascii') body = src[src.index('try {'):] - self.assertIn('$want = [ordered]@{ memory = (WantMemory $WslConfig $MemoryGB);', body) + # -MemoryGB given: it is the cap; not given: a higher one is kept up to what leaves Windows 2 GB + self.assertIn('$most = if ($asked) { $MemoryGB } else { $ram - 2 }', body) + self.assertIn('$want = [ordered]@{ memory = (WantMemory $WslConfig $MemoryGB $most);', body) self.assertIn('("WANT_KB=$(CapKb $want.memory)`n" + $Live)', body) # the cap kept, not -MemoryGB self.assertNotIn('$MemoryGB * 1048576', body) diff --git a/fork-kit/desk/windows/start-desk.ps1 b/fork-kit/desk/windows/start-desk.ps1 index a303b775316bd..018a97b4c001d 100644 --- a/fork-kit/desk/windows/start-desk.ps1 +++ b/fork-kit/desk/windows/start-desk.ps1 @@ -2,7 +2,7 @@ # (fork-kit/desk/README.md, "Windows"). Run it in a normal Windows PowerShell window (not as administrator): # powershell -ExecutionPolicy Bypass -File start-desk.ps1 -Repo OWNER/REPO -Sha RELEASE_SHA -MachineFile machine.json # [-FleetConfig fork-fleet.json] [-Distro fork-desk] [-MemoryGB N] [-SwapGB 4] [-Uninstall] -# -MemoryGB is the least [wsl2] memory cap: a higher one already in .wslconfig is kept. +# -MemoryGB is the [wsl2] memory cap; without it, a higher one already in .wslconfig is kept, up to the RAM less 2 GB. # It prints this PC's facts and asks before it changes anything. Safe to run again: every step checks what is there. # Steps: the distro (Ubuntu 24.04); .wslconfig's [wsl2] memory cap and NAT (a dated copy of the old file is kept; # other keys stay); the distro's own settings (the WSL walls: no interop, no Windows PATH or drives, its own @@ -162,16 +162,19 @@ function Merge-WslConfig([string]$path, $want, [scriptblock]$before = {}) { # Move-Item -LiteralPath "$path.new" $path -Force $true } -# A [wsl2] memory= value in kB (8GB, 8192MB, 8G; no unit: bytes), or -1 when it is not a size +# A [wsl2] memory= value in kB (8GB, 8192MB, 8G; no unit: bytes), or -1 when WSL cannot read it. WSL's parser +# (ParseMemorySize) takes a number and B, K, KB, M, MB, G, GB, T or TB exactly: no space before the unit, upper case +# (`26 GB` or `26gb` makes it warn and run on half the RAM). function CapKb([string]$v) { - if ($v -notmatch '^\s*([0-9]{1,13})\s*([KMGT]?)B?\s*$') { return [long]-1 } - [long][math]::Min(1e15, [math]::Floor([double]$Matches[1] * [math]::Pow(1024, ' KMGT'.IndexOf($Matches[2].ToUpper())) / 1024)) + if ($v -cnotmatch '^\s*([0-9]{1,13})([KMGT]?)B?\s*$') { return [long]-1 } + [long][math]::Min(1e15, [math]::Floor([double]$Matches[1] * [math]::Pow(1024, ' KMGT'.IndexOf($Matches[2])) / 1024)) } -# The memory cap to want: at least $gb GB. A higher one already in .wslconfig (a person raised it) is kept as it is -# written, so it is neither lowered nor a FAIL, and the live check holds WSL to it. -function WantMemory([string]$path, [int]$gb) { +# The memory cap to want: $gb GB, or the one already in .wslconfig, kept as it is written (neither lowered nor a FAIL, +# and the live check holds WSL to it) while WSL reads it and it is $gb to $most GB. +function WantMemory([string]$path, [int]$gb, [int]$most) { $have = (Wsl2Keys (Read-WslConfig $path))['memory'] - if ($have -and (CapKb $have) -ge [long]$gb * 1048576) { return $have } + $kb = if ($have) { CapKb $have } else { -1 } + if ($kb -ge [long]$gb * 1048576 -and $kb -le [long]$most * 1048576) { return $have } "${gb}GB" } function WslVersion([string[]]$lines, [string]$name) { # the WSL version of one distro in `wsl -l -v`; 0: not there @@ -224,7 +227,9 @@ try { $ours = Test-Path -LiteralPath $Made if ($have -and -not $ours) { wsl.exe -d $Distro -u root --cd / -- test -e /etc/fork-kit/desk-distro; $ours = $LASTEXITCODE -eq 0 } - $want = [ordered]@{ memory = (WantMemory $WslConfig $MemoryGB); swap = "${SwapGB}GB"; networkingMode = 'nat'; dnsTunneling = 'true'; guiApplications = 'false' } + # -MemoryGB given: it is the cap; not given: a higher one a person set is kept, up to what leaves Windows 2 GB + $most = if ($asked) { $MemoryGB } else { $ram - 2 } + $want = [ordered]@{ memory = (WantMemory $WslConfig $MemoryGB $most); swap = "${SwapGB}GB"; networkingMode = 'nat'; dnsTunneling = 'true'; guiApplications = 'false' } $changes = @(Changes (Read-WslConfig $WslConfig) $want) Write-Host '' Write-Host "Next: the distro $Distro; .wslconfig (for ALL your WSL distros; a dated copy is kept):" -ForegroundColor Yellow diff --git a/fork-kit/pc-agents/README.md b/fork-kit/pc-agents/README.md index 6bba40102d7bf..2dae0c05e223a 100644 --- a/fork-kit/pc-agents/README.md +++ b/fork-kit/pc-agents/README.md @@ -86,7 +86,9 @@ The kit reads the `pc-agents` section of the machine file `/etc/fork-kit/machine | `tailscaleTag` | the tag this PC's Tailscale node gets (`tag:name`) | `t3Version` must be 0.1.7 or newer: older releases have no managed policy, so they would run with none of the kit's -locks and with updates on. Setup and `check` refuse them, naming the key. (`t3 pair`, step 10, came earlier.) +locks and with updates on. A 0.1.7 pre-release counts as older (some 0.1.7 nightlies were built before the policy), +and a pre-release of a later version as that version. Setup and `check` refuse them, naming the key. (`t3 pair`, +step 10, came earlier.) Every T3 setting can be locked through `lockedSettings`, the fork's own settings (`agentSlots`, `usagePacingGate`, `usagePacing`, `waitFor`, ...) too: the managed policy locks any key of the server settings, upstream's or the fork's @@ -153,12 +155,13 @@ Commands are for Windows PowerShell unless a step says otherwise. `` is `wsl -d -u root -- /opt/pc-agents/current/sbin/pc-agents as-agent -- gh auth login --with-token` and paste a fine-grained token for the agents' repos into that window only. It waits without a prompt: paste the token, press Enter, then Ctrl+D. Then give the agent a git identity (its commits fail without one): - `wsl -d -u root -- /opt/pc-agents/current/sbin/pc-agents as-agent -- git config --global user.name ''` + `wsl -d -u root -e /opt/pc-agents/current/sbin/pc-agents as-agent -- git config --global user.name ''` and - `wsl -d -u root -- /opt/pc-agents/current/sbin/pc-agents as-agent -- git config --global user.email ''`, + `wsl -d -u root -e /opt/pc-agents/current/sbin/pc-agents as-agent -- git config --global user.email ''`, with a name of your choice and an address that is no one's private one, for example the token account's noreply - address (`+@users.noreply.github.com`, shown in its GitHub email settings). Run again, they only set - the same values again. + address (`+@users.noreply.github.com`, shown in its GitHub email settings). `-e`, not `--`: wsl runs + the line without root's shell, so the value reaches git as typed (a name with an apostrophe is `'O''Brien'` in + PowerShell). Run again, they only set the same values again. 9. **Keep the distro running:** WSL stops a distro soon after its last window closes, and does not start one when Windows starts. A logon task of your own (no administrator needed) keeps this one running while you are logged in. In PowerShell, as yourself (the same task as the desk starter's, `fork-kit/desk/windows/start-desk.ps1`): @@ -229,9 +232,15 @@ allowed (connection tracking). `SIGHUP` (a closed window) stop setup the same way, the running download's whole group too. - **Start limit:** a rerun within 10 minutes of five failed starts still starts T3 (`systemctl reset-failed` first). - **An older kit's distro:** an `agent` made with a login shell gets `/usr/sbin/nologin` (`usermod`), once. -- **Packages:** setup installs `nftables`, `libatomic1` and `unzip` when `dpkg` does not list one as installed (a - distro set up by an older kit gets the two new ones at its next run; no T3 restart for them). A package install - stopped half-way is finished first (`dpkg --configure -a`); a failed one stops setup, naming the command to run. +- **Packages:** setup installs `nftables`, `libatomic1` and `unzip` when `dpkg` does not list one as installed (held + counts as installed; a distro set up by an older kit gets the two new ones at its next run; no T3 restart for + them). It installs them after it writes its files, so `apt-get` resolves through the machine file's `nameservers` + as they are now, and before it loads the firewall. A package install stopped half-way is finished first + (`dpkg --configure -a`). Nothing asks a question: a changed config file is kept, debconf takes its defaults, and + `apt-get` waits up to 5 minutes for a lock (apt-daily on a new distro). A failed step stops setup, naming the + command to run; the files it wrote stay, and when one of them changed what T3 runs with, the walls check's + `restart` wall fails (and the 2-minute check stops T3) until setup runs again to the end, as after a failed T3 + download. - **A distro that is not fresh:** refused before anything is written, with the reasons: another login user, an `agent` with another uid or in another group, a file the kit would write that it did not write (or that was edited since, for example a hand-written `wsl.conf` or `policy.json`), a cron, at or ssh unit where the kit masks diff --git a/fork-kit/pc-agents/lib/pc_agents.py b/fork-kit/pc-agents/lib/pc_agents.py index c4e7e1c93c2ed..eb5dd564c6e93 100644 --- a/fork-kit/pc-agents/lib/pc_agents.py +++ b/fork-kit/pc-agents/lib/pc_agents.py @@ -45,7 +45,7 @@ KIT_SETTINGS = {'linkedEnvironmentsEnabled': True} VERSION = re.compile(r'(0|[1-9]\d*)\.(0|[1-9]\d*)\.(0|[1-9]\d*)(-[0-9A-Za-z.-]+)?') # The oldest T3 that applies a managed policy (0.1.7; `t3 pair` is older): an older one would run with none of the -# kit's locks, updates on, and quietly. Its pre-releases count as it. +# kit's locks, updates on, and quietly. Its pre-releases are older than it (semver), as some nightlies are. OLDEST_T3 = (0, 1, 7) # Debian packages setup installs when missing: the agent firewall's nft; libatomic1, which Node (and so most agent # tools) needs to start, while the agent has no sudo to install it; unzip, which install steps of packages use. @@ -79,8 +79,8 @@ def is_ip_list(value, net, glob=False, most=64): 'organization': (str, machine_file.REQUIRED, lambda v: None if v.strip() and len(v) <= 80 else '1 to 80 characters, not blank'), 't3Version': (str, machine_file.REQUIRED, lambda v: 'an exact version, no v' if not VERSION.fullmatch(v) else - None if tuple(map(int, VERSION.fullmatch(v).groups()[:3])) >= OLDEST_T3 else - '%d.%d.%d or newer (the first T3 with the managed policy)' % OLDEST_T3), + None if new_enough(v) else '%d.%d.%d or newer, not a pre-release of it (the first T3 with the ' + 'managed policy)' % OLDEST_T3), 't3ReleaseBaseUrl': (str, machine_file.REQUIRED, lambda v: None if re.fullmatch(r'https://[A-Za-z0-9./_-]+', v) else 'an https URL'), 'port': (int, machine_file.REQUIRED, lambda v: None if 1024 <= v <= 65535 else '1024 to 65535'), @@ -97,6 +97,13 @@ def is_ip_list(value, net, glob=False, most=64): } +def new_enough(version): + """Whether VERSION version is OLDEST_T3 or newer, in semver order: a pre-release ranks below its release.""" + m = VERSION.fullmatch(version) + numbers = tuple(map(int, m.groups()[:3])) + return numbers > OLDEST_T3 or numbers == OLDEST_T3 and not m[4] + + def port_range(text): """(low, high) of a 'LOW-HIGH' port range of 1024..65535, or None.""" m = re.fullmatch(r'([1-9]\d{3,4})-([1-9]\d{3,4})', text) @@ -434,12 +441,13 @@ def user_problem(c, need_agent): return f'user {USER} is in other groups ({", ".join(extra)}): it must have none' if extra else None -def run(argv, timeout=120): +def run(argv, timeout=120, env=None): """Run argv (argv[0] under ROOT) in a process group of its own. Past timeout, or when this process stops (Ctrl-C, or SIGTERM and SIGHUP once stop_on_signals() ran), the whole group is killed (a download's curl too); a timeout - returns exit 124 and says so in stderr.""" + returns exit 124 and says so in stderr. env: variables added to this process's own.""" p = subprocess.Popen([ROOT + argv[0]] + argv[1:], stdin=subprocess.DEVNULL, stdout=subprocess.PIPE, - stderr=subprocess.PIPE, text=True, start_new_session=True) + stderr=subprocess.PIPE, text=True, start_new_session=True, + env={**os.environ, **env} if env else None) try: out, err = p.communicate(timeout=timeout) except BaseException as e: diff --git a/fork-kit/pc-agents/sbin/pc-agents b/fork-kit/pc-agents/sbin/pc-agents index 6445aab948902..14cc472cd5024 100755 --- a/fork-kit/pc-agents/sbin/pc-agents +++ b/fork-kit/pc-agents/sbin/pc-agents @@ -193,22 +193,31 @@ def install_t3(c, version, done): def install_packages(): - """Install the k.PACKAGES that are not (fully) installed. A dpkg run stopped half-way is finished first (apt-get - refuses until it is). Returns why not, or None.""" + """Install the k.PACKAGES that are not (fully) installed; held counts as installed. A dpkg run stopped half-way + is finished first (apt-get refuses until it is). Nothing asks a question (stdin is /dev/null): a changed config + file is kept, debconf takes its defaults, and apt-get waits for a lock apt-daily holds. Returns why not, or None.""" r = k.run(['/usr/bin/dpkg-query', '-W', '-f=${Package}\t${Status}\n', *k.PACKAGES]) have = {p for p, _, status in (line.partition('\t') for line in r.stdout.splitlines()) - if status == 'install ok installed'} + if status.endswith(' ok installed')} missing = [p for p in k.PACKAGES if p not in have] if not missing: return None say('installing ' + ' '.join(missing)) - for argv in (['/usr/bin/dpkg', '--configure', '-a'], ['/usr/bin/apt-get', 'update'], - ['/usr/bin/apt-get', 'install', '-y', *missing]): - r = k.run(argv, timeout=900) + quiet = {'DEBIAN_FRONTEND': 'noninteractive'} + r = k.run(['/usr/bin/dpkg', '--force-confdef', '--force-confold', '--configure', '-a'], timeout=900, env=quiet) + if r.returncode: + return (f'cannot install {" ".join(missing)}: dpkg --configure -a failed ({r.stderr.strip()[-300:]}). If ' + 'another apt-get or dpkg still runs (apt-daily, on a new distro), wait for it to end; then run ' + '`dpkg --configure -a` as root, and setup again') + apt = ['/usr/bin/apt-get', '-o', 'Dpkg::Options::=--force-confdef', '-o', 'Dpkg::Options::=--force-confold', + '-o', 'DPkg::Lock::Timeout=300'] + for argv in (apt + ['update'], apt + ['install', '-y', *missing]): + r = k.run(argv, timeout=900, env=quiet) if r.returncode: - return (f'cannot install {" ".join(missing)}: {" ".join(argv)} failed ({r.stderr.strip()[-300:]}). Fix ' - 'the distro\'s network or package sources, run `apt-get update && apt-get install -y ' - f'{" ".join(missing)}` as root, then setup again') + return (f'cannot install {" ".join(missing)}: apt-get {argv[len(apt)]} failed ({r.stderr.strip()[-300:]}).' + ' Fix the distro\'s network (the machine file\'s nameservers are in /etc/resolv.conf now) or ' + f'package sources, run `apt-get update && apt-get install -y {" ".join(missing)}` as root, then ' + 'setup again') return None @@ -235,8 +244,6 @@ def setup(source, sha): r = k.run(['/usr/sbin/usermod', '--shell', k.NOLOGIN, k.USER]) if r.returncode: return say(f'usermod failed: {r.stderr.strip()[-300:]}') or 1 - if why := install_packages(): - return say(why) or 1 version = f'{k.OPT}/{sha}' for src, (dst, mode) in k.CODE.items(): with open(os.path.join(source, src), 'rb') as f: @@ -254,6 +261,10 @@ def setup(source, sha): put(path, text, mode) done[path] = {k.sha(text)} keep(done) + # The packages after the files (apt-get resolves through the kit's own resolv.conf, the machine file's nameservers + # as they are now) and before the firewall loads (nftables is one of them) + if why := install_packages(): + return say(why) or 1 why = k.load_firewall() # now, not after the install: a tightened firewall must not wait for a download if why: return say(why) or 1 diff --git a/fork-kit/pc-agents/tests/fake_distro.py b/fork-kit/pc-agents/tests/fake_distro.py index 2ab5fbc4f1cc2..442d5ab8d19df 100644 --- a/fork-kit/pc-agents/tests/fake_distro.py +++ b/fork-kit/pc-agents/tests/fake_distro.py @@ -79,10 +79,20 @@ ' [ -e "$S/$u.status" ] && { echo "ExecMainCode=$(cat "$S/$u.code")"; echo "ExecMainStatus=$(cat "$S/$u.status")"; }\n' ' echo "ExecMainStartTimestampMonotonic=$(cat "$S/$u.start" 2>/dev/null || echo 0)";;\n' 'esac\n', - # apt-get: update, or install packages (each then installed in /log/packages; nftables brings the fake nft kept - # in /log/nft-package); /log/apt-fail makes it fail - 'usr/bin/apt-get': 'R="${0%/usr/bin/apt-get}"; echo "$*" >>"$R/log/apt-get"\n' + # apt-get [-o OPTION]... update|install -y PKG...: logs the rest; installs each package in /log/packages (nftables + # brings the fake nft kept in /log/nft-package). As real apt-get: refuses while a package is unpacked (dpkg was + # interrupted). /log/apt-fail makes it fail; /log/dns holds the resolv.conf line `update` needs to reach the + # mirror; with /log/questions it fails unless noninteractive and keeping changed config files. + 'usr/bin/apt-get': 'R="${0%/usr/bin/apt-get}"; o=; while [ "$1" = -o ]; do o="$o $2"; shift 2; done\n' + 'echo "$*" >>"$R/log/apt-get"\n' '[ -e "$R/log/apt-fail" ] && { echo "E: no network (test)" >&2; exit 100; }\n' + 'grep -qs unpacked "$R"/log/packages/* && { echo "E: dpkg was interrupted, you must manually ' + 'run \'dpkg --configure -a\' to correct the problem." >&2; exit 100; }\n' + '[ ! -e "$R/log/questions" ] || { [ "$DEBIAN_FRONTEND" = noninteractive ] && case "$o" in ' + '*Dpkg::Options::=--force-confold*) :;; *) false;; esac; } || ' + '{ echo "E: end of file on stdin at conffile prompt (test)" >&2; exit 100; }\n' + '[ "$1" != update ] || [ ! -e "$R/log/dns" ] || grep -qxF "$(cat "$R/log/dns")" "$R/etc/resolv.conf" ' + '|| { echo "E: Temporary failure resolving the mirror (test)" >&2; exit 100; }\n' '[ "$1" = install ] || exit 0; shift 2\n' 'for p; do echo "install ok installed" >"$R/log/packages/$p"\n' ' [ "$p" != nftables ] || cp "$R/log/nft-package" "$R/usr/sbin/nft"; done\n', @@ -90,8 +100,14 @@ 'usr/bin/dpkg-query': 'R="${0%/usr/bin/dpkg-query}"; echo "$*" >>"$R/log/dpkg-query"; shift 2; rc=0\n' 'for p; do if [ -e "$R/log/packages/$p" ]; then printf "%s\\t%s\\n" "$p" "$(cat "$R/log/packages/$p")"\n' ' else echo "dpkg-query: no packages found matching $p" >&2; rc=1; fi; done; exit $rc\n', - # dpkg --configure -a: finishes what an apt-get stopped half-way left unpacked + # dpkg [--force-...]... --configure -a: finishes what an apt-get stopped half-way left unpacked. /log/dpkg-fail + # makes it fail as with apt-daily holding the lock; /log/questions as for apt-get. 'usr/bin/dpkg': 'R="${0%/usr/bin/dpkg}"; echo "$*" >>"$R/log/dpkg"\n' + '[ -e "$R/log/dpkg-fail" ] && { echo "dpkg: error: dpkg frontend lock was locked by another process ' + '(test)" >&2; exit 2; }\n' + '[ ! -e "$R/log/questions" ] || { [ "$DEBIAN_FRONTEND" = noninteractive ] && case "$*" in ' + '*--force-confold*) :;; *) false;; esac; } || ' + '{ echo "dpkg: error: end of file on stdin at conffile prompt (test)" >&2; exit 1; }\n' 'for f in "$R"/log/packages/*; do [ ! -e "$f" ] || sed -i "s/unpacked/installed/" "$f"; done\n', 'usr/sbin/nft': 'R="${0%/usr/sbin/nft}"\ncase "$1" in -f) cp "$R$2" "$R/log/nft-loaded";;\n' 'list) cat "$R/log/nft-loaded" 2>/dev/null || exit 1;; esac\n', diff --git a/fork-kit/pc-agents/tests/setup.test.py b/fork-kit/pc-agents/tests/setup.test.py index 80408cc24c8b1..1614425aa1ea5 100644 --- a/fork-kit/pc-agents/tests/setup.test.py +++ b/fork-kit/pc-agents/tests/setup.test.py @@ -11,6 +11,8 @@ sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) from fake_distro import CONFIG, KIT, REPO, SHA1, SHA2, Case, blob, cli, k # noqa: E402 +DPKG = '--force-confdef --force-confold --configure -a' # how setup runs dpkg: no config-file question + class Setup(Case): def test_fresh_distro_serves_only_after_the_walls_hold(self): @@ -336,10 +338,63 @@ def test_libatomic1_and_unzip_are_installed_once(self): self.write('/log/packages/unzip', 'install ok unpacked') # an apt-get stopped half-way code, out = self.setup_run() self.assertEqual(code, 0, out) - self.assertEqual(self.log('dpkg'), ['--configure -a']) # first: apt-get refuses while dpkg was interrupted + self.assertEqual(self.log('dpkg'), [DPKG]) # first: apt-get refuses while dpkg was interrupted self.assertEqual(self.log('apt-get'), ['update', 'install -y unzip']) self.assertFalse([line for line in self.log() if line.startswith('restart')]) # no T3 restart for a package + def test_a_held_package_counts_as_installed(self): + """`apt-mark hold` makes the status `hold ok installed`: installed, so no apt-get on every run (#296).""" + self.ready() + for log in ('apt-get', 'dpkg'): + os.unlink(self.path('/log/' + log)) + self.write('/log/packages/unzip', 'hold ok installed') + code, out = self.setup_run() + self.assertEqual(code, 0, out) + self.assertEqual(self.log('apt-get'), []) + self.assertEqual(self.log('dpkg'), []) + + def test_packages_are_installed_after_the_kits_own_resolv_conf(self): + """A distro of an older kit (no libatomic1) whose machine file's nameservers broke DNS: the corrected + nameservers reach resolv.conf before apt-get needs them, and a run stopped at apt-get goes on (#296).""" + self.ready() + os.unlink(self.path('/log/packages/libatomic1')) + self.write('/log/dns', 'nameserver 8.8.8.8') # the mirror resolves only through the corrected nameserver + self.machine({**CONFIG, 'nameservers': ['8.8.8.8']}) + self.write('/log/apt-fail', '') # and the network is down for this run + code, out = self.setup_run() + self.assertEqual(code, 1, out) + self.assertIn('nameserver 8.8.8.8\n', self.read('/etc/resolv.conf')) # written before apt-get + os.unlink(self.path('/log/apt-fail')) + code, out = self.setup_run() + self.assertEqual(code, 0, out) + self.assertEqual(self.read('/log/packages/libatomic1'), 'install ok installed\n') + self.assertEqual(self.check()[0], 0) + + def test_dpkg_and_apt_get_ask_nothing(self): + """stdin is /dev/null: a config-file question (an upgrade stopped half-way over an edited file) or a debconf + one takes its default instead of failing at end of file (#296).""" + self.write('/log/questions', '') + self.ready() + self.assertEqual(self.read('/log/packages/unzip'), 'install ok installed\n') + self.write('/log/packages/unzip', 'install ok unpacked') + code, out = self.setup_run() + self.assertEqual(code, 0, out) + self.assertEqual(self.read('/log/packages/unzip'), 'install ok installed\n') + + def test_a_dpkg_that_fails_is_named_with_its_own_fix(self): + """apt-daily holding dpkg's lock on a new distro: the advice is dpkg's, not an apt-get that fails too.""" + self.ready() + self.write('/log/packages/unzip', 'install ok unpacked') + self.write('/log/dpkg-fail', '') + code, out = self.setup_run() + self.assertEqual(code, 1, out) + self.assertIn('dpkg --configure -a failed', out) + self.assertIn('another apt-get or dpkg', out) + self.assertNotIn('apt-get update && apt-get install', out) + os.unlink(self.path('/log/dpkg-fail')) + code, out = self.setup_run() + self.assertEqual(code, 0, out) + def test_t3_that_does_not_start_is_reported(self): self.ready() self.write('/log/t3-crash', '') @@ -723,6 +778,8 @@ def test_bad_config(self): ({'t3Version': 'latest'}, 'an exact version'), ({'t3Version': '0.1.6'}, '0.1.7 or newer'), # no managed policy: no locks at all ({'t3Version': '0.0.30-fork.1'}, '0.1.7 or newer'), + # built before 0.1.7 had the managed policy: a pre-release is older than its release + ({'t3Version': '0.1.7-nightly.20261031.3991'}, '0.1.7 or newer'), ({'maxRuntimeMode': 'root'}, 'one of')): self.machine({**CONFIG, **change}) code, out = self.setup_run() @@ -731,6 +788,13 @@ def test_bad_config(self): self.assertFalse(os.path.lexists(self.path('/etc/t3code/policy.json'))) + def test_versions_from_0_1_7_on_are_accepted(self): + """Compared as numbers, the release itself included: a string compare would refuse 0.1.10 (#296).""" + for v in ('0.1.7', '0.1.8-rc.1', '0.1.10', '0.1.10-nightly.20261114.4682.fork.1', '0.2.0', '1.0.0'): + self.machine({**CONFIG, 't3Version': v}) + self.assertEqual(k.config()['t3Version'], v) + + class Readme(unittest.TestCase): def test_owner_steps_run_as_written_in_powershell(self): """Every kit command of the owner steps is a whole PowerShell line: `pc-agents ...` alone is not found there.""" @@ -740,11 +804,13 @@ def test_owner_steps_run_as_written_in_powershell(self): spans = re.findall(r'`([^`]*pc-agents (?:pair|as-agent)\b[^`]*)`', steps) self.assertGreaterEqual(len(spans), 5, spans) for span in spans: - self.assertTrue(span.startswith('wsl -d -u root -- /opt/pc-agents/current/sbin/pc-agents '), span) + self.assertRegex(span, r'^wsl -d -u root (--|-e) /opt/pc-agents/current/sbin/pc-agents ') self.assertIn('pc-agents pair --ttl 15m`', steps) for key in ('user.name', 'user.email'): # the agent's git identity: placeholders the owner fills in line = next(s for s in spans if f'git config --global {key} ' in s) self.assertRegex(line, rf"git config --global {key} '<[^>]+>'$") + # -e: wsl runs it without root's shell, so a name with an apostrophe (O'Brien) reaches git as typed + self.assertTrue(line.startswith('wsl -d -u root -e '), line) class ReleaseHolds(unittest.TestCase):