diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 752d410d6..c9b3985f4 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -121,6 +121,23 @@ jobs: - name: scripts/runner-setup.ps1 (private-repo check, dry run under pwsh) run: sh scripts/test-runner-setup-windows.sh + windows-doctor: + name: Windows worker doctor (PowerShell 5.1) + needs: changes + if: needs.changes.outputs.code == 'true' + runs-on: windows-latest + steps: + - uses: actions/checkout@v4 + - name: Parse and check Windows PowerShell compatibility; setup.ps1, runner-setup.ps1 and runner-clean.ps1 dry runs + shell: powershell + run: | + if ($PSVersionTable.PSVersion.Major -ne 5) { throw 'Expected Windows PowerShell 5.1' } + & ./scripts/test-runner-setup.ps1 + if ($LASTEXITCODE -ne 0) { throw 'Runner script checks failed' } + - name: Doctor fixtures and hosted VM read-only probes + shell: powershell + run: ./scripts/test-windows-doctor.ps1 + test: name: Tests (${{ matrix.name }}) needs: changes @@ -673,7 +690,7 @@ jobs: ci-ok: name: ci-ok if: always() - needs: [changes, format, test, examples, integration, verdict, windows-fizzy-backend, sdl] + needs: [changes, format, test, examples, integration, verdict, windows-fizzy-backend, sdl, windows-doctor] runs-on: ubuntu-latest steps: - name: Every job passed or was skipped diff --git a/ci/worker-box/windows/config.psd1 b/ci/worker-box/windows/config.psd1 new file mode 100644 index 000000000..09fb310aa --- /dev/null +++ b/ci/worker-box/windows/config.psd1 @@ -0,0 +1,37 @@ +# Desired worker policy; reading this file never converges the machine. +@{ + # The agents' home, on a drive of its own: runner, workspaces, zig-cache, tools, bin and worker + # state all live under it. The doctor derives RunnerRoot and ZigCache from it. + Root = 'C:\fizzy' + RunnerRepo = 'fizzyedit/work' + RunnerName = 'FOXNNE-DESKTOP' + BuildSlots = 2 + Services = @( + @{ Name = 'Tailscale'; Account = 'LocalSystem' } + @{ Name = 'sshd'; Account = 'LocalSystem' } + # Manual: the FizzyWorker* tasks start and stop it, so Stopped is the worker being Off, not a fault. + @{ Name = 'actions.runner.fizzyedit-work.*'; Account = '.\fizzy-worker'; StartMode = 'Manual'; State = 'Any' } + ) + Tasks = @('FizzyRunnerClean', 'FizzyHeartbeat', 'FizzyUpdate', 'FizzyWorkerOn', 'FizzyWorkerOff', 'FizzyWorkerForceOff') + # SYSTEM tasks in a folder fizzy-worker cannot write. Maintainer may trigger all three over SSH + # (`schtasks /run`); the worker account may trigger Off only. + WorkerTaskPath = '\Fizzy\' + Maintainer = 'foxnn' + Worker = 'fizzy-worker' + # Runner and its children run at BelowNormal on at most this many logical cores (the highest + # numbered ones); a machine with fewer cores than the cap is not restricted. + CoreCap = 8 + AllowedUsers = @('foxnn', 'fizzy-worker') + MinDiskFreeGiB = 100 + MinAvailableRamGiB = 4 + MaxPatchDays = 30 + MaxCacheGiB = 40 + # Empty pins report AMBER until the maintainer chooses a version in a policy PR. + Tools = @( + @{ Name = 'zig'; Args = @('version'); Version = '0.17.0' } + @{ Name = 'git'; Args = @('--version'); Version = '' } + @{ Name = 'jj'; Args = @('--version'); Version = '' } + @{ Name = 'node'; Args = @('--version'); Version = '' } + @{ Name = 'gh'; Args = @('--version'); Version = '' } + ) +} diff --git a/ci/worker-box/windows/doctor.ps1 b/ci/worker-box/windows/doctor.ps1 new file mode 100644 index 000000000..da0915b8e --- /dev/null +++ b/ci/worker-box/windows/doctor.ps1 @@ -0,0 +1,340 @@ +# Read-only Windows worker diagnostics. Unknown or inaccessible evidence is AMBER. +# -DryRun lists checks without probing; normal execution never changes system configuration. +# Exit status is the number of RED results. JSON is an array of status/item/detail objects. +#requires -Version 5.1 +[CmdletBinding()] +param([switch]$Json, [switch]$DryRun, [string]$ConfigPath) +Set-StrictMode -Version 2 +# $PSScriptRoot is empty inside a param() default in Windows PowerShell 5.1, so resolve it here. +if (-not $ConfigPath) { $ConfigPath = Join-Path (Split-Path -Parent $MyInvocation.MyCommand.Path) 'config.psd1' } +$ErrorActionPreference = 'Stop' +$config = Import-PowerShellDataFile -LiteralPath $ConfigPath +# RunnerRoot and ZigCache come from Root unless a config names them (the test fixtures do). +$runnerRoot = ''; $zigCache = '' +if ($config.ContainsKey('Root')) { $runnerRoot = $config.Root.TrimEnd('\') + '\actions-runner'; $zigCache = $config.Root.TrimEnd('\') + '\zig-cache' } +if ($config.ContainsKey('RunnerRoot')) { $runnerRoot = $config.RunnerRoot } +if ($config.ContainsKey('ZigCache')) { $zigCache = $config.ZigCache } +$results = New-Object 'Collections.Generic.List[object]' +function Result($status, $item, $detail) { + $results.Add([pscustomobject]@{ status = $status; item = $item; detail = ("$detail" -replace '[\r\n]+', ' ') }) +} +function Check($item, [scriptblock]$probe) { + if ($DryRun) { Result AMBER $item 'probe skipped (DryRun)'; return } + try { & $probe } catch { Result AMBER $item 'evidence unavailable (query failed or access denied)' } +} +function Native([string]$exe, [string[]]$arguments) { + # Native stderr is a terminating error under Stop in Windows PowerShell 5.1. + $old = $ErrorActionPreference + $ErrorActionPreference = 'Continue' + try { $output = & $exe @arguments 2>$null; $code = $LASTEXITCODE } + finally { $ErrorActionPreference = $old } + if ($code -ne 0) { throw 'native query failed' } + return ($output -join ' ').Trim() +} +foreach ($spec in $config.Services) { + $item = "service $($spec.Name)" + Check $item { + $services = @(Get-CimInstance -ClassName Win32_Service | Where-Object { $_.Name -like $spec.Name }) + if ($services.Count -ne 1) { Result RED $item "expected one service; found $($services.Count)"; return } + $s = $services[0] + $account = $s.StartName -replace "^$([regex]::Escape([Environment]::MachineName))\\", '.\' + $status = 'RED' + $wantMode = 'Auto'; if ($spec.ContainsKey('StartMode')) { $wantMode = $spec.StartMode } + # State = 'Any' is the worker switch's own service: Running (On) and Stopped (Off) are both healthy. + $stateOk = ($s.State -eq 'Running'); if ($spec.ContainsKey('State') -and $spec.State -eq 'Any') { $stateOk = ($s.State -eq 'Running' -or $s.State -eq 'Stopped') } + if ($stateOk -and $s.StartMode -eq $wantMode -and $account -ieq $spec.Account) { $status = 'GREEN' } + $delayed = 'unknown' + if ($s.PSObject.Properties['DelayedAutoStart']) { $delayed = "$($s.DelayedAutoStart)" } + Result $status $item "state=$($s.State); start=$($s.StartMode); delayed=$delayed; account=$($s.StartName)" + } +} +Check 'runner Online' { + if (-not ($env:GH_TOKEN -or $env:GITHUB_TOKEN)) { Result AMBER 'runner Online' 'no token environment; API not queried'; return } + $gh = (Get-Command gh.exe -CommandType Application).Source + $raw = Native $gh @('api', '--paginate', '--slurp', "repos/$($config.RunnerRepo)/actions/runners?per_page=100") + $pages = $raw | ConvertFrom-Json + $runners = @($pages | ForEach-Object { $_.runners } | Where-Object { $_.name -eq $config.RunnerName }) + $status = 'RED' + if ($runners.Count -eq 1 -and $runners[0].status -eq 'online') { $status = 'GREEN' } + if ($status -eq 'RED') { + # An Off worker is offline on purpose. + $svc = @(Get-CimInstance -ClassName Win32_Service | Where-Object { $_.Name -like 'actions.runner.*' }) + if ($svc.Count -eq 1 -and $svc[0].State -eq 'Stopped') { Result GREEN 'runner Online' 'worker is Off (service stopped), so offline is expected'; return } + } + Result $status 'runner Online' "repo=$($config.RunnerRepo); name=$($config.RunnerName); matching online=$(@($runners | Where-Object { $_.status -eq 'online' }).Count)" +} +Check 'power plan' { + Result GREEN 'power plan' (Native "$env:SystemRoot\System32\powercfg.exe" @('/getactivescheme')) +} +# GUIDs and hex indices avoid localized powercfg setting names. +foreach ($setting in @( + @{ Name = 'sleep'; Guid = '29f6c1db-86da-48c5-9fdb-f2b67b1f44da' }, + @{ Name = 'hibernate'; Guid = '9d7815a6-7ee4-497e-8888-515a05f02364' } +)) { + $item = "power $($setting.Name)" + Check $item { + $raw = Native "$env:SystemRoot\System32\powercfg.exe" @('/query', 'SCHEME_CURRENT', '238c9fa8-0aad-41ed-83f4-97be242c8f20', $setting.Guid) + $indices = [regex]::Matches($raw, '0x[0-9a-fA-F]{8}') + if ($indices.Count -lt 2) { throw 'cannot read AC/DC indices' } + $indices = @($indices)[-2..-1] + $status = 'RED' + if ($indices[0].Value -eq '0x00000000' -and $indices[1].Value -eq '0x00000000') { $status = 'GREEN' } + Result $status $item "AC=$($indices[0].Value); DC=$($indices[1].Value); expected never (zero)" + } +} +Check 'hibernation enabled' { + $v = Get-ItemPropertyValue -LiteralPath 'HKLM:\SYSTEM\CurrentControlSet\Control\Power' -Name HibernateEnabled + $status = 'RED'; if ($v -eq 0) { $status = 'GREEN' } + Result $status 'hibernation enabled' "HibernateEnabled=$v" +} +Check 'Fast Startup' { + $v = Get-ItemPropertyValue -LiteralPath 'HKLM:\SYSTEM\CurrentControlSet\Control\Session Manager\Power' -Name HiberbootEnabled + $status = 'RED'; if ($v -eq 0) { $status = 'GREEN' } + Result $status 'Fast Startup' "HiberbootEnabled=$v" +} +Check 'last boot' { + $os = Get-CimInstance -ClassName Win32_OperatingSystem + Result GREEN 'last boot' ($os.LastBootUpTime.ToString('o') + '; BIOS power recovery is not inspected') +} +Check 'unclean shutdowns' { + try { + $events = @(Get-WinEvent -FilterHashtable @{ + LogName = 'System'; Id = @(41, 6008); StartTime = (Get-Date).AddDays(-30) + }) + } catch { + # An empty event query is not an access failure. + if ($_.FullyQualifiedErrorId -notlike 'NoMatchingEventsFound*') { throw } + $events = @() + } + $bad = @($events | Where-Object { ($_.Id -eq 41 -and $_.ProviderName -eq 'Microsoft-Windows-Kernel-Power') -or ($_.Id -eq 6008 -and $_.ProviderName -eq 'EventLog') }) + $status = 'GREEN'; if ($bad.Count) { $status = 'AMBER' } + Result $status 'unclean shutdowns' "30-day event records=$($bad.Count) (41/6008 may describe the same shutdown)" +} +Check 'pending reboot' { + $pending = @() + foreach ($key in @('SOFTWARE\Microsoft\Windows\CurrentVersion\Component Based Servicing\RebootPending', 'SOFTWARE\Microsoft\Windows\CurrentVersion\WindowsUpdate\Auto Update\RebootRequired')) { + if (Test-Path -LiteralPath "HKLM:\$key") { $pending += $key } + } + $session = Get-ItemProperty -LiteralPath 'HKLM:\SYSTEM\CurrentControlSet\Control\Session Manager' + if ($session.PSObject.Properties['PendingFileRenameOperations'] -and $session.PendingFileRenameOperations) { $pending += 'PendingFileRenameOperations' } + $status = 'GREEN'; if ($pending.Count) { $status = 'AMBER' } + Result $status 'pending reboot' "indicators=$($pending -join ',')" +} +Check 'last patch' { + $patch = Get-HotFix | Where-Object { $_.InstalledOn } | Sort-Object InstalledOn -Descending | Select-Object -First 1 + if (-not $patch) { throw 'no patch date' } + $days = [int]((Get-Date) - $patch.InstalledOn).TotalDays + $status = 'GREEN'; if ($days -gt $config.MaxPatchDays -or $days -lt 0) { $status = 'AMBER' } + Result $status 'last patch' "$($patch.HotFixID); days=$days; limit=$($config.MaxPatchDays) (Get-HotFix coverage)" +} +Check 'disk inventory' { + $disks = @(Get-CimInstance -ClassName Win32_LogicalDisk -Filter 'DriveType=3') + if (-not $disks.Count) { throw 'no fixed disks' } + foreach ($d in $disks) { + $free = [math]::Round($d.FreeSpace / 1GB, 1) + $status = 'GREEN'; if ($free -lt $config.MinDiskFreeGiB) { $status = 'RED' } + # With a Root, only its drive is the agents' home: low space there is RED (AMBER within 50% of + # the minimum); another drive is the person's own. + if ($config.ContainsKey('Root')) { + if ($d.DeviceID -ieq [IO.Path]::GetPathRoot($config.Root).TrimEnd('\')) { + if ($free -ge $config.MinDiskFreeGiB) { $status = 'GREEN' } else { $status = 'RED' } + if ($status -eq 'GREEN' -and $free -lt $config.MinDiskFreeGiB * 1.5) { $status = 'AMBER' } + } elseif ($free -lt $config.MinDiskFreeGiB) { $status = 'AMBER' } + } + Result $status "disk $($d.DeviceID)" "free=$free GiB; minimum=$($config.MinDiskFreeGiB) GiB" + } +} +Check 'RAM' { + $os = Get-CimInstance -ClassName Win32_OperatingSystem + $free = [math]::Round($os.FreePhysicalMemory / 1MB, 1) + $total = [math]::Round($os.TotalVisibleMemorySize / 1MB, 1) + $status = 'GREEN'; if ($free -lt $config.MinAvailableRamGiB) { $status = 'AMBER' } + Result $status 'RAM' "available=$free GiB; total=$total GiB; slots=$($config.BuildSlots)" +} +Check 'zig-cache' { + if (-not (Test-Path -LiteralPath $zigCache -PathType Container)) { Result RED 'zig-cache' 'configured directory missing'; return } + # Never follow junctions, even when they occur inside a cache. + $stack = New-Object 'Collections.Generic.Stack[string]' + $stack.Push($zigCache) + $bytes = [long]0 + while ($stack.Count) { + $path = $stack.Pop() + $entry = Get-Item -LiteralPath $path -Force + if ($entry.Attributes -band [IO.FileAttributes]::ReparsePoint) { throw 'reparse point prevents complete sizing' } + foreach ($child in Get-ChildItem -LiteralPath $path -Force) { + if ($child.Attributes -band [IO.FileAttributes]::ReparsePoint) { throw 'reparse point prevents complete sizing' } + if ($child.PSIsContainer) { $stack.Push($child.FullName) } else { $bytes += $child.Length } + } + } + $size = [math]::Round($bytes / 1GB, 2) + $status = 'GREEN'; if ($size -gt $config.MaxCacheGiB) { $status = 'AMBER' } + Result $status 'zig-cache' "$zigCache; size=$size GiB; limit=$($config.MaxCacheGiB) GiB" +} +$runnerPath = $null +Check 'runner PATH' { + $lines = @(Get-Content -LiteralPath (Join-Path $runnerRoot '.env') | Where-Object { $_ -match '^PATH=' }) + if ($lines.Count -ne 1) { throw 'runner PATH missing or ambiguous' } + $script:runnerPath = $lines[0].Substring(5) + if (-not $runnerPath -or $runnerPath -match '%|\$') { $script:runnerPath = $null; throw 'unexpanded PATH' } + Result GREEN 'runner PATH' 'using saved runner .env PATH; account access and job overrides require a runner job to verify' +} +foreach ($tool in $config.Tools) { + $item = "tool $($tool.Name)" + Check $item { + if (-not $runnerPath) { Result AMBER $item 'runner PATH unavailable'; return } + $exe = $null + foreach ($dir in $runnerPath.Split(';')) { + if (-not [IO.Path]::IsPathRooted($dir.Trim('"'))) { throw 'relative runner PATH entry' } + $candidate = Join-Path $dir.Trim('"') ($tool.Name + '.exe') + if (Test-Path -LiteralPath $candidate -PathType Leaf) { $exe = $candidate; break } + } + if (-not $exe) { Result RED $item 'not found on saved runner PATH'; return } + $version = Native $exe $tool.Args + $status = 'AMBER' + if ($tool.Version) { + $status = 'RED' + if ($version -match ('(^|\s)v?' + [regex]::Escape($tool.Version) + '(?=\s|$)')) { $status = 'GREEN' } + } + Result $status $item "path=$exe; version=$version; pin=$($tool.Version); executed as diagnostic caller" + } +} +foreach ($task in $config.Tasks) { + $item = "task $task" + Check $item { + $found = @(Get-ScheduledTask | Where-Object { $_.TaskName -eq $task }) + $status = 'RED'; if ($found.Count) { $status = 'GREEN' } + Result $status $item "present=$($found.Count -gt 0)" + } +} +Check 'local users' { + $strays = @(Get-LocalUser | Where-Object { $_.SID.Value -notmatch '-(500|501|503|504)$' -and $config.AllowedUsers -notcontains $_.Name }) + $status = 'GREEN'; if ($strays.Count) { $status = 'AMBER' } + Result $status 'local users' "unexpected=$(@($strays | ForEach-Object { $_.Name }) -join ','); includes disabled accounts" +} +# --- the worker switch and the agents' home (policy in config.psd1; absent from minimal fixtures) --- +function ConvertTo-Sid($name) { + return (New-Object Security.Principal.NTAccount($name)).Translate([Security.Principal.SecurityIdentifier]) +} +$readRights = [Security.AccessControl.FileSystemRights]::ListDirectory +# Whether the worker could read $path, from its ACL: a deny for the worker (or a broad group) first, +# then any allow for the worker, Users, Authenticated Users or Everyone. Effective access also depends +# on group policy, so this is the file system's half of the answer. +function Test-WorkerRead($path, $workerSid) { + $broad = @($workerSid.Value, 'S-1-5-32-545', 'S-1-5-11', 'S-1-1-0') + foreach ($rule in (Get-Acl -LiteralPath $path).Access) { + try { $sid = $rule.IdentityReference.Translate([Security.Principal.SecurityIdentifier]).Value } catch { continue } + if ($broad -notcontains $sid) { continue } + if (-not ($rule.FileSystemRights -band $readRights)) { continue } + if ($rule.AccessControlType -eq 'Deny') { return $false } + if ($rule.AccessControlType -eq 'Allow') { return $true } + } + return $false +} +if ($config.ContainsKey('Worker')) { + Check 'worker switch' { + $svc = @(Get-CimInstance -ClassName Win32_Service | Where-Object { $_.Name -like 'actions.runner.*' }) + if ($svc.Count -ne 1) { throw 'runner service not found' } + $jobs = @(Get-Process -Name Runner.Worker -ErrorAction SilentlyContinue).Count + $state = 'ON'; if ($svc[0].State -eq 'Stopped') { $state = 'OFF' } elseif ($svc[0].State -ne 'Running') { $state = $svc[0].State } + # On and Off are both policy: report which, GREEN either way. + $status = 'GREEN'; if ($state -notin 'ON', 'OFF') { $status = 'AMBER' } + Result $status 'worker switch' "worker=$state; service=$($svc[0].State); running jobs=$jobs" + } + $workerSid = $null + Check 'worker account' { + $script:workerSid = ConvertTo-Sid $config.Worker + Result GREEN 'worker account' "$($config.Worker) = $($script:workerSid.Value)" + } + # Tasks run as SYSTEM, in a folder the worker cannot write; the maintainer may run all three, + # the worker the drain only. Write-like rights: DELETE, WRITE_DAC, WRITE_OWNER, FILE_WRITE_DATA, GW, GA. + $writeMask = 0x10000 -bor 0x40000 -bor 0x80000 -bor 0x2 -bor 0x40000000 -bor 0x10000000 + foreach ($t in @( + @{ Name = 'FizzyWorkerOn'; Maintainer = $true; Worker = $false }, + @{ Name = 'FizzyWorkerOff'; Maintainer = $true; Worker = $true }, + @{ Name = 'FizzyWorkerForceOff'; Maintainer = $true; Worker = $false } + )) { + $item = "task access $($t.Name)" + Check $item { + if (-not $script:workerSid) { Result AMBER $item 'worker account unknown'; return } + $maintainerSid = ConvertTo-Sid $config.Maintainer + $com = New-Object -ComObject 'Schedule.Service' + $com.Connect() + $folder = $com.GetFolder($config.WorkerTaskPath.TrimEnd('\')) + $sd = New-Object Security.AccessControl.RawSecurityDescriptor ($folder.GetTask($t.Name).GetSecurityDescriptor(4)) + $canRun = @{}; $canWrite = $false; $runsAsSystem = $true + foreach ($ace in $sd.DiscretionaryAcl) { + if ($ace.AceType -ne 'AccessAllowed') { continue } + $sid = $ace.SecurityIdentifier.Value + if ($sid -in $script:workerSid.Value, 'S-1-5-32-545', 'S-1-5-11', 'S-1-1-0') { + $canRun[$sid] = $true + if ($ace.AccessMask -band $writeMask) { $canWrite = $true } + } + if ($sid -eq $maintainerSid.Value) { $canRun['maintainer'] = $true } + } + $workerRuns = [bool]$canRun[$script:workerSid.Value] -or [bool]$canRun['S-1-5-32-545'] -or [bool]$canRun['S-1-5-11'] -or [bool]$canRun['S-1-1-0'] + $info = $folder.GetTask($t.Name).Definition.Principal + if ($info.UserId -notin 'SYSTEM', 'S-1-5-18', 'NT AUTHORITY\SYSTEM') { $runsAsSystem = $false } + $ok = $runsAsSystem -and -not $canWrite -and ($workerRuns -eq $t.Worker) -and ([bool]$canRun['maintainer'] -eq $t.Maintainer) + $status = 'RED'; if ($ok) { $status = 'GREEN' } + Result $status $item "runs as SYSTEM=$runsAsSystem; worker can trigger=$workerRuns (want $($t.Worker)); worker can write=$canWrite; maintainer can trigger=$([bool]$canRun['maintainer']) (want $($t.Maintainer))" + } + } + Check 'task folder not writable by worker' { + if (-not $script:workerSid) { Result AMBER 'task folder not writable by worker' 'worker account unknown'; return } + $com = New-Object -ComObject 'Schedule.Service' + $com.Connect() + $sd = New-Object Security.AccessControl.RawSecurityDescriptor ($com.GetFolder($config.WorkerTaskPath.TrimEnd('\')).GetSecurityDescriptor(4)) + $bad = @($sd.DiscretionaryAcl | Where-Object { + $_.AceType -eq 'AccessAllowed' -and ($_.AccessMask -band $writeMask) -and + $_.SecurityIdentifier.Value -in $script:workerSid.Value, 'S-1-5-32-545', 'S-1-5-11', 'S-1-1-0' + }) + $status = 'GREEN'; if ($bad.Count) { $status = 'RED' } + Result $status 'task folder not writable by worker' "folder=$($config.WorkerTaskPath); writable ACEs=$($bad.Count)" + } + foreach ($exe in 'Runner.Listener.exe', 'Runner.Worker.exe') { + $item = "priority $exe" + Check $item { + $v = Get-ItemPropertyValue -LiteralPath "HKLM:\SOFTWARE\Microsoft\Windows NT\CurrentVersion\Image File Execution Options\$exe\PerfOptions" -Name CpuPriorityClass + $status = 'RED'; if ($v -eq 5) { $status = 'GREEN' } + Result $status $item "CpuPriorityClass=$v; expected 5 (BelowNormal); children inherit it; core cap=$($config.CoreCap) applied by the switch" + } + } + Check 'heartbeat' { + $file = ($config.Root.TrimEnd('\') + '\state\heartbeat.txt') + if (-not (Test-Path -LiteralPath $file)) { Result AMBER 'heartbeat' 'no heartbeat recorded yet'; return } + $age = [int]((Get-Date) - (Get-Item -LiteralPath $file).LastWriteTime).TotalMinutes + $status = 'GREEN'; if ($age -gt 30) { $status = 'AMBER' } + Result $status 'heartbeat' "last written $age minutes ago" + } + Check 'root access' { + if (-not $script:workerSid) { Result AMBER 'root access' 'worker account unknown'; return } + # The root is closed to everyone but SYSTEM and Administrators, and to the worker's read but + # for the traversal grant; no broad group may appear on it. + $broad = @((Get-Acl -LiteralPath $config.Root).Access | Where-Object { + try { $_.IdentityReference.Translate([Security.Principal.SecurityIdentifier]).Value -in 'S-1-5-32-545', 'S-1-5-11', 'S-1-1-0' } catch { $false } + }) + $status = 'GREEN'; if ($broad.Count) { $status = 'RED' } + Result $status 'root access' "root=$($config.Root); entries for Users/Authenticated Users/Everyone=$($broad.Count)" + } + Check 'worker isolation' { + if (-not $script:workerSid) { Result AMBER 'worker isolation' 'worker account unknown'; return } + $homeDrive = [IO.Path]::GetPathRoot($config.Root).TrimEnd('\') + $reachable = @() + foreach ($d in @(Get-CimInstance -ClassName Win32_LogicalDisk -Filter 'DriveType=3')) { + if ($d.DeviceID -ieq $homeDrive -or $d.DeviceID -ieq $env:SystemDrive) { continue } + if (Test-WorkerRead ($d.DeviceID + '\') $script:workerSid) { $reachable += "drive $($d.DeviceID)" } + } + $users = Join-Path $env:SystemDrive 'Users' + foreach ($p in @(Get-ChildItem -LiteralPath $users -Directory -Force -ErrorAction Stop)) { + if ($p.Name -in 'Public', 'Default', 'Default User', 'All Users', $config.Worker) { continue } + if ($p.Attributes -band [IO.FileAttributes]::ReparsePoint) { continue } + if (Test-WorkerRead $p.FullName $script:workerSid) { $reachable += 'a user profile' } + } + $status = 'GREEN'; if ($reachable.Count) { $status = 'RED' } + # Profiles are not named: this output may end up in a public place. + Result $status 'worker isolation' "other fixed drives and user profiles readable by the worker=$($reachable.Count): $(($reachable | Sort-Object -Unique) -join ', ')" + } +} +if ($Json) { ConvertTo-Json -InputObject @($results.ToArray()) -Depth 4 } +else { foreach ($r in $results) { Write-Output "$($r.status) $($r.item): $($r.detail)" } } +exit @($results | Where-Object { $_.status -eq 'RED' }).Count diff --git a/ci/worker-box/windows/setup.ps1 b/ci/worker-box/windows/setup.ps1 new file mode 100644 index 000000000..3f79e3e01 --- /dev/null +++ b/ci/worker-box/windows/setup.ps1 @@ -0,0 +1,164 @@ +# Converge the Windows worker box to config.psd1. Run from an elevated PowerShell (5.1 or newer): +# powershell -ExecutionPolicy Bypass -File .\ci\worker-box\windows\setup.ps1 [-DryRun] +# Idempotent: every item is checked, fixed if wrong, and rechecked by doctor.ps1 at the end. +# -DryRun prints each step and changes nothing (no elevation, token or downloads needed). +# The runner itself (account, folders, tools, service) is scripts/runner-setup.ps1, run first with the +# root from config.psd1; this adds the on/off switch and the machine settings around it. The worker +# is ON after setup and after every reboot; the maintainer turns it Off by hand (docs/AGENT_WORK.md). +# Needs RUNNER_TOKEN in the environment on the first run, as runner-setup.ps1 does. +[CmdletBinding()] +param([switch]$DryRun) +Set-StrictMode -Version 2 +$ErrorActionPreference = 'Stop' +$here = Split-Path -Parent $MyInvocation.MyCommand.Path +$config = Import-PowerShellDataFile -LiteralPath (Join-Path $here 'config.psd1') +$repoScripts = Join-Path $here '..\..\..\scripts' +$root = $config.Root.TrimEnd('\') +$bin = "$root\bin" +$worker = $config.Worker +$taskPath = $config.WorkerTaskPath +$engine = (Get-Process -Id $PID).Path + +function Fail($msg) { [Console]::Error.WriteLine("ERROR: $msg"); exit 1 } +function Step($text, [scriptblock]$action) { + Write-Host "+ $text" + if (-not $DryRun) { & $action } +} +# Native stderr is a terminating error under Stop in Windows PowerShell 5.1: run under Continue. +function Native([scriptblock]$cmd) { + $old = $ErrorActionPreference + $ErrorActionPreference = 'Continue' + try { $out = & $cmd; $script:nativeExit = $LASTEXITCODE } finally { $ErrorActionPreference = $old } + return $out +} +$nativeExit = 0 + +if (-not $DryRun) { + if ($env:OS -ne 'Windows_NT') { Fail 'Run on Windows.' } + $me = [Security.Principal.WindowsPrincipal][Security.Principal.WindowsIdentity]::GetCurrent() + if (-not $me.IsInRole([Security.Principal.WindowsBuiltInRole]::Administrator)) { Fail 'Run from an elevated (Administrator) PowerShell.' } +} + +# --- the runner: account, folders under the root, tools, service, FizzyRunnerClean ------------------ +Write-Host "+ scripts/runner-setup.ps1 with FIZZY_WORKER_ROOT=$root" +$env:FIZZY_WORKER_ROOT = $root +$setupArgs = @('-NoProfile', '-ExecutionPolicy', 'Bypass', '-File', (Join-Path $repoScripts 'runner-setup.ps1')) +if ($DryRun) { $setupArgs += '-DryRun' } +& $engine @setupArgs +if ($LASTEXITCODE -ne 0) { Fail "runner-setup.ps1 failed ($LASTEXITCODE)" } + +# --- the agents' home ------------------------------------------------------------------------------ +Step "worker state folder $root\state (the worker may read it, not write it)" { + New-Item -ItemType Directory -Force -Path "$root\state" | Out-Null + $null = Native { & icacls "$root\state" /grant "${worker}:(OI)(CI)RX" /C /Q 2>&1 } + if ($nativeExit -ne 0) { Fail "icacls on $root\state failed ($nativeExit)" } +} +Step "$worker may not read any other fixed drive (a deny on each drive root; the OS drive's profiles are private by default and the doctor checks them)" { + $homeDrive = [IO.Path]::GetPathRoot($root).TrimEnd('\') + foreach ($d in @(Get-CimInstance -ClassName Win32_LogicalDisk -Filter 'DriveType=3')) { + if ($d.DeviceID -ieq $homeDrive -or $d.DeviceID -ieq $env:SystemDrive) { continue } + $null = Native { & icacls "$($d.DeviceID)\" /deny "${worker}:(OI)(CI)(F)" /C /Q 2>&1 } + if ($nativeExit -ne 0) { Write-Warning "could not deny $worker on $($d.DeviceID) (icacls $nativeExit); the doctor will report it" } + } +} +Step "disk free space on the root's drive (minimum $($config.MinDiskFreeGiB) GiB)" { + $drive = [IO.Path]::GetPathRoot($root).TrimEnd('\') + $disk = Get-CimInstance -ClassName Win32_LogicalDisk -Filter "DeviceID='$drive'" + $free = [math]::Round($disk.FreeSpace / 1GB, 1) + if ($free -lt $config.MinDiskFreeGiB) { Write-Warning "only $free GiB free on the root's drive" } else { Write-Host " $free GiB free" } +} + +# --- the machine stays up and awake --------------------------------------------------------------- +Step 'Fast Startup off (HiberbootEnabled=0), hibernation off, never sleep or hibernate' { + Set-ItemProperty -LiteralPath 'HKLM:\SYSTEM\CurrentControlSet\Control\Session Manager\Power' -Name HiberbootEnabled -Value 0 -Type DWord + $null = Native { & "$env:SystemRoot\System32\powercfg.exe" /hibernate off 2>&1 } + foreach ($t in 'standby-timeout-ac', 'standby-timeout-dc', 'hibernate-timeout-ac', 'hibernate-timeout-dc') { + $null = Native { & "$env:SystemRoot\System32\powercfg.exe" /change $t 0 2>&1 } + if ($nativeExit -ne 0) { Fail "powercfg /change $t failed ($nativeExit)" } + } +} +Step 'sshd and Tailscale start automatically when present (the way in when nobody is logged on)' { + foreach ($name in 'sshd', 'Tailscale') { + if (Get-Service -Name $name -ErrorAction SilentlyContinue) { + Set-Service -Name $name -StartupType Automatic + if ((Get-Service -Name $name).Status -ne 'Running') { Start-Service -Name $name } + } else { Write-Warning "service $name is not installed; install it by hand (setup does not install software)" } + } +} + +# --- the switch ------------------------------------------------------------------------------------ +Step 'runner and its children at BelowNormal priority (Image File Execution Options CpuPriorityClass=5; children inherit it)' { + foreach ($exe in 'Runner.Listener.exe', 'Runner.Worker.exe') { + $key = "HKLM:\SOFTWARE\Microsoft\Windows NT\CurrentVersion\Image File Execution Options\$exe\PerfOptions" + if (-not (Test-Path -LiteralPath $key)) { New-Item -Path $key -Force | Out-Null } + Set-ItemProperty -LiteralPath $key -Name CpuPriorityClass -Value 5 -Type DWord + } +} +Step "install worker-switch.ps1 to $bin (admin-write only)" { + Copy-Item (Join-Path $here 'worker-switch.ps1') "$bin\worker-switch.ps1" -Force +} +Step "register the SYSTEM tasks in $taskPath : FizzyWorkerOn (at boot and on demand), FizzyWorkerOff (drain), FizzyWorkerForceOff, FizzyHeartbeat (5 min), FizzyUpdate (weekly)" { + $principal = New-ScheduledTaskPrincipal -UserId 'SYSTEM' -LogonType ServiceAccount -RunLevel Highest + $settings = New-ScheduledTaskSettingsSet -StartWhenAvailable -AllowStartIfOnBatteries -DontStopIfGoingOnBatteries ` + -ExecutionTimeLimit (New-TimeSpan -Hours 13) -MultipleInstances IgnoreNew + $quick = New-ScheduledTaskSettingsSet -StartWhenAvailable -AllowStartIfOnBatteries -DontStopIfGoingOnBatteries ` + -ExecutionTimeLimit (New-TimeSpan -Minutes 5) -MultipleInstances IgnoreNew + function New-SwitchAction($mode) { + New-ScheduledTaskAction -Execute 'powershell.exe' -Argument ("-NoProfile -ExecutionPolicy Bypass -File `"$bin\worker-switch.ps1`" -Mode $mode -Root `"$root`" -CoreCap $($config.CoreCap)") + } + $specs = @( + @{ Name = 'FizzyWorkerOn'; Mode = 'On'; Trigger = (New-ScheduledTaskTrigger -AtStartup); Settings = $quick } + @{ Name = 'FizzyWorkerOff'; Mode = 'Off'; Trigger = $null; Settings = $settings } + @{ Name = 'FizzyWorkerForceOff'; Mode = 'ForceOff'; Trigger = $null; Settings = $quick } + @{ Name = 'FizzyHeartbeat'; Mode = 'Apply'; Trigger = (New-ScheduledTaskTrigger -Once -At (Get-Date) -RepetitionInterval (New-TimeSpan -Minutes 5) -RepetitionDuration (New-TimeSpan -Days 3650)); Settings = $quick } + @{ Name = 'FizzyUpdate'; Mode = 'Update'; Trigger = (New-ScheduledTaskTrigger -Weekly -DaysOfWeek Sunday -At 3am); Settings = $settings } + ) + foreach ($s in $specs) { + $reg = @{ TaskName = $s.Name; TaskPath = $taskPath; Action = (New-SwitchAction $s.Mode); Principal = $principal; Settings = $s.Settings; Force = $true } + if ($s.Trigger) { $reg.Trigger = $s.Trigger } + Register-ScheduledTask @reg | Out-Null + } +} +Step "task security: $($config.Maintainer) may run all three switch tasks, $worker the drain only, neither may change them" { + function Get-Sid($name) { return (New-Object Security.Principal.NTAccount($name)).Translate([Security.Principal.SecurityIdentifier]).Value } + $maintainer = Get-Sid $config.Maintainer + $workerSid = Get-Sid $worker + $system = 'D:P(A;;FA;;;SY)(A;;FA;;;BA)' + $com = New-Object -ComObject 'Schedule.Service' + $com.Connect() + $folder = $com.GetFolder($taskPath.TrimEnd('\')) + # GR|GX lets a caller read and run a task, not change or delete it. + $folder.SetSecurityDescriptor("$system(A;;GRGX;;;$maintainer)(A;;GRGX;;;$workerSid)", 0) + $folder.GetTask('FizzyWorkerOn').SetSecurityDescriptor("$system(A;;GRGX;;;$maintainer)", 0) + $folder.GetTask('FizzyWorkerForceOff').SetSecurityDescriptor("$system(A;;GRGX;;;$maintainer)", 0) + $folder.GetTask('FizzyWorkerOff').SetSecurityDescriptor("$system(A;;GRGX;;;$maintainer)(A;;GRGX;;;$workerSid)", 0) +} +Step 'runner service to Manual, then On: the worker is ON after setup and, through FizzyWorkerOn at boot, after every reboot' { + $name = (Get-Content "$root\actions-runner\.service" -Raw).Trim() + Set-Service -Name $name -StartupType Manual + Start-ScheduledTask -TaskPath $taskPath -TaskName FizzyWorkerOn + $deadline = (Get-Date).AddSeconds(120) + do { Start-Sleep -Seconds 2 } while ((Get-ScheduledTask -TaskPath $taskPath -TaskName FizzyWorkerOn).State -eq 'Running' -and (Get-Date) -lt $deadline) + $result = (Get-ScheduledTaskInfo -TaskPath $taskPath -TaskName FizzyWorkerOn).LastTaskResult + if ($result -ne 0) { Fail ("FizzyWorkerOn last result 0x{0:X}" -f $result) } +} + +# --- report, never delete --------------------------------------------------------------------------- +Step 'local accounts outside the allowed list (reported only; setup never deletes an account)' { + $strays = @(Get-LocalUser | Where-Object { $_.SID.Value -notmatch '-(500|501|503|504)$' -and $config.AllowedUsers -notcontains $_.Name }) + if ($strays.Count) { Write-Warning "unexpected local accounts: $(@($strays | ForEach-Object { $_.Name }) -join ', ') (remove by hand if not wanted)" } + else { Write-Host ' none' } +} + +# --- recheck ----------------------------------------------------------------------------------------- +Write-Host '+ recheck with doctor.ps1' +if ($DryRun) { + & $engine -NoProfile -ExecutionPolicy Bypass -File (Join-Path $here 'doctor.ps1') -DryRun | Select-Object -First 3 + Write-Host 'Dry run complete; no changes made.' + exit 0 +} +$doctor = @(& $engine -NoProfile -ExecutionPolicy Bypass -File (Join-Path $here 'doctor.ps1')) +$doctor | Where-Object { $_ -notmatch '^GREEN ' } | ForEach-Object { Write-Host $_ } +$reds = @($doctor | Where-Object { $_ -match '^RED ' }).Count +if ($reds) { Fail "$reds doctor check(s) still RED after setup (listed above)" } +Write-Host 'Worker box converged; doctor has no RED.' diff --git a/ci/worker-box/windows/worker-switch.ps1 b/ci/worker-box/windows/worker-switch.ps1 new file mode 100644 index 000000000..ed1c9649f --- /dev/null +++ b/ci/worker-box/windows/worker-switch.ps1 @@ -0,0 +1,120 @@ +# The worker's switch. The FizzyWorker* scheduled tasks (SYSTEM) run this from \bin; nobody +# needs to be logged in, so `schtasks /run /tn \Fizzy\FizzyWorkerOn` works over SSH. +# On start the runner service (it is Manual, so only this starts it) and cap it. +# Off drain: take no more work, let the running job finish, then stop the service. +# ForceOff stop the service now and kill the job and everything it started. +# Apply (heartbeat) re-apply priority and core cap, write state\heartbeat.txt. +# Update refresh Git and gh with winget when no job is running. +# Windows PowerShell 5.1 only syntax: see the pitfall note in scripts/runner-setup.ps1. +[CmdletBinding()] +param( + [Parameter(Mandatory)][ValidateSet('On', 'Off', 'ForceOff', 'Apply', 'Update')][string]$Mode, + [Parameter(Mandatory)][string]$Root, + [int]$CoreCap = 0, + [int]$DrainHours = 12, + [switch]$DryRun +) +Set-StrictMode -Version 2 +$ErrorActionPreference = 'Stop' +$runner = Join-Path $Root 'actions-runner' +$state = Join-Path $Root 'state' +$log = Join-Path $state 'worker.log' + +function Write-Log($text) { + $line = "$((Get-Date).ToString('o')) $Mode $text" + Write-Output $line + if (-not $DryRun) { + if (-not (Test-Path -LiteralPath $state)) { New-Item -ItemType Directory -Path $state | Out-Null } + Add-Content -LiteralPath $log -Value $line + } +} +function Get-RunnerService { + $file = Join-Path $runner '.service' + if (-not (Test-Path -LiteralPath $file)) { throw "no runner service recorded at $file; run setup first" } + return (Get-Content -LiteralPath $file -Raw).Trim() +} +function Get-Jobs { return @(Get-Process -Name Runner.Worker -ErrorAction SilentlyContinue) } +# The cap is the highest-numbered logical cores, leaving the low ones (where Windows schedules its +# own work first) to the machine's owner. Children inherit affinity and BelowNormal priority. +function Set-Cap { + $count = [Environment]::ProcessorCount + $mask = $null + # Bit 63 would overflow Int64, so a 64-core machine is left uncapped. + if ($CoreCap -gt 0 -and $CoreCap -lt $count -and $count -le 63) { + $mask = (([int64]1 -shl $CoreCap) - 1) -shl ($count - $CoreCap) + } + foreach ($p in @(Get-Process -Name Runner.Listener, Runner.Worker -ErrorAction SilentlyContinue)) { + try { + $p.PriorityClass = 'BelowNormal' + if ($mask) { $p.ProcessorAffinity = [IntPtr]$mask } + } catch { Write-Log "cannot cap $($p.Name) $($p.Id): $_" } + } + Write-Log "cap applied: BelowNormal, cores=$CoreCap of $count" +} + +if ($DryRun) { + Write-Output "Would $Mode the runner under $Root (core cap $CoreCap); no changes made." + exit 0 +} + +switch ($Mode) { + 'On' { + $name = Get-RunnerService + $svc = Get-Service -Name $name + if ($svc.Status -ne 'Running') { + Start-Service -Name $name + $svc.WaitForStatus('Running', (New-TimeSpan -Seconds 60)) + } + $deadline = (Get-Date).AddSeconds(30) + while (-not (Get-Process -Name Runner.Listener -ErrorAction SilentlyContinue) -and (Get-Date) -lt $deadline) { Start-Sleep -Seconds 1 } + Write-Log "service $name running" + Set-Cap + } + 'Off' { + $name = Get-RunnerService + # The runner has no drain command, so poll. After a job ends the listener may take the next + # queued job within a moment, so a job counts as finished only after three quiet polls; a job + # accepted in that window is cancelled by the stop, which GitHub shows as a cancelled run. + $deadline = (Get-Date).AddHours($DrainHours) + $quiet = 0 + Write-Log "draining; $(@(Get-Jobs).Count) job(s) running" + while ($quiet -lt 3 -and (Get-Date) -lt $deadline) { + if (@(Get-Jobs).Count) { $quiet = 0 } else { $quiet++ } + if ($quiet -lt 3) { Start-Sleep -Seconds 2 } + } + if ($quiet -lt 3) { Write-Log "job still running after $DrainHours h; leaving the service up (use ForceOff)"; exit 1 } + Stop-Service -Name $name -ErrorAction Stop + (Get-Service -Name $name).WaitForStatus('Stopped', (New-TimeSpan -Seconds 60)) + Write-Log "service $name stopped" + } + 'ForceOff' { + $name = Get-RunnerService + $svc = Get-Service -Name $name + if ($svc.Status -ne 'Stopped') { Stop-Service -Name $name -Force -ErrorAction SilentlyContinue } + # /T takes the job's whole process tree (zig, git, shells) with it. + foreach ($p in @(Get-Process -Name Runner.Worker, Runner.Listener -ErrorAction SilentlyContinue)) { + & cmd.exe /c "taskkill.exe /F /T /PID $($p.Id) >nul 2>&1" + } + Write-Log "service $name stopped; processes killed" + } + 'Apply' { + if (-not (Test-Path -LiteralPath $state)) { New-Item -ItemType Directory -Path $state | Out-Null } + Set-Cap + $name = Get-RunnerService + $status = (Get-Service -Name $name).Status + Set-Content -LiteralPath (Join-Path $state 'heartbeat.txt') -Value "$((Get-Date).ToString('o')) service=$status jobs=$(@(Get-Jobs).Count)" + } + 'Update' { + if (@(Get-Jobs).Count) { Write-Log 'a job is running; skipping this week'; exit 0 } + $winget = @(Get-ChildItem -Path "$env:ProgramFiles\WindowsApps\Microsoft.DesktopAppInstaller_*\winget.exe" -ErrorAction SilentlyContinue | Sort-Object FullName -Descending | Select-Object -First 1) + if (-not $winget.Count) { Write-Log 'winget not available to SYSTEM; nothing updated'; exit 0 } + foreach ($id in 'Git.Git', 'GitHub.cli') { + # Native stderr is a terminating error under Stop in Windows PowerShell 5.1. + $ErrorActionPreference = 'Continue' + & cmd.exe /c "`"$($winget[0].FullName)`" upgrade --id $id --exact --silent --accept-source-agreements --accept-package-agreements >nul 2>&1" + $code = $LASTEXITCODE + $ErrorActionPreference = 'Stop' + Write-Log "winget upgrade $id exit $code" + } + } +} diff --git a/docs/AGENT_WORK.md b/docs/AGENT_WORK.md index 08e838eb7..8ea0e8e54 100644 --- a/docs/AGENT_WORK.md +++ b/docs/AGENT_WORK.md @@ -276,6 +276,38 @@ by a logon Scheduled Task instead of the service (`config.cmd` without `--runass keeps a password in the registry and an always-logged-in desktop, so it is not done here until there is such a job. +**The whole box, and the worker switch.** `ci/worker-box/windows/setup.ps1` (elevated, idempotent, +`-DryRun`) runs `runner-setup.ps1` with the root from `ci/worker-box/windows/config.psd1` (`Root`, the +agents' home, normally a drive of its own; `FIZZY_WORKER_ROOT` is the same value for `runner-setup.ps1` +alone), then converges the rest: Fast Startup and hibernation off, never sleep, `sshd` and Tailscale +automatic, the runner and its children at BelowNormal (`CoreCap` logical cores at most), `gh` on the +runner's PATH, a deny for `fizzy-worker` on every other fixed drive (an inheriting ACE on that drive's root, so it changes ACLs outside the worker's root and is hard to undo), and the SYSTEM tasks below. It +reports (never deletes) accounts outside `AllowedUsers`, and ends by running `doctor.ps1`; any RED +fails it. Every item has a doctor check, so `doctor.ps1` alone tells you whether the box still matches. + +The runner service is **Manual**; the tasks in `\Fizzy\` (a folder `fizzy-worker` cannot write) start +and stop it, run as SYSTEM, and need nobody logged in: + +| Task | Does | Who may run it | +|---|---|---| +| `FizzyWorkerOn` | start the runner (also at every boot, so the worker is **On** by default) | maintainer | +| `FizzyWorkerOff` | drain: no new job once the running one ends, then stop (up to 12 h) | maintainer, `fizzy-worker` | +| `FizzyWorkerForceOff` | stop now and kill the job's process tree | maintainer | +| `FizzyHeartbeat`, `FizzyUpdate` | every 5 minutes re-apply the priority and core cap; weekly refresh Git and gh with winget when idle | schedule | + +Remote on/off, from anywhere the box is reachable over SSH (Tailscale), with nobody logged in: + +```sh +ssh foxnn@ schtasks /run /tn "\Fizzy\FizzyWorkerOn" +ssh foxnn@ schtasks /run /tn "\Fizzy\FizzyWorkerOff" # waits for the running job +ssh foxnn@ schtasks /run /tn "\Fizzy\FizzyWorkerForceOff" # kills it +ssh foxnn@ powershell -Command "(Get-Service actions.runner.*).Status; Get-Content \state\worker.log -Tail 5" +``` + +A drain polls for `Runner.Worker` (the runner has no drain command); a job the listener accepts in the +two seconds between polls is cancelled by the stop. The doctor reports On and Off both GREEN +(`worker switch`), and a stopped worker's runner being offline as expected. + **Verify** (the maintainer; nothing here has run on real Windows): 1. `gh api repos/fizzyedit/work/actions/runners --jq '.runners[]|[.name,.status,([.labels[].name]|join(","))]|@tsv'` diff --git a/scripts/runner-clean.ps1 b/scripts/runner-clean.ps1 index 33da73a3d..4b632b455 100644 --- a/scripts/runner-clean.ps1 +++ b/scripts/runner-clean.ps1 @@ -2,10 +2,10 @@ # jj/git checkouts do not track. Mirrors scripts/runner-clean.sh; the opt-in deletion of merged # PR workspaces (the *.pr enrollment) is not ported to Windows. [CmdletBinding()] -param([switch]$DryRun) +param([switch]$DryRun, [string]$Root) Set-StrictMode -Version 2 $ErrorActionPreference = 'Stop' -$root = if ($env:FIZZY_WORKER_ROOT) { $env:FIZZY_WORKER_ROOT } else { 'C:\fizzy' } +$root = if ($Root) { $Root } elseif ($env:FIZZY_WORKER_ROOT) { $env:FIZZY_WORKER_ROOT } else { 'C:\fizzy' } $cacheNames = @('.zig-cache', 'zig-out', 'zig-pkg') if ($DryRun -and $env:OS -ne 'Windows_NT') { diff --git a/scripts/runner-setup.ps1 b/scripts/runner-setup.ps1 index b4c3d4c2d..6c84111e6 100644 --- a/scripts/runner-setup.ps1 +++ b/scripts/runner-setup.ps1 @@ -6,7 +6,7 @@ # Environment (all optional except RUNNER_TOKEN on first run): RUNNER_TOKEN, RUNNER_URL, # RUNNER_NAME, RUNNER_LABELS, FIZZY_BUILD_SLOTS, FIZZY_BUILD_MAX_LOAD, ZIG_VERSION, ZIG_SHA256, # JJ_VERSION, RUNNER_VERSION, DEPS_CACHE_URL, DEPS_CACHE_SHA256, ZSTD_VERSION, ZSTD_SHA256, FIZZY_RUNNER_ALLOW_UNCHECKED, -# FIZZY_RUNNER_DEPS_PARTIAL_OK, FIZZY_RUNNER_NO_DEFENDER_EXCLUSION. +# FIZZY_RUNNER_DEPS_PARTIAL_OK, FIZZY_RUNNER_NO_DEFENDER_EXCLUSION, FIZZY_WORKER_ROOT (default C:\fizzy). # # Windows PowerShell 5.1 pitfall: with $ErrorActionPreference = 'Stop', a native command that # writes anything to stderr (even `2>$null`) raises a terminating NativeCommandError. Every native @@ -40,7 +40,9 @@ function Native([scriptblock]$cmd) { } $worker = 'fizzy-worker' -$root = 'C:\fizzy' +$root = (Get-Setting 'FIZZY_WORKER_ROOT' 'C:\fizzy').TrimEnd('\') +# The agents' home: a folder, never a bare drive root (its ACL is rewritten below). +if ($root -notmatch '^[A-Za-z]:\\[^\\/:*?"<>|]+(\\[^\\/:*?"<>|]+)*$') { Fail "FIZZY_WORKER_ROOT must be an absolute folder path such as C:\fizzy, not a drive root: $root" } $runner = "$root\actions-runner" $workspaces = "$root\workspaces" $tools = "$root\tools" @@ -450,7 +452,11 @@ try { if (Test-Path "$runner\.env") { $keep = @(Get-Content "$runner\.env" | Where-Object { $_ -notmatch '^(FIZZY_BUILD_SLOTS|FIZZY_BUILD_MAX_LOAD|ZIG_GLOBAL_CACHE_DIR|PATH)=' }) } - $jobPath = "$gitRoot\cmd;$gitRoot\bin;$tools\zig;$tools\bin;" + [Environment]::GetEnvironmentVariable('Path', 'Machine') + # gh (Git for Windows does not ship it) is found where it is installed, so workflows can call it. + $ghDir = '' + $gh = Get-Command gh.exe -ErrorAction SilentlyContinue + if ($gh) { $ghDir = (Split-Path -Parent $gh.Source) + ';' } else { Write-Warning 'gh.exe not found: workflows cannot call gh until it is installed and setup is rerun' } + $jobPath = "$gitRoot\cmd;$gitRoot\bin;$ghDir$tools\zig;$tools\bin;" + [Environment]::GetEnvironmentVariable('Path', 'Machine') $lines = $keep + @("FIZZY_BUILD_SLOTS=$buildSlots", "FIZZY_BUILD_MAX_LOAD=$buildMaxLoad", "ZIG_GLOBAL_CACHE_DIR=$zigCache", "PATH=$jobPath") [IO.File]::WriteAllText("$runner\.env", (($lines -join "`r`n") + "`r`n"), (New-Object Text.UTF8Encoding $false)) } @@ -501,7 +507,7 @@ Step 'start the runner service' { Step "install runner-clean.ps1 to $bin and register the daily task FizzyRunnerClean (as $worker, S4U: no stored password)" { Copy-Item (Join-Path $sourceDir 'runner-clean.ps1') "$bin\runner-clean.ps1" -Force $action = New-ScheduledTaskAction -Execute 'powershell.exe' ` - -Argument "-NoProfile -ExecutionPolicy Bypass -File `"$bin\runner-clean.ps1`"" + -Argument "-NoProfile -ExecutionPolicy Bypass -File `"$bin\runner-clean.ps1`" -Root `"$root`"" $trigger = New-ScheduledTaskTrigger -Daily -At 4am -RandomDelay (New-TimeSpan -Minutes 30) $principal = New-ScheduledTaskPrincipal -UserId "$([Environment]::MachineName)\$worker" -LogonType S4U -RunLevel Limited $settings = New-ScheduledTaskSettingsSet -StartWhenAvailable -ExecutionTimeLimit (New-TimeSpan -Hours 2) -MultipleInstances IgnoreNew diff --git a/scripts/test-runner-setup-windows.sh b/scripts/test-runner-setup-windows.sh index c3eb543b8..178f32c69 100755 --- a/scripts/test-runner-setup-windows.sh +++ b/scripts/test-runner-setup-windows.sh @@ -6,4 +6,5 @@ if ! command -v pwsh >/dev/null 2>&1; then echo 'skip: pwsh not installed' exit 0 fi -exec pwsh -NoProfile -ExecutionPolicy Bypass -File ./test-runner-setup.ps1 +pwsh -NoProfile -ExecutionPolicy Bypass -File ./test-runner-setup.ps1 || exit 1 +exec pwsh -NoProfile -ExecutionPolicy Bypass -File ./test-windows-doctor.ps1 diff --git a/scripts/test-runner-setup.ps1 b/scripts/test-runner-setup.ps1 index 044e30009..1f51a6eee 100644 --- a/scripts/test-runner-setup.ps1 +++ b/scripts/test-runner-setup.ps1 @@ -1,7 +1,7 @@ # Checks the pure parts of scripts/runner-setup.ps1 through -DryRun, as test-runner-setup.sh does -# for the Ubuntu script. Needs pwsh and a POSIX or Windows host; skips cleanly without pwsh. -$pwsh = Get-Command pwsh -ErrorAction SilentlyContinue -if (-not $pwsh) { Write-Host 'skip: pwsh not installed'; exit 0 } +# for the Ubuntu script. The child scripts run under the engine running this one, so CI's Windows +# PowerShell 5.1 job exercises 5.1 and the Linux job pwsh. +$engine = (Get-Process -Id $PID).Path $dir = Split-Path -Parent $MyInvocation.MyCommand.Path $tmp = Join-Path ([IO.Path]::GetTempPath()) ('runner-test-' + [Guid]::NewGuid().ToString('N')) New-Item -ItemType Directory $tmp | Out-Null @@ -14,10 +14,19 @@ if ($IsWindows -or $env:OS -eq 'Windows_NT') { chmod +x "$tmp/gh" } $sep = [IO.Path]::PathSeparator +# Windows PowerShell 5.1 turns a native command's stderr under 2>&1 into an ErrorRecord, which +# terminates when the caller runs with $ErrorActionPreference = 'Stop' (GitHub's powershell shell +# does). The child's refusal text is data here, so capture it with the preference relaxed. +function Invoke-Child($script) { + $old = $ErrorActionPreference + $ErrorActionPreference = 'Continue' + try { return (& $engine -NoProfile -ExecutionPolicy Bypass -File $script -DryRun 2>&1 | Out-String) } + finally { $ErrorActionPreference = $old } +} function Setup($extraEnv) { $env:PATH = "$tmp$sep$script:origPath" foreach ($k in $extraEnv.Keys) { Set-Item "Env:$k" $extraEnv[$k] } - $out = & $pwsh.Source -NoProfile -ExecutionPolicy Bypass -File "$dir/runner-setup.ps1" -DryRun 2>&1 | Out-String + $out = Invoke-Child "$dir/runner-setup.ps1" $code = $LASTEXITCODE foreach ($k in $extraEnv.Keys) { Remove-Item "Env:$k" -ErrorAction SilentlyContinue } return @{ Out = $out; Code = $code } @@ -64,12 +73,17 @@ $pwsh7Only = 'AsHashtable', 'NoEmphasis', 'Parallel', 'ThrottleLimit', 'SkipCert 'StatusCodeVariable', 'ResponseHeadersVariable', 'AllowUnencryptedAuthentication', 'PreserveAuthorizationOnRedirect', 'Resume', 'UseQuotes' $allowed = @{ + 'Import-PowerShellDataFile' = 'Path', 'LiteralPath' + 'Get-CimInstance' = 'ClassName', 'Filter' + 'Get-WinEvent' = 'FilterHashtable' + 'Get-ItemPropertyValue' = 'Path', 'LiteralPath', 'Name' + 'Get-HotFix' = 'Id', 'Description', 'ComputerName', 'Credential' 'New-LocalUser' = 'Name', 'Password', 'FullName', 'Description', 'PasswordNeverExpires', 'UserMayNotChangePassword', 'AccountNeverExpires', 'AccountExpires', 'Disabled', 'NoPassword' 'Set-LocalUser' = 'Name', 'Password', 'FullName', 'Description', 'PasswordNeverExpires', 'UserMayChangePassword', 'AccountNeverExpires' 'Get-LocalUser' = 'Name', 'SID' 'Register-ScheduledTask' = 'TaskName', 'Action', 'Trigger', 'Principal', 'Settings', 'Description', 'Force', 'TaskPath', 'User', 'Password', 'RunLevel' 'New-ScheduledTaskAction' = 'Execute', 'Argument', 'WorkingDirectory' - 'New-ScheduledTaskTrigger' = 'Daily', 'Weekly', 'Once', 'AtLogOn', 'AtStartup', 'At', 'RandomDelay', 'DaysInterval' + 'New-ScheduledTaskTrigger' = 'Daily', 'Weekly', 'Once', 'AtLogOn', 'AtStartup', 'At', 'RandomDelay', 'DaysInterval', 'DaysOfWeek', 'WeeksInterval', 'RepetitionInterval', 'RepetitionDuration' 'New-ScheduledTaskPrincipal' = 'UserId', 'LogonType', 'RunLevel', 'GroupId', 'Id' 'New-ScheduledTaskSettingsSet' = 'StartWhenAvailable', 'ExecutionTimeLimit', 'MultipleInstances', 'AllowStartIfOnBatteries', 'DontStopIfGoingOnBatteries', 'Hidden' 'Start-ScheduledTask' = 'TaskName', 'TaskPath' @@ -77,7 +91,7 @@ $allowed = @{ 'Get-ScheduledTaskInfo' = 'TaskName', 'TaskPath' 'Add-MpPreference' = 'ExclusionPath', 'ExclusionExtension', 'ExclusionProcess' } -foreach ($name in 'runner-setup.ps1', 'runner-clean.ps1') { +foreach ($name in 'runner-setup.ps1', 'runner-clean.ps1', '../ci/worker-box/windows/doctor.ps1', '../ci/worker-box/windows/config.psd1', 'test-windows-doctor.ps1', '../ci/worker-box/windows/setup.ps1', '../ci/worker-box/windows/worker-switch.ps1') { $tokens = $null; $errors = $null $ast = [Management.Automation.Language.Parser]::ParseFile("$dir/$name", [ref]$tokens, [ref]$errors) if ($errors.Count) { Fail "$name does not parse: $($errors[0].Message)" } @@ -100,8 +114,18 @@ foreach ($name in 'runner-setup.ps1', 'runner-clean.ps1') { } } +# setup.ps1 -DryRun: the runner dry run, every convergence step printed, nothing changed. The +# private-repo check is skipped (it is covered above), so no gh or network is needed. $env:PATH = $origPath -$out = & $pwsh.Source -NoProfile -ExecutionPolicy Bypass -File "$dir/runner-clean.ps1" -DryRun 2>&1 | Out-String +$env:FIZZY_RUNNER_ALLOW_UNCHECKED = '1' +$out = Invoke-Child "$dir/../ci/worker-box/windows/setup.ps1" +$code = $LASTEXITCODE +Remove-Item Env:\FIZZY_RUNNER_ALLOW_UNCHECKED -ErrorAction SilentlyContinue +if ($code -ne 0) { Fail "setup.ps1 dry run failed: $out" } +foreach ($want in 'FIZZY_WORKER_ROOT=', 'FizzyWorkerOff', 'Fast Startup off', 'BelowNormal', 'Dry run complete') { + if ($out -notmatch [regex]::Escape($want)) { Fail "setup.ps1 dry run did not mention '$want': $out" } +} +$out = Invoke-Child "$dir/runner-clean.ps1" if ($LASTEXITCODE -ne 0) { Fail "runner-clean dry run failed: $out" } Remove-Item -Recurse -Force $tmp Write-Host 'ok' diff --git a/scripts/test-windows-doctor.ps1 b/scripts/test-windows-doctor.ps1 new file mode 100644 index 000000000..f9fb74288 --- /dev/null +++ b/scripts/test-windows-doctor.ps1 @@ -0,0 +1,95 @@ +# Pester-free doctor contract tests. Runs on pwsh 7 and Windows PowerShell 5.1. +$ErrorActionPreference = 'Stop' +$doctor = [IO.Path]::GetFullPath("$PSScriptRoot/../ci/worker-box/windows/doctor.ps1") +$engine = (Get-Process -Id $PID).Path +function Assert($condition, $message) { if (-not $condition) { throw $message } } +# Windows PowerShell 5.1 emits a JSON array as ONE object (pwsh 7 unrolls it), and joins nothing +# from piped lines, so join first and enumerate the parsed array explicitly. +function ConvertFrom-JsonRows($lines) { + $parsed = ConvertFrom-Json -InputObject (@($lines) -join "`n") + $rows = @(); foreach ($row in $parsed) { $rows += $row } + return , $rows +} +$preview = & $engine -NoProfile -ExecutionPolicy Bypass -File $doctor -DryRun +Assert ($LASTEXITCODE -eq 0) 'DryRun must have no reds' +Assert (@($preview).Count -ge 25) 'DryRun omitted checks' +foreach ($line in $preview) { Assert ($line -match '^AMBER [^:]+: .+$') "Invalid preview line: $line" } +$previewJson = ConvertFrom-JsonRows (& $engine -NoProfile -ExecutionPolicy Bypass -File $doctor -DryRun -Json) +Assert ($previewJson.Count -eq @($preview).Count) 'JSON and text checks differ' +Assert (@($previewJson | Where-Object { $_.status -ne 'AMBER' }).Count -eq 0) 'DryRun queried a probe' + +$tmp = Join-Path ([IO.Path]::GetTempPath()) ('doctor-test-' + [guid]::NewGuid().ToString('N')) +New-Item -ItemType Directory -Path $tmp | Out-Null +try { + # Isolate policy and query providers. No registry, services, credentials or installs are changed. + $fixture = @' +@{ + Services = @(@{ Name = 'test-service'; Account = 'LocalSystem' }) + RunnerRoot = 'missing-runner'; RunnerRepo = 'test/repo'; RunnerName = 'test' + Tasks = @('test-task'); AllowedUsers = @('allowed') + Tools = @(@{ Name = 'missing-tool'; Args = @('--version'); Version = '1.0' }) + MinDiskFreeGiB = 100; MinAvailableRamGiB = 4; MaxPatchDays = 30 + MaxCacheGiB = 40; ZigCache = 'missing-cache'; BuildSlots = 2 +} +'@ + Set-Content -LiteralPath "$tmp/config.psd1" -Value $fixture + $harness = @' +param($doctor, $config, $mode) +$env:GH_TOKEN = ''; $env:GITHUB_TOKEN = '' +function Get-CimInstance { + param($ClassName, $Filter) + switch ($ClassName) { + 'Win32_Service' { + [pscustomobject]@{ Name = 'test-service'; State = 'Stopped'; StartMode = 'Manual'; StartName = 'LocalSystem' } + } + 'Win32_OperatingSystem' { + [pscustomobject]@{ LastBootUpTime = (Get-Date).AddDays(-1); FreePhysicalMemory = 8MB; TotalVisibleMemorySize = 32MB } + } + 'Win32_LogicalDisk' { [pscustomobject]@{ DeviceID = 'X:'; FreeSpace = 10GB } } + } +} +function Get-LocalUser { + [pscustomobject]@{ Name = 'stray'; SID = [pscustomobject]@{ Value = 'S-1-5-21-123-1001' } } +} +function Get-ScheduledTask { param($TaskName, $ErrorAction) return } +function Get-HotFix { [pscustomobject]@{ HotFixID = 'fixture'; InstalledOn = (Get-Date).AddDays(-40) } } +function Get-WinEvent { param($FilterHashtable) throw 'fixture denied' } +function Get-ItemProperty { param($LiteralPath) throw 'fixture denied' } +function Get-ItemPropertyValue { param($LiteralPath, $Name) return 1 } +if ($mode -eq 'json') { & $doctor -ConfigPath $config -Json } +else { & $doctor -ConfigPath $config } +exit $LASTEXITCODE +'@ + Set-Content -LiteralPath "$tmp/harness.ps1" -Value $harness + $raw = & $engine -NoProfile -ExecutionPolicy Bypass -File "$tmp/harness.ps1" $doctor "$tmp/config.psd1" json + $code = $LASTEXITCODE + $rows = ConvertFrom-JsonRows $raw + $reds = @($rows | Where-Object { $_.status -eq 'RED' }).Count + Assert ($reds -ge 5 -and $code -eq $reds) "Fixture exit=$code; reds=$reds" + foreach ($pair in @( + @('service test-service', 'RED'), @('disk X:', 'RED'), @('task test-task', 'RED'), + @('local users', 'AMBER'), @('last patch', 'AMBER'), @('unclean shutdowns', 'AMBER'), + @('runner Online', 'AMBER'), @('RAM', 'GREEN'), @('tool missing-tool', 'AMBER') + )) { + $row = @($rows | Where-Object { $_.item -eq $pair[0] }) + Assert ($row.Count -eq 1 -and $row[0].status -eq $pair[1]) "Wrong fixture result: $($pair[0])" + } + $text = & $engine -NoProfile -ExecutionPolicy Bypass -File "$tmp/harness.ps1" $doctor "$tmp/config.psd1" text + Assert ($LASTEXITCODE -eq $reds) 'Text exit differs' + Assert (@($text).Count -eq $rows.Count) 'Text and JSON counts differ' + foreach ($line in $text) { Assert ($line -match '^(GREEN|RED|AMBER) .+: .+$') "Invalid line: $line" } + Write-Output "ok: preview, JSON, fixture statuses, text and exit count ($reds reds)" +} finally { Remove-Item -LiteralPath $tmp -Recurse -Force } + +if ($env:OS -eq 'Windows_NT') { + # Hosted VM health is evidence about these probes, not the dedicated worker's health. + $raw = & $engine -NoProfile -ExecutionPolicy Bypass -File $doctor -Json + $code = $LASTEXITCODE + Write-Output $raw + $rows = ConvertFrom-JsonRows $raw + Assert ($code -eq @($rows | Where-Object { $_.status -eq 'RED' }).Count) 'Hosted VM exit count differs' + Assert ($rows.Count -ge 25) 'Hosted VM omitted checks' + Write-Output "ok: hosted VM read-only probes; exit=$code (worker policy reds are expected here)" +} +# Successful contract checks must not inherit the expected unhealthy probe exit status. +exit 0