diff --git a/fork-kit/desk/README.md b/fork-kit/desk/README.md index 08f89ba9786c..9f7373ba7e99 100644 --- a/fork-kit/desk/README.md +++ b/fork-kit/desk/README.md @@ -118,7 +118,7 @@ three quarters of the RAM, at most the RAM less 6 GB, never under 4 GB; one that refused; without `-MemoryGB`, a higher one already there, raised by hand, is kept up to the RAM less 2 GB; one WSL cannot read, such as `26 GB` or `26gb`, is rewritten, since WSL would run on half the RAM), swap (`-SwapGB`, 4), NAT and DNS tunnelling, for every WSL distro (read and written as UTF-8 without a BOM; a dated copy of the old file is kept, other keys and sections stay; a section header is read as WSL reads it, -so a line WSL rejects, such as `[ wsl2 ]`, starts no section and the key under it is not taken as set; an old `networkingMode` or `dnsTunneling` +so a line WSL rejects, such as `[ wsl2 ]`, starts no section and the key under it is not taken as set; values too: the first line of a key is the one WSL uses, a `# comment` after a value and quotes are dropped, and a line ending in `\` goes on to the next, so a key there is part of that value; a merge WSL would not read as the question says (a lone CR between keys, or a `\` at the very end of the file) is refused and nothing changes; an old `networkingMode` or `dnsTunneling` line under `[experimental]`, the form before WSL 2.0 that can still win over `[wsl2]`, is removed, and the check FAILs while one is there; the question lists each change, old -> new, and the file is written only when it lists one, so "no change" means the file is left alone); the distro's own settings the WSL2 walls diff --git a/fork-kit/desk/tests/windows.test.py b/fork-kit/desk/tests/windows.test.py index 025e9cc46dee..6da53d5d6358 100644 --- a/fork-kit/desk/tests/windows.test.py +++ b/fork-kit/desk/tests/windows.test.py @@ -36,6 +36,152 @@ def wslconfig_part(): return src[src.index('# --- wslconfig ---'):src.index('# --- end wslconfig ---')] +def wsl_reads(text): + """WSL's own .wslconfig parser, ported statement by statement to be the tests' oracle: ParseConfigFile in parse + mode (microsoft/WSL src/shared/configfile/configfile.cpp at db6dc6cb, lines 239-825), with the + CFG_SKIP_INVALID_LINES that src/windows/common/WslCoreConfig.cpp:150 passes. It returns the (key, value) pairs it + hands SetConfig, in order; ConfigKey::Parse (configfile.cpp:187) keeps the first one of each key. WSL opens the + file as text (WslCoreConfig.cpp:136, rt,ccs=UTF-8), so CR LF reads as LF and a lone CR stays.""" + s = text.replace('\r\n', '\n') + st = {'pos': 0, 'eof': False} + + def getc(): # fgetwc; None is WEOF + if st['pos'] < len(s): + st['pos'] += 1 + return s[st['pos'] - 1] + st['eof'] = True + return None + + def hspace(c): + return c == ' ' or c == '\t' + + def alpha(c): # isalpha/isalnum in the C locale: ASCII only + return c is not None and c.isascii() and c.isalpha() + + def alnum(c): + return c is not None and c.isascii() and c.isalnum() + + def cstr(x): # SetConfig gets key.c_str(): a NUL ends it + return x.split('\0')[0] + + calls, ch, section_length, key, label = [], 0, 0, '', 'NewLine' + while True: + if label == 'NewLine': + while True: + if ch == '#': # skip a pending comment + while True: + ch = getc() + if ch == '\r': + ch = getc() + if ch == '\n' or ch is None: + break + if st['eof']: + return calls + ch = getc() + while hspace(ch): + ch = getc() + if ch is None or ch == '\n' or ch == '#': + continue + if ch == '\r': + if getc() not in ('\n', None): + st['pos'] -= 1 # ungetwc + continue + label = 'ParseSection' if ch == '[' else 'ParseKeyValue' if alpha(ch) else 'InvalidLine' + break + elif label == 'ParseSection': + ch = getc() + if not alpha(ch): + label = 'InvalidLine' + continue + key = '' + while True: + key += ch + ch = getc() + if not alnum(ch): + break + if ch != ']': + label = 'InvalidLine' + continue + ch = getc() + while hspace(ch): + ch = getc() + if not (ch is None or ch in ('\n', '\r', '#')): + label = 'InvalidLine' + continue + section_length = len(key) + label = 'NewLine' + elif label == 'ParseKeyValue': + key = (key + '\0' * section_length)[:section_length] + ('.' if section_length else '') # key.resize + while True: + key += ch + ch = getc() + if not alnum(ch): + break + while hspace(ch): + ch = getc() + if ch != '=': + label = 'InvalidLine' + continue + ch = getc() + while hspace(ch): + ch = getc() + value, trimmed, in_quote, invalid = [], 0, False, False + while ch is not None and ch != '\n' and ch != '\r': + if ch == '"': + in_quote = not in_quote + elif ch == '\\': + ch2 = getc() + if ch2 in ('\\', '"'): + value.append(ch2) + elif ch2 in ('b', 'n', 't'): + value.append({'b': '\b', 'n': '\n', 't': '\t'}[ch2]) + elif ch2 != '\n': # '\n': the line goes on + invalid = True + break + elif ch == '#' and not in_quote: + break + else: + value.append(ch) + if not hspace(ch): + trimmed = len(value) + ch = getc() + if invalid or in_quote: + label = 'InvalidLine' + continue + calls.append((cstr(key), cstr(''.join(value[:trimmed])))) + label = 'NewLine' + else: # InvalidLine + while ch is not None and ch != '\n': + ch = getc() + label = 'NewLine' + + +# The names WSL reads each key the desk manages under (WslCoreConfig.cpp:116-117: the old [experimental] names are +# the same key, so whichever comes first in the file is the one used). +WSL_NAMES = {'memory': ('wsl2.memory',), 'swap': ('wsl2.swap',), 'guiapplications': ('wsl2.guiapplications',), + 'networkingmode': ('wsl2.networkingmode', 'experimental.networkingmode'), + 'dnstunneling': ('wsl2.dnstunneling', 'experimental.dnstunneling')} + + +def wsl_value(calls, name): + """The value WSL uses for one key: the first line of it, even one whose value WSL then rejects. None: not set.""" + for k, v in calls: + if k.lower() in WSL_NAMES[name.lower()]: + return v + return None + + +def wsl_others(calls): + """What WSL reads besides the keys the desk manages: these must be the same before and after a merge.""" + managed = {n for names in WSL_NAMES.values() for n in names} + return [(k.lower(), v) for k, v in calls if k.lower() not in managed] + + +def wsl_same(value, want): + """Whether WSL reads value as want: a size exactly (ParseMemorySize: case matters), a word in any ASCII case.""" + return value == want or (want.isalpha() and value is not None and value.isascii() and value.lower() == want) + + def walls(root): os.environ['FORK_FLEET_WALLS_ROOT'] = root try: @@ -248,7 +394,7 @@ def test_a_stopped_run_keeps_its_marks(self): self.assertLess(body.index('Mark $Made'), body.index('wsl.exe --install')) self.assertIn('Merge-WslConfig $WslConfig $want { Mark $Owed }', body) self.assertIn('& $before', wslconfig_part()) - self.assertLess(wslconfig_part().index('& $before'), wslconfig_part().index('WriteAllLines')) + self.assertLess(wslconfig_part().index('& $before'), wslconfig_part().index('WriteAllText("$path.new"')) shutdown = body.index('Run \'wsl --shutdown\'') self.assertLess(body.index('if (Test-Path -LiteralPath $Owed)'), shutdown) self.assertGreater(body.index('Remove-Item -LiteralPath $Owed -ErrorAction'), shutdown) @@ -275,9 +421,9 @@ def test_old_experimental_networking_keys_are_removed_named_and_checked(self): part = wslconfig_part() self.assertIn("$Experimental = @('networkingMode', 'dnsTunneling')", part) merge = part[part.index('function MergeLines'):part.index('function WslVersion')] - self.assertIn("$sec -eq 'experimental'", merge) - self.assertIn('$Matches[1] -in $Experimental) {', merge) - self.assertIn(': removed (the [wsl2] one is used)"); continue }', merge) + self.assertIn("foreach ($k in $Experimental) { $old[$k.ToLowerInvariant()] = 1 }", merge) + self.assertIn("elseif ($e.Sec -ceq 'experimental' -and $old.ContainsKey($lk)) {", merge) + self.assertIn(': removed (the [wsl2] one is used)"); Drop $e.First $e.Last }', merge) src = text().decode('ascii') body = src[src.index('try {'):] ask = min(body.index("Read-Host 'Type yes"), body.index('Read-Host "Type the distro')) @@ -285,6 +431,7 @@ def test_old_experimental_networking_keys_are_removed_named_and_checked(self): check = body[body.index("Head 'check'"):] self.assertIn("SectionKeys (Read-WslConfig $WslConfig) 'experimental'", check) self.assertIn('FAIL wslconfig [experimental] $k=', check) + self.assertIn('if (SameValue $cfg[$k] $want[$k]) { "PASS wslconfig', check) # as WSL compares (#387) def test_the_question_and_the_merge_decide_the_same_way(self): """lens d8min r3 N2: the question compared trimmed values, the merge whole text, so "no change" could be shown @@ -432,6 +579,188 @@ def test_section_headers_are_read_as_wsl_reads_them(self): if n.startswith('.wslconfig.bak-'): os.unlink(os.path.join(d, n)) + # The desk's own keys (start-desk.ps1's $want), -MemoryGB not given: 8 GB, a higher cap kept up to 30 GB. + VALUES_SCRIPT = ('\nforeach ($path in $args) {\n try {\n' + ' $want = [ordered]@{ memory = (WantMemory $path 8 30); swap = "4GB"; networkingMode = "nat"; ' + 'dnsTunneling = "true"; guiApplications = "false" }\n' + ' $k = Wsl2Keys (Read-WslConfig $path); $x = SectionKeys (Read-WslConfig $path) "experimental"\n' + ' $o = [ordered]@{ path = $path; want = $want.memory; memory = $k["memory"]; swap = $k["swap"]; ' + 'networkingMode = $k["networkingMode"]; dnsTunneling = $k["dnsTunneling"]; ' + 'guiApplications = $k["guiApplications"]; experimental = @($x.Keys | Sort-Object) }\n' + ' try { $o.changed = Merge-WslConfig $path $want; $o.after = @(Changes (Read-WslConfig $path) $want).Count;' + ' $o.again = Merge-WslConfig $path $want } catch { $o.refused = $_.Exception.Message }\n' + ' } catch { $o = @{ path = $path; error = "$_" } }\n' + ' $o | ConvertTo-Json -Compress\n}\n') + WANT = {'swap': '4GB', 'networkingMode': 'nat', 'dnsTunneling': 'true', 'guiApplications': 'false'} + + def run_values(self, files, culture='C'): + """Each file (None: no file) merged by the real Read-WslConfig/Wsl2Keys/WantMemory/Merge-WslConfig, in one + pwsh; for each: what the script read, its merge, and the file after it.""" + import json + d = tempfile.mkdtemp(prefix='desk-wslconfig-test-') + self.addCleanup(shutil.rmtree, d, True) + script = os.path.join(d, 'values.ps1') + with open(script, 'w', encoding='ascii') as f: + f.write(wslconfig_part() + self.VALUES_SCRIPT) + paths = [] + for i, old in enumerate(files): + os.mkdir(os.path.join(d, str(i))) + paths.append(os.path.join(d, str(i), '.wslconfig')) + if old is not None: + with open(paths[-1], 'w', encoding='utf-8', newline='') as f: + f.write(old) + r = subprocess.run(['pwsh', '-NoProfile', '-NonInteractive', '-File', script] + paths, capture_output=True, + text=True, timeout=600, stdin=subprocess.DEVNULL, env={**os.environ, 'LC_ALL': culture}) + self.assertEqual(r.returncode, 0, r.stdout + r.stderr) + out = [json.loads(line) for line in r.stdout.splitlines() if line.startswith('{')] + self.assertEqual([o['path'] for o in out], paths, r.stdout + r.stderr) + for o, path in zip(out, paths): + o['dir'] = os.path.dirname(path) + if os.path.exists(path): + with open(path, encoding='utf-8-sig', newline='') as f: + o['new'] = f.read() + else: + o['new'] = None + return out + + def check_merge(self, old, o): + """What WSL itself reads (the oracle) agrees with the script before the merge and after it.""" + before = wsl_reads(old or '') + self.assertNotIn('error', o) + for k in ('memory', 'swap', 'networkingMode', 'dnsTunneling', 'guiApplications'): # the [wsl2] line WSL uses + self.assertEqual(o[k], next((v for n, v in before if n.lower() == 'wsl2.' + k.lower()), None), k) + self.assertEqual(o['experimental'], sorted({n.split('.', 1)[1] for n, _ in before + if n.lower().startswith('experimental.')}, key=str.lower)) + leftovers = [n for n in os.listdir(o['dir']) if n != '.wslconfig' and not n.startswith('.wslconfig.bak-')] + self.assertEqual(leftovers, []) + if 'refused' in o: # refused before anything changed + self.assertEqual(o['new'], old) + return + after = wsl_reads(o['new']) + for k, w in {'memory': o['want'], **self.WANT}.items(): + self.assertTrue(wsl_same(wsl_value(after, k), w), f'{k}: WSL reads {wsl_value(after, k)!r}, not {w}') + self.assertEqual(wsl_others(after), wsl_others(before)) # every other key WSL reads, as it was + self.assertEqual(o['after'], 0) # run again: nothing to change, file left alone + self.assertIs(o['again'], False) + backups = [n for n in os.listdir(o['dir']) if n.startswith('.wslconfig.bak-')] + self.assertEqual(len(backups), int(o['changed'] and old is not None)) # a dated copy of a file that was there + + @unittest.skipUnless(shutil.which('pwsh'), 'pwsh is not installed') + def test_values_are_read_as_wsl_reads_them(self): + """#387: the value of a key=value line as WSL's parser reads it (configfile.cpp): the first line of a key is the + one used (ConfigKey::Parse warns of a later one and skips it), a # outside quotes starts a comment, quotes are + dropped, and a \\ at the end of a line goes on to the next line, which then is no key of its own.""" + w = 'swap=4GB\nnetworkingMode=nat\ndnsTunneling=true\nguiApplications=false\n' + # (old file (None: no file), the [wsl2] memory WSL reads, the file after the merge: None unchanged, or REFUSED) + rows = ( + # repeated key: WSL uses the first, so a cap raised by hand above an older one is kept + ('[wsl2]\nmemory=20GB\n' + w + 'memory=12GB\n', '20GB', '[wsl2]\nmemory=20GB\n' + w), + # ... even one WSL rejects: it then runs on its default, and the later line is not read + ('[wsl2]\nmemory=26 GB\n' + w + 'memory=20GB\n', '26 GB', '[wsl2]\nmemory=8GB\n' + w), + # a comment after the value + ('[wsl2]\nmemory=20GB # raised by hand\n' + w, '20GB', None), + ('[wsl2]\nmemory=20GB#raised\n' + w, '20GB', None), + ('[wsl2]\r\nmemory=20GB # raised\r\n' + w.replace('\n', '\r\n'), '20GB', None), + # quotes are dropped; a # inside them is part of the value (WSL then rejects this one) + ('[wsl2]\nmemory="20GB"\n' + w, '20GB', None), + ('[wsl2]\nmemory="20GB # raised"\n' + w, '20GB # raised', '[wsl2]\nmemory=8GB\n' + w), + # a \ at the end goes on: memory= is part of kernelCommandLine, not a key, so the cap is added + ('[wsl2]\nkernelCommandLine=quiet \\\nmemory=20GB\n' + w, None, + '[wsl2]\nkernelCommandLine=quiet \\\nmemory=20GB\n' + w + 'memory=8GB\n'), + # ... and a header too: the keys after it are still [wsl2]'s + ('[wsl2]\nmemory=20GB\nkernelCommandLine=quiet \\\n[experimental]\n' + w, '20GB', None), + # ... on the last line: an empty line ends it before the [wsl2] added after it + ('[experimental]\nautoProxy=true\nignoredPorts=1,2 \\\n', None, + '[experimental]\nautoProxy=true\nignoredPorts=1,2 \\\n\n[wsl2]\nmemory=8GB\n' + w), + # ... with no LF after it, WSL drops the line; a line added after it would be read: refused + ('[wsl2]\nmemory=20GB\nswap=4GB\nnetworkingMode=nat\ndnsTunneling=true\nkernelCommandLine=quiet \\', + '20GB', 'REFUSED'), + # an escaped \ (\\) at the end does not go on + ('[wsl2]\nkernelCommandLine=quiet \\\\\nmemory=20GB\n' + w, '20GB', None), + # a line WSL drops (an unknown \ escape, a quote left open) is not read and left as it is + ('[wsl2]\nmemory=20\\GB\nmemory=12GB\n' + w, '12GB', None), + ('[wsl2]\nmemory="20GB\n' + w, None, '[wsl2]\nmemory="20GB\n' + w + 'memory=8GB\n'), + # ... a quote left open drops the line up to its LF, past a lone CR: swap= after it is not read either + ('[wsl2]\nmemory="2GB\rswap=1GB\nkernel=y\n', None, + '[wsl2]\nmemory="2GB\rswap=1GB\nkernel=y\nmemory=8GB\n' + w), + # a key replaced with the lines its value goes on to (\ at the end), so none of them is read as a key + ('[wsl2]\nmemory=2\\\nGB\nswap=1\\\nkernel=x\n', '2GB', '[wsl2]\nmemory=8GB\n' + w), + # a lone CR does not end a comment (it ends a value or a header) + ('[wsl2]\n# raised\rmemory=20GB\n' + w, None, '[wsl2]\n# raised\rmemory=20GB\n' + w + 'memory=8GB\n'), + # ... a lone CR between two keys on one line: replacing memory= would delete the kernel= WSL reads: refused + ('[wsl2]\nmemory=2GB\rkernel=C:\\\\k\\\\bz\n' + w, '2GB', 'REFUSED'), + # sizes are compared exactly (ParseMemorySize: 4gb is rejected), words in any case + ('[wsl2]\nmemory=20GB\nswap=4gb\nnetworkingMode=NAT\ndnsTunneling=TRUE\nguiApplications=False\n', '20GB', + '[wsl2]\nmemory=20GB\nswap=4GB\nnetworkingMode=NAT\ndnsTunneling=TRUE\nguiApplications=False\n'), + # old or empty state: an empty file, no file, no LF at the end + ('', None, '[wsl2]\nmemory=8GB\n' + w), + (None, None, '[wsl2]\nmemory=8GB\n' + w), + ('[wsl2]\nmemory=20GB', '20GB', '[wsl2]\nmemory=20GB\n' + w), + ) + for culture in ('C', 'tr_TR.UTF-8'): + for (old, memory, new), o in zip(rows, self.run_values([r[0] for r in rows], culture)): + with self.subTest(old, culture=culture): + self.assertEqual(o['memory'], memory) + self.assertEqual(wsl_value(wsl_reads(old or ''), 'memory'), memory) # the row is WSL's + self.check_merge(old, o) + if new == 'REFUSED': + self.assertIn('nothing was changed', o.get('refused', '')) + else: + self.assertNotIn('refused', o) + self.assertEqual(o['new'], old if new is None else new) + self.assertIs(o['changed'], new is not None) + + @unittest.skipUnless(shutil.which('pwsh'), 'pwsh is not installed') + def test_random_files_are_read_and_merged_as_wsl_reads_them(self): + """The approach of the #354 re-check: random files from lines that WSL reads in odd ways; the script's view + must be WSL's (the oracle above), and after a merge WSL must read the desk's keys and every other key as it + did. A refusal must leave the file alone.""" + import random + rnd = random.Random(387) + parts = ['[wsl2]', '[WSL2]', '[experimental]', '[boot]', '[wsl2] # c', '[ wsl2 ]', '[wsl2] x', '\t[wsl2]', + '[experimental] ;c', '[1x]', 'memory=20GB', 'memory = 9GB', 'memory=20GB # c', 'memory="20GB"', + 'memory="20GB # c"', 'memory=12GB', 'memory=lots', 'memory=20\\GB', 'memory="20GB', 'memory=9GB\\n', + 'swap=4GB', 'swap=4gb', 'networkingMode=nat', 'networkingMode=mirrored', 'NetworkingMode=NAT', + 'dnsTunneling=true', 'guiApplications=false', 'kernelCommandLine=quiet \\', 'kernel=C:\\\\k\\\\', + 'autoProxy=true', 'sparseVhd=true # c', '# comment \\', '# c\rmemory=7GB', 'x', '', ' ', + 'memory=10GB\rswap=2GB', 'swap=1GB\rautoProxy=true', 'firewall=true'] + files = [] + for _ in range(240): + eol = rnd.choice(['\n', '\r\n']) + lines = [rnd.choice(parts) for _ in range(rnd.randint(0, 9))] + files.append(eol.join(lines) + rnd.choice([eol, eol, ''])) + refused = 0 + for old, o in zip(files, self.run_values(files)): + with self.subTest(old): + self.check_merge(old, o) + refused += 'refused' in o + # the two known kinds (a lone CR between two keys, a \ at the end with no LF) refuse these: a new kind shows + self.assertEqual(refused, 5) + + @unittest.skipUnless(shutil.which('pwsh'), 'pwsh is not installed') + def test_a_rewrite_stopped_half_way_is_done_again(self): + """A run stopped while rewriting .wslconfig, before its move: the old file is still there, with a half-written + .new beside it. The next run merges the old file as a clean run does, and its own .new replaces that one.""" + old = '[wsl2]\nmemory=20GB # raised\nswap=1GB\n' + clean = self.run_values([old])[0]['new'] + d = tempfile.mkdtemp(prefix='desk-wslconfig-test-') + self.addCleanup(shutil.rmtree, d, True) + path = os.path.join(d, '.wslconfig') + put(path, old) + put(path + '.new', '[wsl2]\nmemory=2GB\n') + script = os.path.join(d, 'merge.ps1') + with open(script, 'w', encoding='ascii') as f: + f.write(wslconfig_part() + '\n$want = [ordered]@{ memory = (WantMemory $args[0] 8 30); swap = "4GB"; ' + 'networkingMode = "nat"; dnsTunneling = "true"; guiApplications = "false" }\n' + '"changed=$(Merge-WslConfig $args[0] $want)"\n') + r = subprocess.run(['pwsh', '-NoProfile', '-NonInteractive', '-File', script, path], capture_output=True, + text=True, timeout=120, stdin=subprocess.DEVNULL) + self.assertEqual(r.returncode, 0, r.stdout + r.stderr) + self.assertIn('changed=True', r.stdout) + self.assertEqual(slurp(path), clean) + self.assertFalse(os.path.exists(path + '.new')) + self.assertEqual(wsl_value(wsl_reads(clean), 'memory'), '20GB') # the raised cap, as WSL reads it + @unittest.skipUnless(shutil.which('pwsh'), 'pwsh is not installed') def test_the_memory_cap_kept_is_one_wsl_reads_within_its_bounds(self): """Without -MemoryGB, a higher cap a person set (more agents or jobs in WSL) is neither lowered nor a FAIL, nor @@ -498,6 +827,13 @@ def test_wslconfig_is_utf8_without_a_bom_never_ascii(self): self.assertNotRegex(src, r'(?i)(Get|Set)-Content[^\n]*\$WslConfig') self.assertIn('[Text.UTF8Encoding]::new($false)', wslconfig_part()) + def test_the_wslconfig_parser_searches_ordinally(self): + """lens #414 nit 6: a string IndexOf compares by culture, and under ICU "\r\n".IndexOf("\n") can be -1, which + would skip the rest of the file after a lone CR; a [char] search is ordinal.""" + part = wslconfig_part() + self.assertIn('$t.IndexOf([char]10, $i)', part) + self.assertNotRegex(part, r'\.IndexOf\("') + @unittest.skipUnless(shutil.which('pwsh'), 'pwsh is not installed') def test_wslconfig_merge_keeps_every_other_letter_and_key(self): d = tempfile.mkdtemp(prefix='desk-wslconfig-test-') diff --git a/fork-kit/desk/windows/start-desk.ps1 b/fork-kit/desk/windows/start-desk.ps1 index 863d0fa852bd..ac18075977f2 100644 --- a/fork-kit/desk/windows/start-desk.ps1 +++ b/fork-kit/desk/windows/start-desk.ps1 @@ -114,56 +114,145 @@ fi # BOM as ANSI and Set-Content with the ASCII encoding turns every other letter into '?' (a kernel= path with an accent in it # then stops every distro). The tests run this part alone (between the two marker lines) under pwsh. # --- wslconfig --- +# The lines of .wslconfig as WSL opens it (rt,ccs=UTF-8: CR LF reads as LF, a lone CR is a character in a line), split +# on LF only. The last one is what follows the last LF ('' when the file ends in one), so joined by LF they are the text. function Read-WslConfig([string]$path) { if (-not (Test-Path -LiteralPath $path)) { return @() } - @([IO.File]::ReadAllLines($path, [Text.UTF8Encoding]::new($false))) # a BOM (UTF-8 or UTF-16) is still read + @([IO.File]::ReadAllText($path, [Text.UTF8Encoding]::new($false)) -replace "`r`n", "`n" -split "`n") # a BOM (UTF-8 or UTF-16) is still read } -# networkingMode and dnsTunneling were [experimental] keys before WSL 2.0; an old line there can still win over -# [wsl2], so Merge-WslConfig removes them there (the [wsl2] ones are what the desk needs) and the check FAILs on them. +# networkingMode and dnsTunneling were [experimental] keys before WSL 2.0; WSL takes both names for one key and uses the +# first line of it, so an old one there can still win over [wsl2]: Merge-WslConfig removes them there (the [wsl2] ones +# are what the desk needs) and the check FAILs on them. $Experimental = @('networkingMode', 'dnsTunneling') -# The section after line $l, as WSL's ParseSection reads a header: space or tab, an open bracket, a letter then letters -# or digits, a close bracket, space or tab, then the end or a # comment; any ASCII case. A bracket line it rejects keeps -# the length of $sec of the letters read (none: $sec as it was; fewer pad with '*' and match no section; WSL pads with -# NUL, but PowerShell's -eq ignores NUL, so 'wsl2' and NULs would equal 'wsl2'). $null: no bracket. -function NextSection([string]$l, [string]$sec) { - if ($l -cnotmatch '^[ \t]*\[(([A-Za-z][A-Za-z0-9]*)(\][ \t]*(#.*)?$)?)?') { return $null } - if ($Matches[3]) { return $Matches[2].ToLowerInvariant() } - if ($Matches[2]) { return $Matches[2].ToLowerInvariant().PadRight($sec.Length, '*').Substring(0, $sec.Length) } - $sec +# What WSL reads, in order, as ParseConfigFile in its src/shared/configfile/configfile.cpp does (parse mode, invalid +# lines skipped): one item per header and per key=value line. Lines start after space or tab; # starts a comment. +# A header: '[', a letter then letters or digits, ']', space or tab, then the end or a # comment; any ASCII case. A header +# WSL rejects after reading a name keeps the old section's length of it (fewer letters pad with '*', which matches no +# section; WSL pads with NUL, but PowerShell's -eq ignores NUL); one with no name read is no item. Sec: the section of +# the keys after it. A key: a letter then letters or digits, space or tab, '='. Its Value as WSL reads it: space and +# tab after '=' and at the end dropped, a # outside quotes starts a comment, quotes dropped, \\ \" \b \n \t, and a \ +# at the end of a line goes on with the next line (which then is part of this value, header or key=value alike); +# $null when WSL drops the line (another \ escape, or a quote left open). First, Last: the lines it spans. +# Open: the text ends inside a value, after a \ and its LF, so a line added after it would be part of that value. +function WslItems([string[]]$lines) { + $t = $lines -join "`n"; $n = $t.Length; $i = 0; $sec = ''; $open = $false + $items = [Collections.Generic.List[object]]::new() + $ln = [int[]]::new($n + 1); for ($p = 0; $p -lt $n; $p++) { $ln[$p + 1] = $ln[$p] + [int]($t[$p] -eq "`n") } + $hs = { $i -lt $n -and ($t[$i] -eq ' ' -or $t[$i] -eq "`t") } + $eol = { $e = $t.IndexOf([char]10, $i); if ($e -lt 0) { $n } else { $e + 1 } } # past the LF that ends this line + while ($i -lt $n) { + while (& $hs) { $i++ } + if ($i -ge $n) { break } + $s = $i; $c = $t[$i]; $i++ + if ($c -eq "`n" -or $c -eq "`r") { continue } + if ($c -eq '#') { $i = & $eol; continue } + if ($c -eq '[') { + if ($i -ge $n -or $t[$i] -cnotmatch '[A-Za-z]') { $i = & $eol; continue } # no name read: the section stays + $j = $i; while ($i -lt $n -and $t[$i] -cmatch '[A-Za-z0-9]') { $i++ } + $name = $t.Substring($j, $i - $j).ToLowerInvariant(); $ok = $false + if ($i -lt $n -and $t[$i] -eq ']') { + $i++; while (& $hs) { $i++ } + $ok = $i -ge $n -or $t[$i] -eq "`n" -or $t[$i] -eq "`r" -or $t[$i] -eq '#' + } + $sec = if ($ok) { $name } else { $name.PadRight($sec.Length, '*').Substring(0, $sec.Length) } + $items.Add([pscustomobject]@{ Sec = $sec; Name = $null; First = $ln[$s]; Last = $ln[$s] }) + if (-not $ok -or ($i -lt $n -and $t[$i] -eq '#')) { $i = & $eol } elseif ($i -lt $n) { $i++ } + continue + } + if ($c -cnotmatch '[A-Za-z]') { $i = & $eol; continue } + while ($i -lt $n -and $t[$i] -cmatch '[A-Za-z0-9]') { $i++ } + $name = $t.Substring($s, $i - $s) + while (& $hs) { $i++ } + if ($i -ge $n -or $t[$i] -ne '=') { $i = & $eol; continue } + $i++; while (& $hs) { $i++ } + $v = [Text.StringBuilder]::new(); $keep = 0; $q = $false; $bad = $false; $end = $n + while ($i -lt $n) { + $c = $t[$i]; $i++ + if ($c -eq "`n" -or $c -eq "`r") { $end = $i - 1; break } + if ($c -eq '"') { $q = -not $q } + elseif ($c -eq '\') { + $d = if ($i -lt $n) { $t[$i] } else { $null }; $i++ + if ($d -eq '\' -or $d -eq '"') { [void]$v.Append($d) } + elseif ($d -ceq 'b') { [void]$v.Append("`b") } elseif ($d -ceq 'n') { [void]$v.Append("`n") } + elseif ($d -ceq 't') { [void]$v.Append("`t") } + elseif ($d -ne "`n") { $bad = $true; $i = [math]::Min($i, $n); break } # WSL drops the line + } + elseif ($c -eq '#' -and -not $q) { $end = $i - 1; $i = & $eol; break } + else { [void]$v.Append($c) } + if ($c -ne ' ' -and $c -ne "`t") { $keep = $v.Length } # space and tab at the end dropped + } + if ($i -ge $n -and $end -eq $n -and -not $bad -and $n -gt 0 -and $t[$n - 1] -eq "`n") { $open = $true } + # a line WSL drops: it skips the rest of it to its LF (past a lone CR) + if ($bad -or ($q -and $end -lt $n -and $t[$end] -eq "`r")) { $i = & $eol; $end = $i - 1 } + $value = if ($bad -or $q) { $null } else { $v.ToString().Substring(0, $keep) } + $items.Add([pscustomobject]@{ Sec = $sec; Name = $name; Value = $value; First = $ln[$s]; Last = $ln[[math]::Min($end, $n)] }) + } + [pscustomobject]@{ Items = $items; Open = $open } } -# A key=value line, trimmed of space and tab only, as WSL's config parser does: a non-breaking space or a form feed -# stays in the value (WSL then rejects a memory= value, so it is rewritten), where .NET's \s would drop it. -function SectionKeys([string[]]$lines, [string]$section) { # the keys of one section only - $keys = @{}; $sec = '' - foreach ($l in $lines) { - $next = NextSection $l $sec - if ($null -ne $next) { $sec = $next; continue } - if ($sec -eq $section -and $l -match '^[ \t]*([^=#; \t]+)[ \t]*=[ \t]*(.*?)[ \t]*$') { $keys[$Matches[1]] = $Matches[2] } +# The keys of one section as WSL reads them: the first line of each (WSL warns of a later one and skips it; a line it +# drops is not one), names in any ASCII case. +function FirstKeys($items, [string]$section) { + $keys = [hashtable]::new([StringComparer]::OrdinalIgnoreCase) + foreach ($e in $items) { + if ($null -ne $e.Name -and $null -ne $e.Value -and $e.Sec -ceq $section -and -not $keys.ContainsKey($e.Name)) { $keys[$e.Name] = $e.Value } } $keys } +function SectionKeys([string[]]$lines, [string]$section) { FirstKeys (WslItems $lines).Items $section.ToLowerInvariant() } function Wsl2Keys([string[]]$lines) { SectionKeys $lines 'wsl2' } +# Whether WSL reads $v as $w: a size (8GB) exactly (ParseMemorySize: 8gb is rejected), a word (nat, true) in any ASCII case. +function SameValue([string]$v, [string]$w) { + if ($w -cmatch '^[a-z]+\z' -and $v -cmatch '^[A-Za-z]+\z') { $v = $v.ToLowerInvariant() } + [string]::Equals($v, $w, [StringComparison]::Ordinal) +} # The merged lines and one note per change (old -> new). The question shows these notes and the file is written only -# when there is one, so "no change" always means the file is left alone. A [wsl2] line whose value already matches -# (trimmed, any case: as the final check compares) is kept as it is; a repeated key's later lines are removed. +# when there is one, so "no change" always means the file is left alone. Only lines WSL reads count: the first +# [wsl2] line of a key is kept when WSL reads it as wanted (SameValue) and replaced otherwise, with the lines its value +# goes on to; a later one is removed; a line WSL drops is left as it is. Missing keys go at the end of [wsl2], or in a +# [wsl2] added at the end. Then the merged text is read again: unless WSL would read the wanted keys and every other +# key exactly as before (a lone CR, or a \ with no LF after it at the end, can make that so), it refuses. function MergeLines([string[]]$lines, $want) { - $out = [Collections.Generic.List[string]]::new(); $notes = [Collections.Generic.List[string]]::new(); $sec = ''; $done = @{} - function AddMissing { foreach ($k in $want.Keys) { if (-not $done[$k]) { $out.Add("$k=$($want[$k])"); $done[$k] = 1; $notes.Add(" [wsl2] ${k}: (not set) -> $($want[$k])") } } } - foreach ($l in $lines) { - $next = NextSection $l $sec - if ($null -ne $next) { if ($sec -eq 'wsl2' -and $next -ne 'wsl2') { AddMissing }; $sec = $next; $out.Add($l); continue } - if ($sec -eq 'wsl2' -and $l -match '^[ \t]*([^=#; \t]+)[ \t]*=[ \t]*(.*?)[ \t]*$' -and $want.Contains($Matches[1])) { - $k = $Matches[1]; $v = $Matches[2] - if ($done[$k]) { $notes.Add(" [wsl2] $k=${v}: a repeated line, removed"); continue } + $lines = @(if ($null -ne $lines) { $lines }); $read = WslItems $lines; $n = $lines.Count + if ($n -gt 0 -and $lines[$n - 1] -eq '') { $n-- } # the last item after the last LF + $wanted = @{}; foreach ($k in $want.Keys) { $wanted[$k.ToLowerInvariant()] = $k } + $old = @{}; foreach ($k in $Experimental) { $old[$k.ToLowerInvariant()] = 1 } + $out = [Collections.Generic.List[string]]::new(); $notes = [Collections.Generic.List[string]]::new(); $end = [Collections.Generic.List[string]]::new() + $put = @{}; $drop = @{}; $before = @{}; $done = @{}; $sec = '' + function Missing { foreach ($k in $want.Keys) { if (-not $done[$k]) { "$k=$($want[$k])"; $done[$k] = 1; $notes.Add(" [wsl2] ${k}: (not set) -> $($want[$k])") } } } + function Drop([int]$a, [int]$b) { for ($l = $a; $l -le $b; $l++) { $drop[$l] = 1 } } + foreach ($e in $read.Items) { + if ($null -eq $e.Name) { if ($sec -ceq 'wsl2' -and $e.Sec -cne 'wsl2') { $before[$e.First] = @(Missing) }; $sec = $e.Sec; continue } + if ($null -eq $e.Value) { continue } + $lk = $e.Name.ToLowerInvariant() + if ($e.Sec -ceq 'wsl2' -and $wanted.ContainsKey($lk)) { + $k = $wanted[$lk] + if ($done[$k]) { $notes.Add(" [wsl2] $($e.Name)=$($e.Value): a repeated line, removed"); Drop $e.First $e.Last; continue } $done[$k] = 1 - if ($v -eq $want[$k]) { $out.Add($l) } else { $out.Add("$k=$($want[$k])"); $notes.Add(" [wsl2] ${k}: $v -> $($want[$k])") } - continue } - if ($sec -eq 'experimental' -and $l -match '^[ \t]*([^=#; \t]+)[ \t]*=[ \t]*(.*?)[ \t]*$' -and $Matches[1] -in $Experimental) { - $notes.Add(" [experimental] $($Matches[1])=$($Matches[2]): removed (the [wsl2] one is used)"); continue } - $out.Add($l) + if (SameValue $e.Value $want[$k]) { continue } + $put[$e.First] = "$k=$($want[$k])"; Drop ($e.First + 1) $e.Last; $notes.Add(" [wsl2] ${k}: $($e.Value) -> $($want[$k])") + } + elseif ($e.Sec -ceq 'experimental' -and $old.ContainsKey($lk)) { + $notes.Add(" [experimental] $($e.Name)=$($e.Value): removed (the [wsl2] one is used)"); Drop $e.First $e.Last } + } + if ($sec -ceq 'wsl2') { foreach ($x in @(Missing)) { $end.Add($x) } } + if ($done.Count -lt $want.Count) { $end.Add('[wsl2]'); foreach ($x in @(Missing)) { $end.Add($x) } } + for ($l = 0; $l -lt $n; $l++) { + if ($before.ContainsKey($l)) { foreach ($x in $before[$l]) { $out.Add($x) } } + if ($put.ContainsKey($l)) { $out.Add($put[$l]) } elseif (-not $drop[$l]) { $out.Add($lines[$l]) } + } + if ($end.Count) { if ($read.Open) { $out.Add('') }; $out.AddRange($end) } # an empty line ends a value that goes on + if ($end.Count -or $n -lt $lines.Count) { $out.Add('') } # the file ends in a LF + # the check: WSL reads the wanted keys, no old [experimental] ones, and the same values of every other key in order + $new = (WslItems $out).Items; $why = $null + $got = FirstKeys $new 'wsl2'; $gone = FirstKeys $new 'experimental' + foreach ($k in $want.Keys) { if (-not (SameValue $got[$k] $want[$k])) { $why = "WSL would not read [wsl2] $k=$($want[$k])" } } + foreach ($k in $Experimental) { if ($gone.ContainsKey($k)) { $why = "WSL would still read [experimental] $k" } } + $others = { param($items) foreach ($e in $items) { if ($null -ne $e.Name -and $null -ne $e.Value -and -not ($e.Sec -ceq 'wsl2' -and $wanted.ContainsKey($e.Name.ToLowerInvariant())) -and -not ($e.Sec -ceq 'experimental' -and $old.ContainsKey($e.Name.ToLowerInvariant()))) { "[$($e.Sec)] $($e.Name.ToLowerInvariant())=$($e.Value)" } } } + $a = @(& $others $read.Items); $b = @(& $others $new) + for ($x = 0; $x -lt [math]::Max($a.Count, $b.Count); $x++) { + if ($x -ge $a.Count -or $x -ge $b.Count -or -not [string]::Equals($a[$x], $b[$x], [StringComparison]::Ordinal)) { $why = "WSL would read other keys differently"; break } } - if ($sec -eq 'wsl2') { AddMissing } - if ($done.Count -lt $want.Count) { $out.Add('[wsl2]'); AddMissing } + if ($why) { throw ".wslconfig cannot be merged so that WSL reads it as the question says ($why); nothing was changed. Edit it by hand (a line ending in \ goes on to the next line), then run this again" } [pscustomobject]@{ Out = $out; Notes = $notes } } function Changes([string[]]$lines, $want) { (MergeLines $lines $want).Notes } # what Merge-WslConfig changes @@ -172,7 +261,7 @@ function Merge-WslConfig([string]$path, $want, [scriptblock]$before = {}) { # if ($merged.Notes.Count -eq 0) { return $false } # the same test as the question's & $before if (Test-Path -LiteralPath $path) { Copy-Item -LiteralPath $path "$path.bak-$(Get-Date -Format yyyyMMdd-HHmmss)" } - [IO.File]::WriteAllLines("$path.new", [string[]]$merged.Out, [Text.UTF8Encoding]::new($false)) + [IO.File]::WriteAllText("$path.new", [string]::Join([Environment]::NewLine, [string[]]$merged.Out), [Text.UTF8Encoding]::new($false)) Move-Item -LiteralPath "$path.new" $path -Force $true } @@ -319,7 +408,7 @@ sh /root/desk-bootstrap.sh --repo $Repo --sha $Sha$extra if ($LASTEXITCODE -ne 0) { $bad++ } $cfg = Wsl2Keys (Read-WslConfig $WslConfig); $old = SectionKeys (Read-WslConfig $WslConfig) 'experimental' foreach ($k in $want.Keys) { - if ($cfg[$k] -eq $want[$k]) { "PASS wslconfig [wsl2] $k=$($want[$k])" } else { "FAIL wslconfig [wsl2] no $k=$($want[$k])"; $bad++ } + if (SameValue $cfg[$k] $want[$k]) { "PASS wslconfig [wsl2] $k=$($want[$k])" } else { "FAIL wslconfig [wsl2] no $k=$($want[$k])"; $bad++ } } foreach ($k in $Experimental) { if ($old.ContainsKey($k)) { "FAIL wslconfig [experimental] $k=$($old[$k]) is still there"; $bad++ } } $t = Get-ScheduledTask $Task -ErrorAction SilentlyContinue