diff --git a/.github/fork-fleet.md b/.github/fork-fleet.md index 613bee1dcc0e1..2229047ea63b9 100644 --- a/.github/fork-fleet.md +++ b/.github/fork-fleet.md @@ -909,7 +909,9 @@ the global ones of the whole tar, at most 1 MiB; sparse members are refused, so make a seed without `--sparse`; a hard link must name a file unpacked before it, and a link whose path is already there, or that cannot be made, is refused, never copied from its target, which tarfile would look for in the rest of the -stream), checked against the trailer (its size +stream); only a block of zeros ends the tar and only zeros may follow it (tarfile +would end it quietly at a bad header after the first, and a short seed would be +swapped in), checked against the trailer (its size and date must be the list's too), checked again for links that lead out of it, made readable by every job (a cache's 0600 files too), and swapped in by a rename; ssh's stderr and the list are read with a limit. A seed that fails in @@ -922,7 +924,11 @@ link too slow for it does not spend every hour on it; a fetch that kills the run counts as torn; the times are approximate: the timer's 10 minute random delay and a 5 minute margin can make a wait up to an hour longer); another copy on the host is tried at once, but only once per wait (the try counts, so a -host that remakes the seed every hour still meets the back-off). One torn on +host that remakes the seed every hour still meets the back-off: the next wait +doubles, but runs from the last try that waited, not from the early try, so the +next normal try comes at most one step later, never past 24 hours after the +last try that waited). So a fix made right after another new copy waits for +that doubled wait, up to 24 hours. One torn on every try for more than a day makes the run exit 1, so `desk-install check` shows FAIL until it is fetched (`/var/lib/fork-fleet/seeds/.torn.json` names it and why); so does a run that cannot write that file (a full disk). A host that gives no list (down, @@ -930,7 +936,9 @@ an older kit, a half-done install) means no new seeds: each one here stays until its age drops it, the torn record is left as it is, and the run exits 0 (a host down for good shows in `desk-install check` only as seeds that age out). A desktop whose own fleet config is broken or lacks its pool, or whose Python's -tarfile lacks the private parts the limits above rely on, fetches nothing and +tarfile lacks the private parts the limits above rely on (or is not proven to +refuse: at every run a small tar of each sparse form, and one of a header over +the limit, must each be refused by its own check), fetches nothing and exits 2 (`desk-install check` shows the seed service's last result), but still drops each seed past the `max_age_hours` it was fetched with. Jobs see `/var/lib/fork-fleet/seeds/NAME` read-only (the launcher's `ALLOW_PATHS`; diff --git a/.github/scripts/fork-fleet-desk-seed.py b/.github/scripts/fork-fleet-desk-seed.py index 3ac5a87db1ca1..56a625bc23054 100644 --- a/.github/scripts/fork-fleet-desk-seed.py +++ b/.github/scripts/fork-fleet-desk-seed.py @@ -32,14 +32,19 @@ a new one under every_hours after the last nothing yet a torn copy (the stream cut or past FETCH_WAIT, a trailer that does not match it or the list, more than max_mb, a tar member that would leave its folder, is a device, too deep or too long, has a size below 0 (or one that is - not a file has a size), a link whose path is already there or that cannot be made, or any other error) + not a file has a size), a link whose path is already there or that cannot be made, a bad header anywhere (only a + block of zeros ends the tar) or anything but zeros after the tar's end, or any other error) the copy is removed, the old seed stays, the other seeds go on, old versions are still pruned; tried again after about 1, 2, 4, 8, 16, then 24 hours (TornLog; the hourly timer's RandomizedDelaySec and a 5 min margin make each wait up to an hour longer); another copy on the host skips the wait once per back-off step (the try still counts, so a - host that remakes it hourly still backs off); torn on every try for - over TORN_DAY: exit 1 + host that remakes it hourly still backs off: the next wait doubles, + but runs from the last try that waited, not from the early try, so + it ends at most one step later, never past 24 h after that try), so a + fix made right after another new copy waits for that doubled wait + (up to 24 h); + torn on every try for over TORN_DAY: exit 1 .torn.json cannot be written (a full disk) the run goes on, counts its tries in memory, and exits 1 a fetch that kills the run (MemoryMax, TimeoutStartSec) counted as torn before it began; the next run prunes first @@ -68,7 +73,7 @@ MIN_RATE bytes a second), or the file cannot be read again after it was sent: the pool's lock is then let go. Standard library only. """ -import argparse, fcntl, hashlib, importlib.util, json, math, os, re, signal, stat, subprocess, sys +import argparse, fcntl, hashlib, importlib.util, io, json, math, os, re, signal, stat, subprocess, sys import tarfile, threading, time sys.dont_write_bytecode = True @@ -357,13 +362,26 @@ def whole(x): PRIVATE = ('_proc_member', '_proc_sparse', '_proc_gnusparse_00', '_proc_gnusparse_01', '_proc_gnusparse_10', '_find_link_target') # the private tarfile parts Info's and Tar's limits rely on: without one, no fetch +SPARSE = {'0.0': {'GNU.sparse.size': '0', 'GNU.sparse.offset': '0', 'GNU.sparse.numbytes': '0'}, + '0.1': {'GNU.sparse.map': '0,0'}, '1.0': {'GNU.sparse.major': '1', 'GNU.sparse.minor': '0'}} # pax forms class Info(tarfile.TarInfo): """A tar header read with limits tarfile has not: it reads an extended header (pax, or a GNU long name) into memory whole and its parsed form takes many times its size, and a sparse member's map can grow without end. So the extended headers before one member may hold at most HEADER_MOST bytes together, the global pax headers of the - whole tar too, and a sparse member is refused (make a seed without --sparse). Torn before tarfile reads one.""" + whole tar too, and a sparse member is refused (make a seed without --sparse). Torn before tarfile reads one. + Only a block of zeros ends the tar: tarfile ends it quietly at any other bad header after the first (a pax size + below 0, a bad checksum), and a short seed would be swapped in (fromtarfile).""" + + @classmethod + def fromtarfile(cls, tarfile_): + try: + return super().fromtarfile(tarfile_) + except tarfile.EOFHeaderError: # a block of zeros: the tar's end + raise + except tarfile.HeaderError as e: # tarfile.next() would end the tar here + raise Torn(f'a bad tar header at byte {tarfile_.offset} ({e}), not the tar\'s end') from None def _proc_member(self, tarfile_): if self.size < 0: # tarfile refuses it only from the CVE-2025-8194 fix on; here it would lower `used` @@ -383,7 +401,11 @@ def _proc_member(self, tarfile_): def sparse_refused(self, member=None, *_): raise Torn(f'{(member or self).name[:80]!r} is a sparse member (make the seed without --sparse)') - _proc_gnusparse_00 = _proc_gnusparse_01 = _proc_gnusparse_10 = sparse_refused # pax sparse forms + + +for _k in dir(tarfile.TarInfo): # the pax sparse forms: 00, 01, 10 and any a later tarfile adds by that name + if _k.startswith('_proc_gnusparse_'): + setattr(Info, _k, Info.sparse_refused) class Tar(tarfile.TarFile): @@ -394,6 +416,45 @@ def _find_link_target(self, tarinfo): raise Torn(f'{tarinfo.name[:80]!r} is a link that cannot be made here (its target is not copied instead)') +def probe(form): + """The small tar sparse_read reads for FORM: a pax sparse form (SPARSE), an old-GNU sparse member ('old GNU', its + refusal is in Info._proc_member) or an extended header over HEADER_MOST ('headers': only its header, as Info + refuses it before its data is read).""" + info = tarfile.TarInfo('s') + if form == 'headers': + info.type, info.size = tarfile.XHDTYPE, HEADER_MOST + 1 + return info.tobuf(tarfile.USTAR_FORMAT, 'utf-8', 'surrogateescape') + info.size = 512 + if form == 'old GNU': + info.type, fmt = tarfile.GNUTYPE_SPARSE, tarfile.GNU_FORMAT + else: + info.pax_headers, fmt = SPARSE[form], tarfile.PAX_FORMAT + return info.tobuf(fmt, 'utf-8', 'surrogateescape') + b'0\n'.ljust(512, b'\0') + bytes(1024) # 1.0: its map + + +PROBES = {**{form: 'is a sparse member' for form in SPARSE}, 'old GNU': 'is a sparse member', + 'headers': 'headers hold more than'} # each probe's form: the refusal it must be torn by + + +def sparse_read(): + """The probes (PROBES) this Python's tarfile reads past Info's limits: none while tarfile still calls the + handlers and the _proc_member Info refuses them in (PRIVATE proves only that they are there). Each probe must be + torn by its own refusal; anything else is named (alone: read as a sparse member; with why: read whole, torn by + another limit, another error, or the probe could not even be made), as it is not proven refused.""" + read = [] + for form, refusal in PROBES.items(): + try: + with Tar.open(fileobj=io.BytesIO(probe(form)), mode='r|', tarinfo=Info) as t: + got = list(iter(t.next, None)) + read.append(form if any(m.sparse is not None for m in got) else f'{form}: read whole') + except Torn as e: + if refusal not in str(e): # torn, but not by the refusal this form proves + read.append(f'{form}: {str(e)[:80]}') + except Exception as e: # tarfile's own error, or one this code did not foresee: not known to be refused + read.append(f'{form}: {e.__class__.__name__}: {str(e)[:80]}') + return read + + def unpack(t, data, most): """The members of the open stream T (a Tar) into DATA, one at a time: tarfile keeps every member it has read in `members` (and extractall each folder besides), so they are let go as they are made; only `guarded`'s count @@ -493,8 +554,9 @@ def fetch(agent, conf, item, root, now, wait=FETCH_WAIT): got = Summed(p.stdout, most) with Tar.open(fileobj=got, mode='r|', tarinfo=Info) as t: unpack(t, os.path.join(new, 'data'), most) - while got.read(PIECE): # the tar's end blocks and padding count in the trailer's size and sha256 - pass + while rest := t.fileobj.read(PIECE): # the end blocks and padding (in the trailer's size and sha256), + if rest.strip(b'\0'): # read through tarfile's buffer: zeros only + raise Torn(f'bytes after the tar\'s end (at byte {t.offset}): not a whole tar') code = p.wait(timeout=30) finally: timer.cancel() @@ -604,14 +666,18 @@ def own_seeds(conf, config): def run(agent, conf, root=SEEDS, now=time.time, say=print, config=CONFIG): """One run (the module's docstring). A seed that fails in any way, a deep tree included, is said and the run goes on with the next; old versions are pruned in every run. When it may fetch nothing (no data filter or no - private tarfile part in PRIVATE, its own fleet config broken or without its pool) it still drops each seed past - its age, by its label's max_age_hours, and then says why (agent.Bad: exit 2, the unit's last result shows it).""" + private tarfile part in PRIVATE, a probe of sparse_read not torn by its own refusal, its own fleet config broken or + without its pool) it still drops each seed past its age, by its label's max_age_hours, and then says why + (agent.Bad: exit 2, the unit's last result shows it).""" cannot, own = None, None if not hasattr(tarfile, 'data_filter'): # releases with known filter bypasses pass this: settle() checks again cannot = 'this Python has no tarfile data filter (3.12, or 3.8.17 and later)' elif gone := [k for k in PRIVATE if not callable(getattr(tarfile.TarFile if k == '_find_link_target' else tarfile.TarInfo, k, None))]: cannot = f'this Python\'s tarfile has no {", ".join(gone)} (the limits of Info and Tar need them)' + elif read := sparse_read(): + cannot = (f'this Python\'s tarfile is not proven to refuse sparse members and long headers ' + f'({"; ".join(read)})') try: own = own_seeds(conf, config) except Torn as e: @@ -667,10 +733,12 @@ class TornLog: and cleared when the seed is fetched or the same as the host's. A seed torn TRIES times in a row is tried again only min(2 ** (TRIES - 1), 24) hours after the last try (a link too slow for it, a host that cuts it: not every hour), or at once when the host offers another copy (`copy`: its [bytes, made] when last tried), but that only - once per back-off step (`early`: the last try was such a one); one torn on every try since more than TORN_DAY - is `late`. Only seeds in the host's list are kept, and only in a run that read the list. A missing or unreadable - file is an empty record; one that cannot be written (a full disk) is kept in memory for this run and said in - `unsaved`.""" + once per back-off step (`early`: the last try was such a one). Such a try counts (the next wait doubles) but + keeps `last`, the try the wait runs from: the next normal try comes at most one step later than it would have + (h 5, not 3, after an early try at h 1.5), and only at the 24 h cap not at all; never from the early try. So a + fix made right after another new copy waits for that doubled wait (up to 24 h). One torn on every try since more than TORN_DAY is `late`. Only seeds + in the host's list are kept, and only in a run that read the list. A missing or unreadable file is an empty + record; one that cannot be written (a full disk) is kept in memory for this run and said in `unsaved`.""" def __init__(self, root, now): self.path, self.now, self.seeds = root and os.path.join(root, '.torn.json'), now, {} @@ -706,11 +774,12 @@ def due(self, name, item=None): def trying(self, name, item=None): """Counted as torn before the fetch, saved: `why` says so until the fetch ends; `early` when it is tried - before its back-off is over (a new copy).""" + before its back-off is over (a new copy): its `last` stays, so the next wait, doubled, runs from the last + try that waited, not from this one (it ends one step later than it would have, but for the 24 h cap).""" early = self.left(name) is not None r = self.seeds.get(name) or {'first': int(self.now()), 'tries': 0} - self.seeds[name] = {**r, 'last': int(self.now()), 'tries': r['tries'] + 1, 'early': early, - 'why': 'the run stopped during the fetch (its memory or time limit?)'} + self.seeds[name] = {**r, 'last': r['last'] if early else int(self.now()), 'tries': r['tries'] + 1, + 'early': early, 'why': 'the run stopped during the fetch (its memory or time limit?)'} if item: self.seeds[name]['copy'] = [item['bytes'], item['made']] self.save() @@ -806,8 +875,7 @@ def take(agent, conf, item, root, now, say, torn=None, end=None): except Exception as e: # any: a torn stream, a full disk, a tar this Python cannot unpack why = f'{e.__class__.__name__}: {str(e)[:300]}' torn.torn(name, why) - r = torn.seeds[name] - hours = min(2 ** (min(r['tries'], 6) - 1), 24) + r, hours = torn.seeds[name], torn.due(name) or 1 # from `last`: an early try keeps it say(f'seed {name}: keeps {"the old one" if label else "none"}: {why}; tried again in about {hours} h' + ('' if r['early'] else ' (at once if the host makes another copy before then)')) return False diff --git a/.github/scripts/fork-fleet-desk-seed.test.py b/.github/scripts/fork-fleet-desk-seed.test.py index 4ff8ee858196e..86c5c07fac254 100644 --- a/.github/scripts/fork-fleet-desk-seed.test.py +++ b/.github/scripts/fork-fleet-desk-seed.test.py @@ -3,7 +3,7 @@ offer in-process, the desktop's run end to end through a fake ssh that runs the host's side, and the forced command's divert in fork-fleet-desk.py. Temp folders only; no root, no ssh, no network. Run: python3 -I -B . Linux only.""" import contextlib, fcntl, hashlib, importlib.util, io, json, os, shutil, stat, subprocess, sys, tarfile, tempfile, threading -import errno, signal, time, unittest +import errno, re, signal, time, unittest from unittest import mock sys.dont_write_bytecode = True @@ -819,11 +819,12 @@ def broken(agent_, conf, item, root, now, **kw): self.assertIsNone(self.files('m2')) self.assertEqual(os.listdir(os.path.join(self.root, '.new')), ['.lock']) - def raw(self, name, blob): - """FILES/POOL/NAME.tar holding BLOB (tar blocks made by hand), newer than any made before.""" + def raw(self, name, blob, end=b'\0' * 1024): + """FILES/POOL/NAME.tar holding BLOB (tar blocks made by hand) and END (the end blocks), newer than any made + before.""" tmp = os.path.join(self.t, 'files', POOL, f'.{name}.tmp') with open(tmp, 'wb') as f: - f.write(blob + b'\0' * 1024) + f.write(blob + end) os.chmod(tmp, 0o644) os.utime(tmp, (time.time() + 5,) * 2) os.replace(tmp, self.tar(name)) @@ -1078,6 +1079,91 @@ def test_a_python_without_the_private_tarfile_parts_fetches_nothing(self): # l self.assertEqual(self.run_seed(), 0) self.assertEqual(self.files('m2'), {'a': b'one'}) + def test_a_sparse_form_a_later_tarfile_adds_is_refused_too(self): # #258 item 3 (lens237sr5 nit 2) + with mock.patch.object(tarfile.TarInfo, '_proc_gnusparse_11', lambda self_, next_, pax: None, create=True): + later = load('fork_fleet_desk_seed_later', 'fork-fleet-desk-seed.py') + with self.assertRaisesRegex(later.Torn, 'sparse member'): + later.Info('x')._proc_gnusparse_11(tarfile.TarInfo('s'), {}) + + def test_a_python_that_reads_a_sparse_member_past_info_fetches_nothing(self): # #258 item 3: tarfile still + self.make('m2', {'a': b'one'}) # calls what Info refuses in + self.assertEqual(seed.sparse_read(), []) + for k, form in (('_proc_gnusparse_00', '0.0'), ('_proc_gnusparse_01', '0.1'), ('_proc_gnusparse_10', '1.0')): + with self.subTest(form), mock.patch.object(seed.Info, k, getattr(tarfile.TarInfo, k)): + with self.assertRaisesRegex(agent.Bad, f'long headers \\({form}\\)'): + self.run_seed() + self.assertEqual(self.fetches(), []) + self.assertEqual(self.run_seed(), 0) + self.assertEqual(self.files('m2'), {'a': b'one'}) + + def test_only_its_own_refusal_proves_a_probe(self): # PR #729 lens A minors 1, 3 and lens B minor 3 + def raises(e): + return mock.Mock(side_effect=e) + cases = { # before: the first two counted as refused, and no probe needed _proc_member to be called + '0.0: another limit': (seed.Info, '_proc_gnusparse_00', raises(seed.Torn('another limit'))), + '0.1: ReadError: refused by tarfile': (seed.Info, '_proc_gnusparse_01', raises(tarfile.ReadError( + 'refused by tarfile'))), + 'old GNU; headers: ': (seed.Info, '_proc_member', tarfile.TarInfo._proc_member), # a later tarfile that + } # stops calling it: Info's limits go + self.make('m2', {'a': b'one'}) + for want, (owner, k, new) in cases.items(): + with self.subTest(want), mock.patch.object(owner, k, new): + self.assertIn(want.split('; ')[0], seed.sparse_read()) + with self.assertRaisesRegex(agent.Bad, re.escape(want)): + self.run_seed() + self.assertEqual(self.fetches(), []) + self.assertEqual(seed.sparse_read(), []) + self.assertEqual(self.run_seed(), 0) + self.assertEqual(self.files('m2'), {'a': b'one'}) + + def test_a_probe_that_cannot_be_made_still_drops_an_old_seed_and_is_exit_2(self): # PR #729 lens A 2, B 2 + self.make('m2', {'a': b'one'}) + self.run_seed() + fetched = len(self.fetches()) + with mock.patch.object(tarfile.TarInfo, 'tobuf', mock.Mock(side_effect=ValueError('no GNU.sparse keys'))): + self.assertEqual(len(seed.sparse_read()), len(seed.PROBES)) # each named, none raised + with self.assertRaisesRegex(agent.Bad, 'ValueError: no GNU.sparse keys.*no seeds fetched'): + self.run_seed(now=lambda: time.time() + 999 * 3600) # before: the ValueError itself, exit 1 + self.assertEqual(self.files('m2'), {'a': b'one'}) # under its max_age_hours: kept + with self.assertRaises(agent.Bad): + self.run_seed(now=lambda: time.time() + 1001 * 3600) + self.assertIsNone(self.files('m2')) # past it: dropped, as the docs say + self.assertTrue(any('older than 1000 h' in x for x in self.said), self.said) + self.assertEqual(len(self.fetches()), fetched) + + def test_only_a_block_of_zeros_ends_the_tar_and_only_zeros_follow(self): # #258 item 4 (lens237sr5 nit 3) + bad = bytearray(tarfile.TarInfo('b').tobuf(format=tarfile.GNU_FORMAT)) + bad[148:156] = b'0000001\0' # a checksum that does not match + neg = tarfile.TarInfo('pax') + neg.type = tarfile.XHDTYPE + first = self.block(tarfile.TarInfo('a'), b'two') + cases = { # before #258 each swapped in a seed holding only a (or a and b): the trailer matched + 'a bad checksum after the first header, then end blocks': (first + bytes(bad), b'\0' * 1024), + 'a bad header as the last block': (first + bytes(bad), b''), + 'a bad checksum, then more members': (first + bytes(bad) + self.block(tarfile.TarInfo('c'), b'x'), + b'\0' * 1024), + 'a pax size below 0 after the first header': ( + first + self.block(neg, self.pax(size=-1)) + self.block(tarfile.TarInfo('b'), b'x'), b'\0' * 1024), + 'no end blocks (the stream ends after a member)': (first, b''), + 'bytes after the end blocks': (first, b'\0' * 1024 + b'more'), + 'bytes after the padding': (first, b'\0' * 30000 + b'\1'), + } + self.make('m2', {'a': b'one'}) + self.run_seed() + for i, (what, (blob, end)) in enumerate(cases.items(), 1): + with self.subTest(what): + self.raw('m2', blob, end) + self.run_seed(now=lambda i=i: time.time() + i * 7 * 3600) + self.assertEqual(self.files('m2'), {'a': b'one'}) + self.assertTrue(any('m2: keeps the old one' in x and ('bad tar header' in x or 'after the tar' in x + or 'below 0' in x or 'the size -1' in x) + for x in self.said), self.said) # 'the size -1': guarded, on 3.13.5 + with contextlib.suppress(FileNotFoundError): # each case tried at once, not after a wait + os.unlink(os.path.join(self.root, '.torn.json')) + self.raw('m2', first, b'\0' * 30000) # end blocks and any zero padding: a whole tar + self.run_seed(now=lambda: time.time() + 99 * 3600) + self.assertEqual(self.files('m2'), {'a': b'two'}) + def test_a_host_that_remakes_each_hour_never_meets_the_back_off(self): # lens237cr4 + sr4: the MUST start, tried = int(time.time()) - 13 * 3600, [] @@ -1090,12 +1176,54 @@ def torn(agent_, conf, item, root, now, **kw): self.make('m2', {'a': b'%d' % h}, mtime=start + h * 3600 - 60) self.run_seed(now=lambda h=h: start + h * 3600) said += self.said - self.assertEqual(tried, [0, 1, 2, 6, 7]) # a new copy skips a wait once per step; before round 5: 0..11 - fails = [x for x in said if 'keeps' in x] + self.assertEqual(tried, [0, 1, 2, 5, 6]) # a new copy skips a wait once per step; before round 5: 0..11; + fails = [x for x in said if 'keeps' in x] # before #258: [0, 1, 2, 6, 7] (each early try moved the step) self.assertEqual(len(fails), 5, said) - self.assertTrue(all('tried again in about' in x for x in fails), fails) + self.assertEqual([int(x.split('tried again in about ')[1].split(' h')[0]) for x in fails], [1, 2, 3, 8, 15]) self.assertEqual(['at once if' in x for x in fails], [True, True, False, True, False]) + def test_an_early_try_keeps_the_wait_running_from_the_last_try_that_waited(self): # #258 item 2 (lens237cr5 nit 2) + start, tried, saved, said = int(time.time()) - 12 * 3600, [], [], {} + + def torn(agent_, conf, item, root, now, **kw): + tried.append((now() - start) / 3600) + with open(os.path.join(root, '.torn.json')) as f: # saved before the fetch: what a run killed in it leaves + saved.append(json.load(f)['m2']) + raise seed.Torn('cut off') + with mock.patch.object(seed, 'fetch', torn): + for h, copy in ((0, 0), (1, 1), (1.5, 2), (2, 3), (4.5, 3), (5, 3)): # copy: the host's, by its date + self.make('m2', {'a': b'x'}, mtime=start - 60 + copy) + self.run_seed(now=lambda h=h: start + h * 3600) + said[h] = self.said + self.assertEqual(tried, [0, 1, 1.5, 5]) # before #258: [0, 1, 1.5] (the early try moved the step to h 5.4) + self.assertEqual([(r['last'] - start, r['tries'], r['early']) for r in saved], + [(0, 1, False), (3600, 2, False), (3600, 3, True), (5 * 3600, 4, False)]) + self.assertTrue(any('torn 3 times' in x and 'about 3 h' in x for x in said[2]), said[2]) # before: 4 h + # the early try at h 1.5 still doubled the wait: the next normal try is at h 5 (h 1 + 4), not h 3; at the + # 24 h cap an early try moves nothing + log = seed.TornLog(None, lambda: 10 * 3600) + log.seeds['m2'] = {'first': 0, 'last': 0, 'tries': 6, 'copy': [1, 1]} + before = log.left('m2') + self.assertIsNone(log.due('m2', {'bytes': 2, 'made': 2})) # another copy: tried early + log.trying('m2', {'bytes': 2, 'made': 2}) + self.assertEqual((log.seeds['m2']['last'], log.seeds['m2']['tries'], log.left('m2')), (0, 7, before)) + + def test_an_old_record_whose_early_try_moved_last_still_waits(self): # #258: a record #237 wrote + start, tried = int(time.time()) - 12 * 3600, [] + self.make('m2', {'a': b'x'}, mtime=start - 60) + os.makedirs(self.root) + with open(os.path.join(self.root, '.torn.json'), 'w') as f: # its early try at h 1.5 moved `last` there + json.dump({'m2': {'first': start, 'last': start + 5400, 'tries': 3, 'early': True, 'why': 'cut off', + 'copy': [1, 1]}}, f) + + def torn(agent_, conf, item, root, now, **kw): + tried.append((now() - start) / 3600) + raise seed.Torn('cut off') + with mock.patch.object(seed, 'fetch', torn): + for h in (5, 6): + self.run_seed(now=lambda h=h: start + h * 3600) + self.assertEqual(tried, [6]) # its step runs from h 1.5 (to h 5.4), the new copy already had its early try + def test_one_lost_list_does_not_reset_the_torn_record(self): # lens237cr3 self.make('m2', {'a': b'one'}) start, codes, real, tried = int(time.time()), {}, seed.listing, [] diff --git a/fork-kit/desk/README.md b/fork-kit/desk/README.md index f26bdfeb6615c..da8db291a6a15 100644 --- a/fork-kit/desk/README.md +++ b/fork-kit/desk/README.md @@ -91,7 +91,8 @@ The seed timer (`t3code-fleet-seed.timer`) fetches the pool's warm caches, if it `/var/lib/fork-fleet/seeds`, which jobs see read-only ([fork-fleet.md](../../.github/fork-fleet.md), "Seeds"). A torn seed is tried again after about 1, 2, 4, 8, 16, then 24 hours (up to an hour later: the timer's random delay), or at once when the host offers another copy, but that only once per wait (a host that remakes the seed every hour -still meets the back-off); one torn on every try for more than a day makes each run that reads the host's list exit +still meets the back-off: the next wait doubles, but runs from the last try that waited, not from the early try), so +a fix made right after another new copy waits for that doubled wait, up to 24 hours; one torn on every try for more than a day makes each run that reads the host's list exit 1, so `check` shows the seed service's last run as failed (a run that cannot write the record, a full disk, exits 1 too). `/var/lib/fork-fleet/seeds/.torn.json` names it and the host's reason (jobs can read that file too); a run without the host's list leaves it as it is and exits 0, so a host that stays down shows in `check` only