diff --git a/fork-kit/golive/README.md b/fork-kit/golive/README.md index f3804fb1c2283..efc39020a872c 100644 --- a/fork-kit/golive/README.md +++ b/fork-kit/golive/README.md @@ -329,8 +329,11 @@ start: the new launcher then idled and verify waited for minutes. and the kit must not take that probe for the launcher (it would free the home in the middle of `t3 update`). A rule of "held on every poll for 100 ms" could not tell frequent probes from the launcher when load stretched the polls (#546). For an older build the kit lets go just before the start, as before (its server would meet the - kit's lock and idle). The verify step journals how the lock was let go (`handover`: `launcher`, `timeout`, - `start failed`, `before the start ...`) and any marker renewal that failed (`renewFailed`). + kit's lock and idle). The kit lets go at its first look (every 20 ms) after the launcher's claim, well inside + the 250 ms its server waits for the owner lock (#534). The verify step journals how the lock was let go + (`handover`: `launcher`, `timeout`, `start failed`, `before the start ...`), for `launcher` the measured delay + (`handoverMs`: at most this many ms from the claim to the let-go; with `handoverLate` when it reached 250 ms, so + a server that idled on a very busy host is named), and any marker renewal that failed (`renewFailed`). - **Backstop, a foreign server.** An SSH client older than 0.1.10 reads neither, and a server started by hand ignores both. When the runtime record names a live process that the boot service did not start and that itself holds `userdata/server-owner.sqlite` (a stale record whose pid was reused holds nothing and is never named), for diff --git a/fork-kit/golive/lib/golive/handoff.py b/fork-kit/golive/lib/golive/handoff.py index 14f734791d395..31568a6b95ad8 100644 --- a/fork-kit/golive/lib/golive/handoff.py +++ b/fork-kit/golive/lib/golive/handoff.py @@ -58,6 +58,7 @@ # Seconds between two looks for the launcher. It holds the serving lock from #claimServing on, and its server then # waits 250 ms for the owner lock (serverOwnership.ts busy_timeout): the kit lets go well inside that. HANDOVER_POLL = 0.02 +SERVER_LOCK_WAIT_MS = 250 # serverOwnership.ts busy_timeout (contract.test.py): a later handover may leave it idle FOREIGN_GRACE = 10.0 # seconds a server not started by the boot service must stay before verify stops on it _now_ms = lambda: int(time.time() * 1000) # noqa: E731 (Date.now(), as upstream's reader compares) @@ -147,6 +148,9 @@ def __init__(self, home): # how the last starting() let go of the owner lock: 'launcher' (it held its serving lock), 'timeout' (none did # within HANDOVER_WAIT), 'start failed', 'before the start' (a build that takes no serving lock); None: none self.handover = None + # the last 'launcher' handover: at most this many ms between the launcher's claim of the serving lock and the + # kit letting go of the owner lock (from the start of the kit's last look that did not find it serving) + self.handover_ms = None self._stop = threading.Event() self._thread = None self._mutex = threading.Lock() @@ -184,8 +188,10 @@ def let_go(self): def starting(self, fn, version, service): """Run fn (it starts T3 `version`: a service start, or `t3 update`), and let go of the owner lock only once the new launcher holds its serving lock (or HANDOVER_WAIT after fn returned, or fn failed). service: reads - the boot service (adapter.service). Returns fn's result. Afterwards the kit holds no lock on the home.""" - self.handover = None + the boot service (adapter.service). Returns fn's result. Afterwards the kit holds no lock on the home. + A handover to the launcher measures its delay (handover_ms, an upper bound: the claim came after the start of + the last look that did not find the launcher serving, and the lock was let go before the clock is read).""" + self.handover, self.handover_ms = None, None if self.lock is not None and not takes_serving_lock(version): self.let_go() self.handover = 'before the start (a build that takes no serving lock)' @@ -196,8 +202,10 @@ def starting(self, fn, version, service): serving = os.path.join(self.home, 'runtime', SERVING_LOCK) def watch(): # HANDOVER_WAIT counts from fn's return (`t3 update` installs first, then restarts T3) + looked = time.monotonic() # the start of the last look that did not find the launcher serving try: while not done.is_set(): + began = time.monotonic() if lock_holders(serving): # someone holds it: the launcher, or an SSH launch's probe try: svc = service() @@ -205,7 +213,10 @@ def watch(): # HANDOVER_WAIT counts from fn's return (`t3 update` installs firs svc = {} if launcher_serving(self.home, svc): outcome['how'] = 'launcher' + self.let_go() + outcome['ms'] = (time.monotonic() - looked) * 1000 return + looked = began if returned and time.monotonic() >= returned[0] + HANDOVER_WAIT: outcome['how'] = 'timeout' return @@ -225,6 +236,8 @@ def watch(): # HANDOVER_WAIT counts from fn's return (`t3 update` installs firs finally: watcher.join() self.handover = outcome.get('how', 'start failed') + if 'ms' in outcome: + self.handover_ms = round(outcome['ms']) def close(self): """Stop renewing, let go of any lock. The marker stays to run out by itself (within 60 s).""" @@ -238,14 +251,21 @@ def close(self): def served(self): """The new server is verified: stop renewing and remove the marker if it is still the kit's. Returns the - facts for the verify step: 'marker' (what was done to it), 'handover' (how the owner lock was let go), and - 'renewFailed' when a renewal failed (the marker may have run out mid-stop; the owner lock still held). + facts for the verify step: 'marker' (what was done to it), 'handover' (how the owner lock was let go), + 'handoverMs' (handover_ms; with 'handoverLate' when it reached the server's 250 ms wait), and 'renewFailed' + when a renewal failed (the marker may have run out mid-stop; the owner lock still held). A newer marker renamed in between the read and the unlink would still be removed: a window of microseconds, the same as upstream's own DesktopBackendPool.releaseUpdateMarker.""" self.close() facts = {'marker': self._remove()} if self.handover is not None: facts['handover'] = self.handover + if self.handover_ms is not None: # the journal keeps how close the handover came to the server's wait + facts['handoverMs'] = self.handover_ms + if self.handover_ms >= SERVER_LOCK_WAIT_MS: + facts['handoverLate'] = (f'the kit let go up to {self.handover_ms} ms after the launcher took its ' + f'serving lock; its server waits {SERVER_LOCK_WAIT_MS} ms for the owner lock ' + 'and may have idled (the home is then free until its launcher tries again)') if self.failed: facts['renewFailed'] = f'{self.failed} time(s), last: {self.error}' return facts diff --git a/fork-kit/golive/tests/gap.test.py b/fork-kit/golive/tests/gap.test.py index aedddfdc16837..36bcbe2de2f7f 100644 --- a/fork-kit/golive/tests/gap.test.py +++ b/fork-kit/golive/tests/gap.test.py @@ -125,6 +125,23 @@ def home_owned(home, hold=0.0): PROBING_LIMIT = 300 # seconds: a probing phase that has not ended by then fails its test instead of hanging it +def kit_looks_for_the_launcher(test): + """[(monotonic time a look began, whether it found the launcher)]: each of the kit's handover looks that found + the serving lock held (handoff.launcher_serving), recorded as it ran.""" + from golive import handoff + looks, real = [], handoff.launcher_serving + + def spy(home, svc): + began = time.monotonic() + found = real(home, svc) + looks.append((began, found)) + return found + patcher = mock.patch.object(handoff, 'launcher_serving', spy) + patcher.start() + test.addCleanup(patcher.stop) + return looks + + def kit_meets_probes(test): """An Event set once the kit's handover poll has found the serving lock held by another process than this one (an SSH probe: this process is the fake launcher), so a probing phase surely tested what it is about.""" @@ -681,7 +698,8 @@ def test_a_probe_loop_on_the_serving_lock_is_not_the_launcher(self): g = self.h.Gap(self.m.home) g.adopt(coldcopy.hold_owner_lock(userdata, 0)) met = kit_meets_probes(self) - stop, probing, probes, free = threading.Event(), threading.Event(), [], [] + looks = kit_looks_for_the_launcher(self) + stop, probing, probes, free, claimed = threading.Event(), threading.Event(), [], [], [] def ssh_probes(): try: @@ -701,21 +719,84 @@ def start(): # `t3 update`: it installs while launches probe, then its launcher t.join() launcher = sqlite3.connect(serving, isolation_level=None, timeout=1) launcher.execute('BEGIN EXCLUSIVE') - claimed = time.monotonic() - for _ in range(300): - if not held(owner): - break + claimed.append(time.monotonic()) + end = claimed[0] + PROBING_LIMIT + while held(owner) and time.monotonic() < end: time.sleep(0.005) - free.append(time.monotonic() - claimed) + free.append(not held(owner)) launcher.close() t = threading.Thread(target=ssh_probes, daemon=True) t.start() g.starting(start, NEW, self.service) self.assertEqual(len(free), 1, 'the kit let go of the owner lock while only SSH probes held the serving lock') + self.assertTrue(free[0], 'the kit kept the owner lock after the launcher claimed the serving lock') self.assertGreaterEqual(len(probes), PROBES) self.assertTrue(met.is_set(), 'the kit never met an SSH probe of the serving lock') - self.assertLess(free[0], 0.25) # within its server's 250 ms wait for the owner lock + # #534: let go at the kit's first look after the claim, counted in looks, not in wall-clock time (a slow + # runner stretches both alike): every look that began after the claim found the launcher, and there was one + self.assertEqual([found for at, found in looks if at >= claimed[0]], [True]) self.assertEqual(g.handover, 'launcher') + self.assertIsInstance(g.handover_ms, int) + + def test_the_handover_delay_is_measured_and_journalled(self): + """#534: the verify step journals how long the kit kept the owner lock after the launcher's claim (an upper + bound), and names a handover that reached the 250 ms its server waits for the owner lock.""" + userdata = self.m.path('home/userdata') + os.makedirs(userdata) + serving = self.m.path('home/runtime/service-serving.lock') + owner = os.path.join(userdata, 'server-owner.sqlite') + claims = [] + + def launcher(): # claims the serving lock and keeps it until the kit let go of the owner lock + c = sqlite3.connect(serving, isolation_level=None, check_same_thread=False) + c.execute('BEGIN EXCLUSIVE') + claims.append(c) + # (seconds a look that found the launcher takes, the server's wait in ms): the lateness is fixed by the wait, + # never by how fast this runner is: 0.3 s is surely past 250 ms, and no runner reaches a wait of 10**9 ms + for slow, wait, late in ((0.0, 10**9, False), (0.3, self.h.SERVER_LOCK_WAIT_MS, True)): + real = self.h.launcher_serving + + def seen(home, svc, slow=slow, real=real): # a look stretched by load, after the claim + found = real(home, svc) + if found: + time.sleep(slow) + return found + g = self.h.open_gap(self.m.home) + g.adopt(coldcopy.hold_owner_lock(userdata, 0)) + with mock.patch.object(self.h, 'launcher_serving', seen), \ + mock.patch.object(self.h, 'SERVER_LOCK_WAIT_MS', wait): + g.starting(launcher, NEW, self.service) + self.assertFalse(held(owner)) + facts = self.h.served(self.m.home) + self.assertEqual(facts['handover'], 'launcher') + self.assertIsInstance(facts['handoverMs'], int) + self.assertGreaterEqual(facts['handoverMs'], slow * 1000) # an upper bound: it covers the slow look + self.assertEqual('handoverLate' in facts, late, facts) + if late: + self.assertIn(f'up to {facts["handoverMs"]} ms after the launcher took its serving lock', + facts['handoverLate']) + claims.pop().close() + g = self.h.open_gap(self.m.home) # no launcher: nothing to measure + g.adopt(coldcopy.hold_owner_lock(userdata, 0)) + with mock.patch.object(self.h, 'HANDOVER_WAIT', 0.05): + g.starting(lambda: None, NEW, self.service) + self.assertIsNone(g.handover_ms) # a run again: the last handover's delay is not carried over + facts = self.h.served(self.m.home) + self.assertEqual(facts['handover'], 'timeout') + self.assertNotIn('handoverMs', facts) + g = self.h.open_gap(self.m.home) # a start that failed half-way: nothing to measure + g.adopt(coldcopy.hold_owner_lock(userdata, 0)) + g.handover_ms = 7 # left by an earlier start of this gap + + def fails(): + raise OSError('the start failed') + with self.assertRaises(OSError): + g.starting(fails, NEW, self.service) + self.assertFalse(held(owner)) + facts = self.h.served(self.m.home) + self.assertEqual(facts['handover'], 'start failed') + self.assertNotIn('handoverMs', facts) + self.assertEqual(self.h.served(self.m.home), {'marker': None}) # no gap open: nothing measured def test_only_the_service_s_main_process_holding_the_serving_lock_is_the_launcher(self): """However long another process holds the serving lock, the kit keeps the owner lock; it lets go at once