dashboard: a slow data run can't hold the page's refreshes

docs/devteam/dashboard/index.html

@@ -4933,24 +4933,30 @@ <h4>${linkify(k.statement)}</h4>
   loading = loadNow();
   try { return await loading; } finally { loading = null; }
 }
+// A refresh gives up after READ_GIVE_UP_MS, so one read the server never answers can't hold the
+// one-at-a-time guard: the next poll tries again. The first read waits as long as the server does.
+const READ_GIVE_UP_MS = 30000;
 async function loadNow() {
   let d;
+  const stop = new AbortController();
+  const late = DATA ? setTimeout(() => stop.abort(), READ_GIVE_UP_MS) : null;
   try {
-    const r = await fetch('/api/data');
+    const r = await fetch('/api/data', { signal: stop.signal });
+    // the body first: a read given up on mid-body leaves none of what its headers say applied
+    const body = await r.text();
     const pv = r.headers.get('x-devteam-page');
     if (pv) { if (!PAGE_VERSION) PAGE_VERSION = pv; else if (pv !== PAGE_VERSION) $('updated').hidden = false; }
     // what the cloud box serves; absent on a laptop, and a header that doesn't parse counts as absent
     PUBLIC = null;
     try { if (r.headers.get('x-devteam-public')) PUBLIC = JSON.parse(r.headers.get('x-devteam-public')); } catch {}
     PREVIEWS = r.headers.get('x-devteam-previews') === '1';
     STALE = r.headers.get('x-devteam-stale') === 'true';
-    const body = await r.text();
     try { d = JSON.parse(body); } catch { throw new Error(r.ok ? 'the server sent something that is not JSON' : `${r.status} ${body.slice(0, 400)}`); }
     if (!r.ok) throw new Error(d.error || `HTTP ${r.status}`);
   } catch (e) {
     $('err').textContent = 'could not load (see the top of the page)';
-    return showFailure(e.message || String(e));
-  }
+    return showFailure(stop.signal.aborted ? `the server did not answer in ${READ_GIVE_UP_MS / 1000} s; trying again` : e.message || String(e));
+  } finally { clearTimeout(late); }
   $('err').textContent = '';
   $('banner').hidden = true;
   $('loading').hidden = true;

docs/devteam/scripts/devteam.mjs

@@ -10760,6 +10760,9 @@ const RESTART_MAX = 3;
 const RESTART_CHILD_MS = 100000;
 const RESTART_TICK_MS = 110000;
 const RESTART_MIN_MS = 30000;
+// How long a dashboard read waits on a dashboard-data run before it has the last good data instead
+// (0 is no wait at all; anything that isn't a number is the default).
+const DATA_WAIT_MS = ((n) => (process.env.DEVTEAM_DATA_WAIT_MS?.trim() && Number.isFinite(n) && n >= 0 ? n : 20000))(Number(process.env.DEVTEAM_DATA_WAIT_MS));
 // makeSession --if-gone found the window there after all (it never types into one): not a failure.
 const IF_GONE_LEFT = 3;
 // The last moment a tick may start a cswap switch (its 45 s timeout, inside RESTART_TICK_MS).
@@ -12198,13 +12201,14 @@ async function cmdDashboard() {
       accountsSaid = said;
     });
   };
-  // One dashboard-data run at a time, shared by every page waiting on it, given 2 minutes: each
+  // One dashboard-data run at a time, shared by every page waiting on it, given 2 minutes (a page
+  // with data already waits DATA_WAIT_MS for it, then has that data, marked stale): each
   // refresh used to start its own, and on a slow machine they piled up (one ran for an hour).
   let dataRun = null;
   let lastData = null;
   const dashboardData = () => (dataRun ??= new Promise((resolve) => execFile(process.execPath, [self, 'dashboard-data'], { cwd: mainRoot, maxBuffer: 64 * 1024 * 1024, env: { ...ownerEnv, DEVTEAM_PREVIEW_UP: JSON.stringify(Object.fromEntries(previewUp)) }, timeout: 120000, killSignal: 'SIGKILL' }, (err, stdout, stderr) => {
     dataRun = null;
-    if (!err) lastData = stdout;
+    if (!err && stdout) lastData = stdout;
     resolve({ ok: !err, stdout, stderr: err?.killed ? 'dashboard-data took over 2 minutes and was stopped' : String(stderr ?? '') });
   })));
   // /api/threads: every thread, its turns and the questions asked in it, for the open card's 2 s poll
@@ -12255,7 +12259,12 @@ async function cmdDashboard() {
       return res.end(fs.readFileSync(page));
     }
     if (req.method === 'GET' && req.url.startsWith('/api/data')) {
-      const r = await dashboardData();
+      // A run slower than DATA_WAIT_MS, with data to fall back on, answers with that, marked stale:
+      // the run goes on and the next read has it, but no page waits minutes on a stuck child.
+      let waited;
+      const slow = new Promise((resolve) => { waited = setTimeout(() => resolve({ ok: false, stderr: '' }), DATA_WAIT_MS); });
+      const r = await (lastData ? Promise.race([dashboardData(), slow]) : dashboardData());
+      clearTimeout(waited);
       if (!r.ok && !lastData) return send(res, 500, 'application/json', JSON.stringify({ error: r.stderr.trim() }));
       // The page in use says which version of itself it is: an open tab keeps the code it loaded,
       // so after a kit update it compares this with the first one it saw and offers a reload.

test/dashboard.test.mjs

@@ -16,16 +16,18 @@ import { canDrive, openChrome } from './lib/chrome.mjs';
 // A wait on a condition in a page. Page.navigate answers before the new page is parsed, so an
 // evaluation that throws (the last page still there, or the new one half parsed) is "not yet"; timed
 // out, it says where the tab is and the last error, so a page that never became the one expected
-// reads as that.
-const soonIn = (p, ms = 30000) => async (expr, what) => {
+// reads as that. server, if given, adds the end of what the dashboard's server has said.
+const soonIn = (p, ms = 30000, server = null) => async (expr, what) => {
   const end = Date.now() + ms;
   for (;;) {
     let v, err = null;
     try { v = await p.ev(expr); } catch (e) { err = e; }
     if (v) return v;
     if (Date.now() > end) {
       const at = await p.ev('[location.href, document.title, document.readyState]').catch((e) => e.message);
-      assert.fail(`timed out: ${what}; the tab is at ${JSON.stringify(at)}${err ? `; last: ${err.message}` : ''}`);
+      // with the fixture's fetch hook: the page's last /api/data reads, a stuck one with its age so far
+      const reads = await p.ev('(window.__reads ?? []).slice(-5).map((x) => x.ms === null ? `in flight ${Date.now() - x.at} ms` : `${x.status} in ${x.ms} ms`)').catch(() => []);
+      assert.fail(`timed out: ${what}; the tab is at ${JSON.stringify(at)}${err ? `; last: ${err.message}` : ''}${reads.length ? `; its last data reads: ${reads.join(', ')}` : ''}${server ? `; the server said, last: ${JSON.stringify(String(server()).slice(-600))}` : ''}`);
     }
     await new Promise((r) => setTimeout(r, 50));
   }
@@ -327,7 +329,17 @@ const POLL_FAST = `(() => {
   window.__gate = null;
   window.__answerHold = null;
   const inner = async (u, o) => {
-    if (String(u).startsWith('/api/data')) { window.__polls++; if (window.__gate) await window.__gate; return rf(u, o); }
+    // each read's start and end (a held one starts when the gate opens), so a wait that times out can
+    // say whether a read was stuck on the server, and for how long. The page gives up on a read after
+    // 30 s from its fetch, gate included: a gate held that long reads as a read given up at 0 ms.
+    if (String(u).startsWith('/api/data')) {
+      window.__polls++;
+      if (window.__gate) await window.__gate;
+      const x = { at: Date.now(), ms: null, status: null };
+      (window.__reads ??= []).push(x);
+      if (window.__reads.length > 20) window.__reads.shift();
+      try { const r = await rf(u, o); x.status = r.status; return r; } catch (e) { x.status = String(e.message || e); throw e; } finally { x.ms = Date.now() - x.at; }
+    }
     if (String(u).startsWith('/api/answer') && window.__answerHold) await window.__answerHold;
     // every owner action recorded as sent, and held on the wire, as a slow server would hold it
     if (o?.method === 'POST' && String(u).startsWith('/api/')) { (window.__posts ??= []).push({ url: String(u), body: o.body }); if (window.__postHold) await window.__postHold; }
@@ -429,7 +441,7 @@ test('presses: the page answers before the server, and a refusal puts it back',
   await p.send('Emulation.setDeviceMetricsOverride', { width: 1100, height: 800, deviceScaleFactor: 1, mobile: false });
   await p.send('Page.addScriptToEvaluateOnNewDocument', { source: POLL_FAST });
   await p.send('Page.navigate', { url: `http://127.0.0.1:${port}/#questions` });
-  const soon = soonIn(p, 15000);
+  const soon = soonIn(p, 15000, () => dash.out());
   const polls = async (n = 2) => { const at = await p.ev('window.__polls'); await soon(`window.__polls >= ${at + n}`, `${n} polls`); };
   const hold = () => p.ev('void (window.__postHold = new Promise((r) => { window.__let = r; }))');
   const letGo = () => p.ev('window.__let(); window.__postHold = null; void 0');
@@ -543,7 +555,7 @@ test('typing in the dashboard: a refresh never takes the field, the caret or the
     await p.send('Page.addScriptToEvaluateOnNewDocument', { source: POLL_FAST });
     await p.send('Page.navigate', { url: `http://127.0.0.1:${port}/#questions` });
     const box = `textarea[data-q="${Q}"]`;
-    const soon = soonIn(p, 15000);
+    const soon = soonIn(p, 15000, () => dash.out());
     await soon(`!!document.querySelector('${box}')`, 'the question card');
     // Answered by polls actually arriving, however long /api/data takes.
     const polls = async (n = 2) => { const at = await p.ev('window.__polls'); await soon(`window.__polls >= ${at + n}`, `${n} polls`); };
@@ -676,6 +688,27 @@ test('typing in the dashboard: a refresh never takes the field, the caret or the
   }
 });
 
+test('a slow data run: a read with data to fall back on has it at once, marked stale, and the run still lands', async (t) => {
+  const base = fs.mkdtempSync(path.join(os.tmpdir(), 'devteam-slow-data-'));
+  t.after(() => fs.rmSync(base, { recursive: true, force: true, maxRetries: 5, retryDelay: 100 }));
+  // waiting no time at all on a run: every read after the first is the last good data
+  const env = { ...process.env, DEVTEAM_DATA_WAIT_MS: '0', DEVTEAM_HUB_PORT: String(PORTS + 40), DEVTEAM_PORT_BASE: String(PORTS + 41), CLAUDE_CONFIG_DIR: path.join(base, 'claude'), CODEX_HOME: path.join(base, 'codex') };
+  for (const k of ['DEVTEAM_ROLE', 'DEVTEAM_TICKET', 'DEVTEAM_FEATURE', 'DEVTEAM_RUN', 'DEVTEAM_SUPERVISOR', 'TMUX', 'TMUX_PANE']) delete env[k];
+  const root = bareProject(base, 'slow');
+  const ask = (q) => { const r = spawnSync(process.execPath, [path.join(root, 'docs/devteam/scripts/devteam.mjs'), 'ask', '--question', q], { cwd: root, env: { ...env, DEVTEAM_ROLE: 'supervisor' }, encoding: 'utf8' }); assert.equal(r.status, 0, r.stderr); };
+  ask('First?');
+  const port = await freePort();
+  const dash = startDashboard(root, env, ['--port', String(port)]);
+  t.after(() => dash.stop());
+  await until(() => /dashboard on http/.test(dash.out()), 'the dashboard');
+  const read = async () => { const r = await fetch(`http://127.0.0.1:${port}/api/data`); return { stale: r.headers.get('x-devteam-stale') === 'true', asked: (await r.json()).questions.map((q) => q.question) }; };
+  assert.deepEqual(await read(), { stale: false, asked: ['First?'] }, 'the first read waits for its run: there is nothing to fall back on');
+  ask('Second?');
+  assert.deepEqual(await read(), { stale: true, asked: ['First?'] }, 'a read during a run has the last good data, marked stale');
+  // the run that read started goes on, and a later read has what it found
+  await until(async () => (await read()).asked.includes('Second?'), 'the run landing');
+});
+
 test('the dashboard and hub pages parse: one stray quote and nothing shows at all', () => {
   for (const page of ['index.html', 'hub.html']) {
     const html = fs.readFileSync(path.join(kit, 'dashboard', page), 'utf8');