diff --git a/README.md b/README.md index 7c5c674..0e7820a 100644 --- a/README.md +++ b/README.md @@ -91,8 +91,8 @@ The **Send message next** field lets you insert a single prompt into the active Open the gear tab in the popup for queue settings and logs. -- **Deep research aware**: keeps queue completion detection aware of ChatGPT Deep Research flows. It is enabled by default. -- **Unlimited retry and wait**: allows the automation to keep waiting/retrying instead of stopping at the normal retry boundary. It is disabled by default. +- **Deep research aware**: uses a longer finite wait and stale-progress timeout for ChatGPT Deep Research. It is enabled by default and does not wait forever unless **Unlimited retry and wait** is also enabled. +- **Unlimited retry and wait**: the only setting that removes the finite retry/wait bound. Automatic retries still wait between attempts and remain interruptible. It is disabled by default. - **Automation log**: refresh, copy, or clear the stored automation log for troubleshooting. If you enable unlimited retry/wait, a queue can remain active for much longer during a stuck or repeatedly failing ChatGPT state. diff --git a/background.js b/background.js index 5f683b8..13043e6 100644 --- a/background.js +++ b/background.js @@ -10,6 +10,29 @@ const QUEUE_SETTINGS_DEFAULTS = { queueDeliveryTimeoutRefresh: true }; const UNLIMITED_RETRY_DELAY_MS = 15000; +/** @type {QueueRetryPolicy} */ +const QUEUE_RETRY_POLICY = { + maxAutomaticAttempts: 3, + backoffBaseMs: 2000, + backoffFactor: 2, + backoffMaxMs: 30000, + unlimitedDelayMs: UNLIMITED_RETRY_DELAY_MS, + sleepSliceMs: 250 +}; +/** @type {QueueWaitPolicy} */ +const QUEUE_WAIT_POLICY = { + responseMaxWaitMs: 10 * 60 * 1000, + deepResearchMaxWaitMs: 45 * 60 * 1000, + deepResearchStaleMs: 5 * 60 * 1000, + checkIntervalMs: 1000 +}; +const RETRYABLE_FAILURE_CLASSES = new Set([ + 'transient', + 'generation-error', + 'retry-visible', + 'timeout', + 'stalled-research' +]); const QUEUE_WAKE_ALARM_NAME = 'queue-wake'; const QUEUE_WAKE_ALARM_PERIOD_MINUTES = 0.5; const QUEUE_STATE_COALESCE_DELAY_MS = 50; @@ -480,6 +503,17 @@ function restoreDurableJobs(durableJobs) { currentPhase: rawJob.currentPhase || (rawJob.waitForIdleBeforeSend ? 'waiting-for-idle' : (currentMessage ? 'waiting' : 'queued')), waitForIdleBeforeSend: rawJob.waitForIdleBeforeSend === true, deliveryTimeoutAttempts: Number(rawJob.deliveryTimeoutAttempts || 0), + retryAttemptCount: Number(rawJob.retryAttemptCount || 0), + lastRetryableReason: String(rawJob.lastRetryableReason || ''), + retryClass: String(rawJob.retryClass || ''), + retryMode: rawJob.retryMode === 'unlimited' ? 'unlimited' : (rawJob.retryMode === 'finite' ? 'finite' : ''), + nextRetryDelayMs: Number(rawJob.nextRetryDelayMs || 0), + nextRetryAt: Number(rawJob.nextRetryAt || 0), + retryExhausted: rawJob.retryExhausted === true, + waitStartedAt: Number(rawJob.waitStartedAt || 0), + lastResearchProgressAt: Number(rawJob.lastResearchProgressAt || 0), + sawDeepResearch: rawJob.sawDeepResearch === true, + sawGenerating: rawJob.sawGenerating === true, startedAt: Number(rawJob.startedAt || Date.now()), updatedAt: Number(rawJob.updatedAt || Date.now()) }; @@ -490,7 +524,7 @@ function restoreDurableJobs(durableJobs) { job.currentCommandNumber = Number(job.completedCount || 0) + 1; } - if (job.currentPhase === 'sending' || job.currentPhase === 'retry-wait') { + if (job.currentPhase === 'sending') { logQueueEvent(tabId, 'warn', 'Recovered a command that was not confirmed submitted; retrying it.', { phase: job.currentPhase, commandNumber: job.currentCommandNumber || 0, @@ -501,6 +535,7 @@ function restoreDurableJobs(durableJobs) { job.currentMessage = null; job.currentCommandNumber = 0; job.currentPhase = 'queued'; + resetWaitTracking(job); } jobs.set(tabId, job); @@ -682,6 +717,17 @@ function handleStartSequence(request, sendResponse) { currentPhase: waitForIdleBeforeStart ? 'waiting-for-idle' : 'queued', waitForIdleBeforeSend: waitForIdleBeforeStart, deliveryTimeoutAttempts: 0, + retryAttemptCount: 0, + lastRetryableReason: '', + retryClass: '', + retryMode: '', + nextRetryDelayMs: 0, + nextRetryAt: 0, + retryExhausted: false, + waitStartedAt: 0, + lastResearchProgressAt: 0, + sawDeepResearch: false, + sawGenerating: false, startedAt: Date.now(), updatedAt: Date.now() }); @@ -791,6 +837,17 @@ async function startNewJobFromEnqueueResult(tabId, message, waitForIdleBeforeSta currentPhase: waitForIdleBeforeStart ? 'waiting-for-idle' : 'queued', waitForIdleBeforeSend: waitForIdleBeforeStart, deliveryTimeoutAttempts: 0, + retryAttemptCount: 0, + lastRetryableReason: '', + retryClass: '', + retryMode: '', + nextRetryDelayMs: 0, + nextRetryAt: 0, + retryExhausted: false, + waitStartedAt: 0, + lastResearchProgressAt: 0, + sawDeepResearch: false, + sawGenerating: false, startedAt: Date.now(), updatedAt: Date.now() }); @@ -953,6 +1010,8 @@ function handleRetryPausedJob(request, sendResponse) { job.lastError = ''; job.currentPhase = 'queued'; job.updatedAt = Date.now(); + resetCommandRetryState(job); + resetWaitTracking(job); logQueueEvent(tabId, 'info', 'Retrying paused queue.', { completedCount: job.completedCount || 0, @@ -1028,6 +1087,12 @@ async function processQueue(tabId) { if (result.action === 'continue') continue; } + if (job.currentMessage && job.currentPhase === 'retry-wait') { + const result = await handleProcessRetryWait(tabId, job); + if (result.action === 'return') return; + if (result.action === 'continue') continue; + } + if (job.currentMessage) { const result = handleProcessRecovered(tabId, job); if (result.action === 'continue') continue; @@ -1148,8 +1213,10 @@ async function handleProcessWaiting(tabId, job) { return { action: 'continue' }; } - pauseJob(tabId, waitResult.error || 'ChatGPT response failed.', { + pauseJob(tabId, job.lastError || waitResult.error || 'ChatGPT response failed.', { phase: 'wait', + retryClass: job.retryClass || '', + retryAttemptCount: Number(job.retryAttemptCount || 0), diagnostics: waitResult.details || {} }); return { action: 'return' }; @@ -1181,6 +1248,27 @@ function handleProcessRecovered(tabId, job) { return { action: 'continue' }; } +async function handleProcessRetryWait(tabId, job) { + const remainingMs = Math.max(0, Number(job.nextRetryAt || 0) - Date.now()); + const interrupted = await interruptibleSleep(tabId, job, remainingMs); + + if (interrupted || !jobs.has(tabId) || job.isStopped || job.isPaused || !job.isRunning) { + return { action: 'return' }; + } + + if (job.currentMessage) { + job.queue.unshift(job.currentMessage); + job.currentMessage = null; + } + + job.currentCommandNumber = 0; + job.currentPhase = 'queued'; + job.nextRetryAt = 0; + job.updatedAt = Date.now(); + updateRunningJobsStorage({ force: true }); + return { action: 'continue' }; +} + async function handleProcessWaitForIdle(tabId, job) { const totalMessages = getTotalMessages(job); const queueSettings = await getQueueSettings(); @@ -1236,6 +1324,7 @@ async function handleProcessSending(tabId, job) { job.lastError = ''; job.currentPhase = 'sending'; job.updatedAt = Date.now(); + resetWaitTracking(job); const totalMessages = getTotalMessages(job); const queueSettings = await getQueueSettings(); @@ -1267,8 +1356,10 @@ async function handleProcessSending(tabId, job) { return { action: 'continue' }; } - pauseJob(tabId, sendResult.error || 'Could not send message to ChatGPT.', { + pauseJob(tabId, job.lastError || sendResult.error || 'Could not send message to ChatGPT.', { phase: 'send', + retryClass: job.retryClass || '', + retryAttemptCount: Number(job.retryAttemptCount || 0), diagnostics: sendResult.details || {} }); return { action: 'return' }; @@ -1321,8 +1412,10 @@ async function handleProcessSending(tabId, job) { return { action: 'continue' }; } - pauseJob(tabId, waitResult.error || 'ChatGPT response failed.', { + pauseJob(tabId, job.lastError || waitResult.error || 'ChatGPT response failed.', { phase: 'wait', + retryClass: job.retryClass || '', + retryAttemptCount: Number(job.retryAttemptCount || 0), diagnostics: waitResult.details || {} }); return { action: 'return' }; @@ -1352,6 +1445,8 @@ function completeCurrentCommand(tabId, job, totalMessages, diagnostics = {}) { job.currentPhase = 'queued'; job.deliveryTimeoutAttempts = 0; job.updatedAt = Date.now(); + resetCommandRetryState(job); + resetWaitTracking(job); updateRunningJobsStorage({ force: true }); } @@ -1373,6 +1468,7 @@ function pauseJob(tabId, reason, details = {}) { job.lastError = job.pausedReason; job.currentPhase = 'paused'; job.updatedAt = Date.now(); + resetWaitTracking(job); logQueueEvent(tabId, 'error', 'Queue paused.', { reason: job.pausedReason, @@ -1395,30 +1491,169 @@ function pauseJob(tabId, reason, details = {}) { }); } +function resetCommandRetryState(job) { + if (!job) return; + job.retryAttemptCount = 0; + job.lastRetryableReason = ''; + job.retryClass = ''; + job.retryMode = ''; + job.nextRetryDelayMs = 0; + job.nextRetryAt = 0; + job.retryExhausted = false; +} + +function resetWaitTracking(job) { + if (!job) return; + job.waitStartedAt = 0; + job.lastResearchProgressAt = 0; + job.sawDeepResearch = false; + job.sawGenerating = false; +} + +function getRetryBackoffDelayMs(attemptCount, unlimited = false) { + if (unlimited) { + return Math.max(1, Number(QUEUE_RETRY_POLICY.unlimitedDelayMs) || UNLIMITED_RETRY_DELAY_MS); + } + + const exponent = Math.max(0, Number(attemptCount) || 0); + const delay = Number(QUEUE_RETRY_POLICY.backoffBaseMs) * Math.pow(Number(QUEUE_RETRY_POLICY.backoffFactor) || 2, exponent); + const maxDelay = Number(QUEUE_RETRY_POLICY.backoffMaxMs) || delay; + return Math.max(1, Math.min(maxDelay, delay)); +} + +/** + * @param {string} phase + * @param {string} [reason] + * @param {Record} [diagnostics] + * @returns {{ class: QueueFailureClass, retryable: boolean, reason: string }} + */ +function classifyQueueFailure(phase, reason, diagnostics = {}) { + const details = diagnostics && typeof diagnostics === 'object' ? diagnostics : {}; + const state = details.state && typeof details.state === 'object' ? details.state : {}; + const message = String(reason || details.error || ''); + const forcedClass = String(details.failureClass || ''); + + let failureClass = forcedClass; + if (!failureClass) { + if (details.stopped === true || /queue was stopped/i.test(message)) { + failureClass = 'user-stop'; + } else if (details.compatibilityFailure || /compatibility failure/i.test(message)) { + failureClass = 'compatibility'; + } else if (details.stalledResearch === true || /stalled deep research/i.test(message)) { + failureClass = 'stalled-research'; + } else if (state.hasTryAgainButton) { + failureClass = 'retry-visible'; + } else if (state.hasDeliveryTimedOut || /delivery time/i.test(message)) { + failureClass = 'timeout'; + } else if (state.hasError || /chatgpt showed an error|error or retry state/i.test(message)) { + failureClass = 'generation-error'; + } else if (/timed out waiting/i.test(message)) { + failureClass = 'timeout'; + } else if (/could not read|could not inspect|could not send/i.test(message)) { + failureClass = 'transient'; + } else if (/queue was paused/i.test(message)) { + failureClass = 'non-retryable'; + } else if (phase === 'send' || phase === 'wait') { + failureClass = 'transient'; + } else { + failureClass = 'non-retryable'; + } + } + + if ((failureClass === 'generation-error' || failureClass === 'retry-visible') && state.hasTryAgainButton) { + failureClass = 'retry-visible'; + } + + const retryable = RETRYABLE_FAILURE_CLASSES.has(failureClass); + return { + class: failureClass, + retryable, + reason: message || 'Queue command failed.' + }; +} + +function getRetryExhaustionReason(job, lastReason) { + const attempts = Number(job?.retryAttemptCount || 0); + const reason = lastReason || job?.lastRetryableReason || job?.lastError || 'ChatGPT response failed.'; + return `Automatic retry exhausted after ${attempts} attempt${attempts === 1 ? '' : 's'}: ${reason}`; +} + +async function interruptibleSleep(tabId, job, ms) { + const duration = Math.max(0, Number(ms) || 0); + const deadline = Date.now() + duration; + const sliceMs = Math.max(5, Number(QUEUE_RETRY_POLICY.sleepSliceMs) || 250); + + while (Date.now() < deadline) { + if (!jobs.has(tabId) || !job || job.isStopped || job.isPaused || !job.isRunning) { + return true; + } + await sleep(Math.min(sliceMs, deadline - Date.now())); + } + + return !jobs.has(tabId) || !job || job.isStopped || job.isPaused || !job.isRunning; +} + async function retryCurrentCommandIfEnabled(tabId, job, phase, reason, diagnostics = {}) { if (!job || job.isStopped || job.isPaused || !job.isRunning) { return false; } - const queueSettings = await getQueueSettings(); + if (!job.currentMessage) { + return false; + } + + const classification = classifyQueueFailure(phase, reason, diagnostics); + job.retryClass = classification.class; + job.lastRetryableReason = classification.reason; - if (!queueSettings.queueUnlimitedRetryWait) { + if (!classification.retryable) { + job.lastError = classification.reason; + job.updatedAt = Date.now(); return false; } - if (!job.currentMessage) { + const queueSettings = await getQueueSettings(); + const unlimited = queueSettings.queueUnlimitedRetryWait === true; + const maxAttempts = unlimited ? Number.POSITIVE_INFINITY : Number(QUEUE_RETRY_POLICY.maxAutomaticAttempts); + const attemptCount = Number(job.retryAttemptCount || 0); + + if (attemptCount >= maxAttempts) { + job.retryExhausted = true; + job.retryMode = unlimited ? 'unlimited' : 'finite'; + job.lastError = getRetryExhaustionReason(job, classification.reason); + job.updatedAt = Date.now(); + logQueueEvent(tabId, 'error', job.lastError, { + phase, + retryClass: classification.class, + retryAttemptCount: attemptCount, + retryMode: job.retryMode, + commandNumber: job.currentCommandNumber || 0, + completedCount: job.completedCount || 0, + totalMessages: getTotalMessages(job), + diagnostics + }); return false; } - job.lastError = reason || 'Queue command failed.'; + const delayMs = getRetryBackoffDelayMs(attemptCount, unlimited); + job.retryAttemptCount = attemptCount + 1; + job.retryMode = unlimited ? 'unlimited' : 'finite'; + job.retryExhausted = false; + job.nextRetryDelayMs = delayMs; + job.nextRetryAt = Date.now() + delayMs; + job.lastError = classification.reason; job.currentPhase = 'retry-wait'; job.updatedAt = Date.now(); + resetWaitTracking(job); updateRunningJobsStorage({ force: true }); - logQueueEvent(tabId, 'warn', `Unlimited retry mode will retry command ${job.currentCommandNumber || '?'}/${getTotalMessages(job)}.`, { + logQueueEvent(tabId, 'warn', `${unlimited ? 'Unlimited' : 'Automatic'} retry ${job.retryAttemptCount}${unlimited ? '' : `/${maxAttempts}`} will retry command ${job.currentCommandNumber || '?'}/${getTotalMessages(job)}.`, { phase, - reason: job.lastError, - retryInSeconds: Math.round(UNLIMITED_RETRY_DELAY_MS / 1000), + reason: classification.reason, + retryClass: classification.class, + retryMode: job.retryMode, + retryAttemptCount: job.retryAttemptCount, + retryInSeconds: Math.round(delayMs / 1000), commandNumber: job.currentCommandNumber || 0, completedCount: job.completedCount || 0, totalMessages: getTotalMessages(job), @@ -1426,9 +1661,8 @@ async function retryCurrentCommandIfEnabled(tabId, job, phase, reason, diagnosti diagnostics }); - await sleep(UNLIMITED_RETRY_DELAY_MS); - - if (!jobs.has(tabId) || job.isStopped || job.isPaused || !job.isRunning) { + const interrupted = await interruptibleSleep(tabId, job, delayMs); + if (interrupted || !jobs.has(tabId) || job.isStopped || job.isPaused || !job.isRunning) { return false; } @@ -1436,6 +1670,7 @@ async function retryCurrentCommandIfEnabled(tabId, job, phase, reason, diagnosti job.currentMessage = null; job.currentCommandNumber = 0; job.currentPhase = 'queued'; + job.nextRetryAt = 0; job.updatedAt = Date.now(); updateRunningJobsStorage({ force: true }); @@ -1936,19 +2171,68 @@ async function sendPromptToSpecificTab(tabId, text) { } async function waitForTabResponse(tabId, context = {}) { - const startedAt = Date.now(); const queueSettings = { ...QUEUE_SETTINGS_DEFAULTS, ...context.queueSettings }; - const maxWaitMs = queueSettings.queueUnlimitedRetryWait - ? Number.POSITIVE_INFINITY - : 10 * 60 * 1000; - let sawGenerating = false; - let sawDeepResearch = false; + const liveJob = jobs.get(tabId); + const now = Date.now(); + if (liveJob && !Number(liveJob.waitStartedAt || 0)) { + liveJob.waitStartedAt = now; + liveJob.updatedAt = now; + updateRunningJobsStorage(); + } + + const startedAt = Number(liveJob?.waitStartedAt) || now; + const unlimited = queueSettings.queueUnlimitedRetryWait === true; + const ordinaryMaxWaitMs = Number(context.maxWaitMs) > 0 + ? Number(context.maxWaitMs) + : QUEUE_WAIT_POLICY.responseMaxWaitMs; + const deepResearchMaxWaitMs = Number(context.deepResearchMaxWaitMs) > 0 + ? Number(context.deepResearchMaxWaitMs) + : QUEUE_WAIT_POLICY.deepResearchMaxWaitMs; + const deepResearchStaleMs = Number(context.deepResearchStaleMs) > 0 + ? Number(context.deepResearchStaleMs) + : QUEUE_WAIT_POLICY.deepResearchStaleMs; + const checkIntervalMs = Number(context.checkIntervalMs) > 0 + ? Number(context.checkIntervalMs) + : QUEUE_WAIT_POLICY.checkIntervalMs; + let sawGenerating = liveJob?.sawGenerating === true; + let sawDeepResearch = liveJob?.sawDeepResearch === true; + let lastResearchPreview = ''; let lastProgressLogAt = startedAt; const waitLabel = getWaitContextLabel(context); + const persistWaitSignals = () => { + const current = jobs.get(tabId); + if (!current) return; + current.sawGenerating = sawGenerating; + current.sawDeepResearch = sawDeepResearch; + current.updatedAt = Date.now(); + updateRunningJobsStorage(); + }; + + const resolveMaxWaitMs = () => { + if (unlimited) return Number.POSITIVE_INFINITY; + if (queueSettings.queueDeepResearchAware && sawDeepResearch) { + return deepResearchMaxWaitMs; + } + return ordinaryMaxWaitMs; + }; + + const buildWaitDetails = (extra = {}) => { + const current = jobs.get(tabId); + return { + elapsedMs: Date.now() - startedAt, + sawGenerating, + sawDeepResearch, + settings: queueSettings, + waitStartedAt: startedAt, + lastResearchProgressAt: Number(current?.lastResearchProgressAt || 0), + ...extra + }; + }; + return new Promise((resolveRaw) => { let settled = false; let checkInterval = null; @@ -1960,6 +2244,7 @@ async function waitForTabResponse(tabId, context = {}) { if (checkInterval) { clearInterval(checkInterval); } + persistWaitSignals(); resolveRaw(value); }; checkInterval = setInterval(() => { @@ -1974,46 +2259,51 @@ async function waitForTabResponse(tabId, context = {}) { resolve({ ok: false, error: 'Queue was stopped.', - details: { - elapsedMs: Date.now() - startedAt, - sawGenerating, - sawDeepResearch, - settings: queueSettings - } + details: buildWaitDetails({ failureClass: 'user-stop' }) }); return; } if (job.isPaused || !job.isRunning) { - clearInterval(checkInterval); resolve({ ok: false, error: 'Queue was paused.', - details: { - elapsedMs: Date.now() - startedAt, - sawGenerating, - sawDeepResearch, - settings: queueSettings - } + details: buildWaitDetails({ failureClass: 'non-retryable' }) }); return; } - if (Date.now() - startedAt > maxWaitMs && !(queueSettings.queueDeepResearchAware && sawDeepResearch)) { - clearInterval(checkInterval); + const maxWaitMs = resolveMaxWaitMs(); + if (Date.now() - startedAt > maxWaitMs) { + const researchTimeout = queueSettings.queueDeepResearchAware && sawDeepResearch; resolve({ ok: false, - error: 'Timed out waiting for ChatGPT response.', - details: { - elapsedMs: Date.now() - startedAt, - sawGenerating, - sawDeepResearch, - settings: queueSettings - } + error: researchTimeout + ? 'Timed out waiting for Deep Research to finish.' + : 'Timed out waiting for ChatGPT response.', + details: buildWaitDetails({ + failureClass: 'timeout', + timedOut: true + }) }); return; } + if (!unlimited && queueSettings.queueDeepResearchAware && sawDeepResearch) { + const lastProgressAt = Number(job.lastResearchProgressAt || startedAt); + if (Date.now() - lastProgressAt > deepResearchStaleMs) { + resolve({ + ok: false, + error: 'Deep Research stalled without progress.', + details: buildWaitDetails({ + failureClass: 'stalled-research', + stalledResearch: true + }) + }); + return; + } + } + try { sendTabMessage(tabId, { type: 'CHECK_GENERATION_STATE' }).then((response) => { if (settled) { @@ -2032,13 +2322,10 @@ async function waitForTabResponse(tabId, context = {}) { ok: false, isDeliveryTimeout: true, error: state.matchedError || 'Message delivery timed out. Please try again.', - details: { - elapsedMs: Date.now() - startedAt, - sawGenerating, - sawDeepResearch, - settings: queueSettings, + details: buildWaitDetails({ + failureClass: 'timeout', state - } + }) }); return; } @@ -2049,18 +2336,16 @@ async function waitForTabResponse(tabId, context = {}) { ok: false, isDeliveryTimeout: false, error: 'ChatGPT showed an error or retry state.', - details: { - elapsedMs: Date.now() - startedAt, - sawGenerating, - sawDeepResearch, - settings: queueSettings, + details: buildWaitDetails({ + failureClass: state.hasTryAgainButton ? 'retry-visible' : 'generation-error', state - } + }) }); return; } const deepResearchActive = queueSettings.queueDeepResearchAware && !!state.deepResearchActive; + const researchPreview = String(state.researchStatusPreview || ''); if (deepResearchActive && !sawDeepResearch) { logQueueEvent(tabId, 'info', `Deep research activity detected for ${waitLabel}.`, { @@ -2068,12 +2353,14 @@ async function waitForTabResponse(tabId, context = {}) { totalMessages: context.totalMessages || 0, elapsedMs: Date.now() - startedAt, matchedResearchMarker: state.matchedResearchMarker || '', - researchStatusPreview: state.researchStatusPreview || '' + researchStatusPreview: researchPreview }); + lastResearchPreview = researchPreview; + job.lastResearchProgressAt = Date.now(); } if (state.generating || deepResearchActive) { - if (!sawGenerating) { + if (!sawGenerating && state.generating) { logQueueEvent(tabId, 'info', `ChatGPT is responding for ${waitLabel}.`, { commandNumber: context.commandNumber || 0, totalMessages: context.totalMessages || 0, @@ -2084,6 +2371,15 @@ async function waitForTabResponse(tabId, context = {}) { sawGenerating = sawGenerating || !!state.generating; sawDeepResearch = sawDeepResearch || deepResearchActive; + job.sawGenerating = sawGenerating; + job.sawDeepResearch = sawDeepResearch; + + if (state.generating || researchPreview !== lastResearchPreview) { + job.lastResearchProgressAt = Date.now(); + lastResearchPreview = researchPreview; + } + + persistWaitSignals(); if (Date.now() - lastProgressLogAt > 30000) { logQueueEvent(tabId, 'info', `Still waiting for ${waitLabel}.`, { @@ -2181,13 +2477,10 @@ async function waitForTabResponse(tabId, context = {}) { resolve({ ok: false, error: executionError.message || 'Could not read ChatGPT tab.', - details: { - elapsedMs: Date.now() - startedAt, - sawGenerating, - sawDeepResearch, - settings: queueSettings, + details: buildWaitDetails({ + failureClass: 'transient', error: serializeError(executionError) - } + }) }); }); } catch (error) { @@ -2195,16 +2488,13 @@ async function waitForTabResponse(tabId, context = {}) { resolve({ ok: false, error: error?.message || 'Could not inspect ChatGPT response state.', - details: { - elapsedMs: Date.now() - startedAt, - sawGenerating, - sawDeepResearch, - settings: queueSettings, + details: buildWaitDetails({ + failureClass: 'transient', error: serializeError(error) - } + }) }); } - }, 1000); + }, checkIntervalMs); }); } @@ -2236,7 +2526,7 @@ function getRunningJobsSnapshot() { conversationType, targetKey, remaining: getRemainingCount(job), - pending: job.queue.length, + pending: Array.isArray(job.queue) ? job.queue.length : 0, isRunning: job.isRunning, isPaused: job.isPaused, isStopped: job.isStopped, @@ -2245,7 +2535,7 @@ function getRunningJobsSnapshot() { lastError: job.lastError || '', currentMessage: job.currentMessage || '', currentMessagePreview: previewText(job.currentMessage || ''), - nextMessagePreview: previewText(job.queue[0] || ''), + nextMessagePreview: previewText((Array.isArray(job.queue) ? job.queue[0] : '') || ''), runId: job.runId || '', totalMessages: getTotalMessages(job), completedCount: job.completedCount || 0, @@ -2253,6 +2543,17 @@ function getRunningJobsSnapshot() { currentPhase: job.currentPhase || '', waitForIdleBeforeSend: job.waitForIdleBeforeSend === true, deliveryTimeoutAttempts: Number(job.deliveryTimeoutAttempts || 0), + retryAttemptCount: Number(job.retryAttemptCount || 0), + lastRetryableReason: job.lastRetryableReason || '', + retryClass: job.retryClass || '', + retryMode: job.retryMode || '', + nextRetryDelayMs: Number(job.nextRetryDelayMs || 0), + nextRetryAt: Number(job.nextRetryAt || 0), + retryExhausted: job.retryExhausted === true, + waitStartedAt: Number(job.waitStartedAt || 0), + lastResearchProgressAt: Number(job.lastResearchProgressAt || 0), + sawDeepResearch: job.sawDeepResearch === true, + sawGenerating: job.sawGenerating === true, startedAt: job.startedAt, updatedAt: job.updatedAt }; @@ -2262,7 +2563,8 @@ function getRunningJobsSnapshot() { } function getRemainingCount(job) { - return job.queue.length + (job.currentMessage ? 1 : 0); + const queued = Array.isArray(job?.queue) ? job.queue.length : 0; + return queued + (job?.currentMessage ? 1 : 0); } function getTotalMessages(job) { @@ -2397,6 +2699,17 @@ function getDurableJobsState() { currentPhase: job.currentPhase || 'queued', waitForIdleBeforeSend: job.waitForIdleBeforeSend === true, deliveryTimeoutAttempts: Number(job.deliveryTimeoutAttempts || 0), + retryAttemptCount: Number(job.retryAttemptCount || 0), + lastRetryableReason: job.lastRetryableReason || '', + retryClass: job.retryClass || '', + retryMode: job.retryMode || '', + nextRetryDelayMs: Number(job.nextRetryDelayMs || 0), + nextRetryAt: Number(job.nextRetryAt || 0), + retryExhausted: job.retryExhausted === true, + waitStartedAt: Number(job.waitStartedAt || 0), + lastResearchProgressAt: Number(job.lastResearchProgressAt || 0), + sawDeepResearch: job.sawDeepResearch === true, + sawGenerating: job.sawGenerating === true, startedAt: job.startedAt, updatedAt: job.updatedAt }; @@ -3309,7 +3622,15 @@ if (typeof module !== 'undefined' && module.exports) { refreshChatGPTTab, waitForTabToRecover, waitForTabResponse, + retryCurrentCommandIfEnabled, + classifyQueueFailure, + getRetryBackoffDelayMs, + handleRetryPausedJob, + completeCurrentCommand, QUEUE_SETTINGS_DEFAULTS, + QUEUE_RETRY_POLICY, + QUEUE_WAIT_POLICY, + UNLIMITED_RETRY_DELAY_MS, MAX_QUEUE_DEBUG_LOG_ENTRIES, jobs, SCHEDULED_MESSAGES_KEY, diff --git a/background.test.js b/background.test.js index 57fc3dc..d69f050 100644 --- a/background.test.js +++ b/background.test.js @@ -90,6 +90,8 @@ const { handleGetQueueDebugLogs, handleClearQueueDebugLogs, resumeDurableQueues, + restoreDurableJobs, + getDurableJobsState, MAX_QUEUE_DEBUG_LOG_ENTRIES } = require('./background.js'); @@ -269,6 +271,53 @@ test('durable recovery still rewrites an unconfirmed command safely', async () = assert.deepStrictEqual(localStorageData.queueDurableJobs[7].queue, ['recover-me']); }); +test('durable recovery preserves retry budget and does not duplicate a retry-wait command', async () => { + await resetQueueFixture(); + localStorageData.queueDurableJobs = { + 29: { + tabId: 29, + provider: 'chatgpt', + conversationId: 'retry-wait', + conversationType: 'existing', + targetKey: 'chatgpt:c:retry-wait', + queue: ['later'], + currentMessage: 'current-command', + isRunning: true, + isPaused: false, + currentPhase: 'retry-wait', + totalMessages: 2, + completedCount: 0, + currentCommandNumber: 1, + retryAttemptCount: 2, + lastRetryableReason: 'Timed out waiting for ChatGPT response.', + retryClass: 'timeout', + retryMode: 'finite', + nextRetryDelayMs: 4000, + nextRetryAt: Date.now() + 8000, + waitStartedAt: 42, + lastResearchProgressAt: 84, + sawDeepResearch: true, + startedAt: 1, + updatedAt: 1 + } + }; + + const restored = restoreDurableJobs(localStorageData.queueDurableJobs); + const job = restored[0]; + assert.strictEqual(job.currentMessage, 'current-command'); + assert.strictEqual(job.currentPhase, 'retry-wait'); + assert.strictEqual(job.retryAttemptCount, 2); + assert.strictEqual(job.waitStartedAt, 42); + assert.strictEqual(job.sawDeepResearch, true); + assert.deepStrictEqual(job.queue, ['later']); + + jobs.set(29, job); + const durable = getDurableJobsState(); + assert.strictEqual(durable[29].retryAttemptCount, 2); + assert.strictEqual(durable[29].currentMessage, 'current-command'); + assert.deepStrictEqual(durable[29].queue, ['later']); +}); + test('batched logs expose pending entries and preserve order and trimming', async () => { await resetQueueFixture(); const clearResponse = await invokeHandler(handleClearQueueDebugLogs); diff --git a/docs/project-memory/architecture.md b/docs/project-memory/architecture.md index 3093dd0..5c23f9e 100644 --- a/docs/project-memory/architecture.md +++ b/docs/project-memory/architecture.md @@ -1,7 +1,7 @@ # Architecture - `manifest.json` defines a Manifest V3 browser extension. `background.js` is the service worker; `popup.html`/`popup.js` provide the main UI; `options.html` is the settings page; `content.js` and `styles.css` run on supported ChatGPT, Gemini, and Claude hosts. -- `background.js` owns per-tab queue jobs in a `Map`, accepts runtime messages for start, enqueue, retry, stop, status, and diagnostics, injects prompt scripts into the selected tab, waits for response state, and records durable queue state in local storage. +- `background.js` owns per-tab queue jobs in a `Map`, accepts runtime messages for start, enqueue, retry, stop, status, and diagnostics, injects prompt scripts into the selected tab, waits for response state with a finite retry/wait policy, and records durable queue state including retry and Deep Research wait metadata in local storage. - `popup.js` selects supported provider tabs through provider-neutral discovery, stores reusable messages and sequences in local storage, stores optimizer and queue settings in sync storage, resolves placeholders, and renders queue instances, status, and diagnostics through runtime messages. - `content.js` owns Enter-to-queue on supported providers and the optimizer on ChatGPT only. It discovers conversation turns through primary and fallback selectors, hides older messages, lazily restores images, adds a load-more banner, observes DOM changes, and can queue Enter-submitted text while the provider is generating. - Queue response detection in `background.js` is DOM-driven through injected functions and provider adapters. It recognizes generating/streaming indicators, selected error markers, and deep-research progress; prompt submission finds a contenteditable input and a supported send-button selector for the active provider. diff --git a/docs/project-memory/decisions.md b/docs/project-memory/decisions.md index 6fcb798..07e28aa 100644 --- a/docs/project-memory/decisions.md +++ b/docs/project-memory/decisions.md @@ -1,8 +1,8 @@ # Decisions - Queue ownership is keyed by tab ID, allowing independent queues while preventing a second sequence from starting on a tab that already has a running or paused queue (`background.js`). -- A command is not completed until `waitForTabResponse()` observes the response cycle or applies its explicit fallback. Failed send/wait phases pause the queue; the retry setting can requeue the current command after a 15-second delay (`background.js`). -- Queue state is persisted in local storage and periodically woken with an alarms entry. Non-critical snapshots are coalesced and unchanged snapshots are skipped, while start, enqueue, send/recovery, pause, stop, completion, tab removal, and worker-wake boundaries force durable writes. On worker wake, an unconfirmed `sending` or `retry-wait` command is put back at the front before processing resumes (`background.js`). +- A command is not completed until `waitForTabResponse()` observes the response cycle or applies its explicit fallback. Failed send/wait phases pause the queue after a finite default of three automatic retries with exponential backoff. `queueUnlimitedRetryWait` is the only explicit unlimited wait/retry mode and still uses a non-zero delay; Deep Research awareness may use a longer progress-aware wait but cannot remove the final bound (`background.js`). +- Queue state is persisted in local storage and periodically woken with an alarms entry. Non-critical snapshots are coalesced and unchanged snapshots are skipped, while start, enqueue, send/recovery, pause, stop, completion, tab removal, and worker-wake boundaries force durable writes. Snapshots include retry budget and Deep Research wait/progress timestamps so a worker wake does not reset them. On worker wake, an unconfirmed `sending` command is put back at the front; `retry-wait` keeps the current command and remaining delay (`background.js`). - Queue debug logs remain ordered and bounded while buffered entries flush in batches; logical reads flush pending entries, and clear operations use a generation barrier so an older pending batch cannot repopulate cleared logs (`background.js`). - The optimizer uses layered selectors and fallback discovery because the page is controlled by ChatGPT. Its windowing keeps at least eight recent messages visible and caps the discovered message set at 1,200 (`content.js`). - Extension API helpers try callback and promise forms so popup, content, and background code can use the same operations across supported browser API variants (`popup.js`, `content.js`, `background.js`). diff --git a/docs/project-memory/known-failures.md b/docs/project-memory/known-failures.md index 575efb0..a833ef7 100644 --- a/docs/project-memory/known-failures.md +++ b/docs/project-memory/known-failures.md @@ -1,9 +1,9 @@ # Known failures and limits - Prompt submission depends on each provider’s current contenteditable and send-button selectors. If either is absent or disabled, the injected send operation returns an error and the queue pauses (`background.js`, `provider-adapter.js`). -- Response detection depends on provider generating, streaming, status, and error markers. An injected-script failure, a detected error/retry state, or a default ten-minute wait timeout pauses the queue (`background.js`). +- Response detection depends on provider generating, streaming, status, and error markers. An injected-script failure, a detected error/retry state, a stalled Deep Research wait, or a finite wait timeout pauses the queue after the automatic retry budget is exhausted (`background.js`). - In the default wait mode, if no generating indicator appears for five seconds, the worker records that it assumed completion and advances. This is an intentional fallback but can misclassify a response when the page exposes no recognized indicator (`background.js`). -- Deep-research-aware waiting only extends the default timeout after recognized research activity has been observed. Unrecognized page wording does not extend the wait (`background.js`). +- Deep-research-aware waiting uses a longer finite maximum duration and a stale-progress timeout. Progress updates reset the stale timer, but only explicit unlimited retry/wait removes the final bound (`background.js`). - If a worker restart finds only legacy running-job state and no recoverable durable queue, it records that the in-memory queue was lost and clears the stale state (`background.js`). - Gemini and Claude optimizer/message-window support is intentionally unsupported; queue Enter interception still runs on those providers (`provider-adapter.js`, `content.js`). - Chrome packaging is skipped when a Chrome executable is unavailable. Firefox persistent installation can be rejected for unsigned release packages; the fallback requires a working Firefox, npm/web-ext, and profile setup (`Installers/install_chatgpt_queue_optimizer.py`, `Installers/README.md`). diff --git a/test/issue-29-retry.test.js b/test/issue-29-retry.test.js new file mode 100644 index 0000000..0e9c34f --- /dev/null +++ b/test/issue-29-retry.test.js @@ -0,0 +1,722 @@ +const test = require('node:test'); +const assert = require('node:assert/strict'); + +const mockTabs = new Map(); +const mockStorageLocal = {}; +let mockSyncStorage = { + queueUnlimitedRetryWait: false, + queueDeepResearchAware: true, + queueDeliveryTimeoutRefresh: true +}; + +global.chrome = { + runtime: { + lastError: null, + onMessage: { addListener: () => {} }, + onInstalled: { addListener: () => {} }, + getURL: () => '', + sendMessage: (_message, callback) => callback && callback() + }, + browserAction: { + onClicked: { addListener: () => {} } + }, + commands: { + onCommand: { addListener: () => {} } + }, + storage: { + sync: { + get: (defaults, callback) => callback && callback({ ...defaults, ...mockSyncStorage }), + set: (items, callback) => { + Object.assign(mockSyncStorage, items); + callback && callback(); + } + }, + local: { + get: (keys, callback) => { + const requested = Array.isArray(keys) + ? keys + : (typeof keys === 'object' && keys !== null ? Object.keys(keys) : [keys]); + const result = {}; + for (const key of requested) { + if (key in mockStorageLocal) result[key] = mockStorageLocal[key]; + } + callback && callback(result); + }, + set: (items, callback) => { + Object.assign(mockStorageLocal, items); + callback && callback(); + } + } + }, + tabs: { + onRemoved: { addListener: () => {} }, + get: (tabId, callback) => { + const tab = mockTabs.get(tabId) || { id: tabId, url: 'https://chatgpt.com/' }; + callback && callback(tab); + return Promise.resolve(tab); + }, + sendMessage: (tabId, message, callback) => { + const tab = mockTabs.get(tabId); + const response = tab?.onMessage ? tab.onMessage(message) : { ok: true }; + callback && callback(response); + return Promise.resolve(response); + } + }, + alarms: { + onAlarm: { addListener: () => {} }, + create: () => {}, + clear: () => {} + }, + scripting: { + executeScript: () => Promise.resolve([{ result: { ok: true, details: {} } }]) + } +}; + +require('../utils.js'); + +const { + jobs, + waitForTabResponse, + retryCurrentCommandIfEnabled, + classifyQueueFailure, + getRetryBackoffDelayMs, + handleRetryPausedJob, + completeCurrentCommand, + pauseJob, + restoreDurableJobs, + getDurableJobsState, + getRunningJobsSnapshot, + QUEUE_RETRY_POLICY, + QUEUE_WAIT_POLICY, + QUEUE_SETTINGS_DEFAULTS, + UNLIMITED_RETRY_DELAY_MS +} = require('../background.js'); + +function muteConsole() { + const original = { + log: console.log, + warn: console.warn, + error: console.error, + info: console.info + }; + console.log = () => {}; + console.warn = () => {}; + console.error = () => {}; + console.info = () => {}; + return () => { + console.log = original.log; + console.warn = original.warn; + console.error = original.error; + console.info = original.info; + }; +} + +async function withRetryPolicy(overrides, fn) { + const original = { ...QUEUE_RETRY_POLICY }; + Object.assign(QUEUE_RETRY_POLICY, overrides); + try { + return await fn(); + } finally { + Object.assign(QUEUE_RETRY_POLICY, original); + } +} + +function createJob(tabId, overrides = {}) { + return { + tabId, + provider: 'chatgpt', + conversationId: `issue-29-${tabId}`, + conversationType: 'existing', + targetKey: `chatgpt:c:issue-29-${tabId}`, + queue: ['next-command'], + currentMessage: 'retry-me', + isRunning: true, + isPaused: false, + isStopped: false, + pausedReason: '', + lastError: '', + runId: `run-${tabId}`, + totalMessages: 2, + completedCount: 0, + currentCommandNumber: 1, + currentPhase: 'waiting', + waitForIdleBeforeSend: false, + deliveryTimeoutAttempts: 0, + retryAttemptCount: 0, + lastRetryableReason: '', + retryClass: '', + retryMode: '', + nextRetryDelayMs: 0, + nextRetryAt: 0, + retryExhausted: false, + waitStartedAt: 0, + lastResearchProgressAt: 0, + sawDeepResearch: false, + sawGenerating: false, + startedAt: Date.now(), + updatedAt: Date.now(), + ...overrides + }; +} + +function generationState(overrides = {}) { + return { + generating: false, + hasError: false, + hasTryAgainButton: false, + hasDeliveryTimedOut: false, + deepResearchActive: false, + researchStatusPreview: '', + matchedError: '', + ...overrides + }; +} + +function installTab(tabId, getState) { + mockTabs.set(tabId, { + id: tabId, + url: `https://chatgpt.com/c/issue-29-${tabId}`, + onMessage: (message) => { + if (message.type === 'CHECK_GENERATION_STATE') { + return { state: getState() }; + } + return { ok: true }; + } + }); +} + +async function waitContext(tabId, overrides = {}) { + return waitForTabResponse(tabId, { + commandNumber: 1, + totalMessages: 1, + queueSettings: { + queueUnlimitedRetryWait: false, + queueDeepResearchAware: true, + queueDeliveryTimeoutRefresh: true + }, + maxWaitMs: 80, + deepResearchMaxWaitMs: 220, + deepResearchStaleMs: 90, + checkIntervalMs: 20, + ...overrides + }); +} + +test('classifyQueueFailure covers retryable and non-retryable classes from generation state', () => { + assert.equal( + classifyQueueFailure('wait', 'Queue was stopped.', { failureClass: 'user-stop' }).class, + 'user-stop' + ); + assert.equal( + classifyQueueFailure('wait', 'Queue was stopped.').retryable, + false + ); + + const compatibility = classifyQueueFailure( + 'send', + 'ChatGPT compatibility failure: required composer signal was not found.', + { compatibilityFailure: 'composer' } + ); + assert.equal(compatibility.class, 'compatibility'); + assert.equal(compatibility.retryable, false); + + const retryVisible = classifyQueueFailure('wait', 'ChatGPT showed an error or retry state.', { + state: generationState({ hasTryAgainButton: true, hasError: true }) + }); + assert.equal(retryVisible.class, 'retry-visible'); + assert.equal(retryVisible.retryable, true); + + const generationError = classifyQueueFailure('wait', 'ChatGPT showed an error or retry state.', { + state: generationState({ hasError: true, matchedError: 'something went wrong' }) + }); + assert.equal(generationError.class, 'generation-error'); + assert.equal(generationError.retryable, true); + + const timeout = classifyQueueFailure('wait', 'Timed out waiting for ChatGPT response.', { + failureClass: 'timeout' + }); + assert.equal(timeout.class, 'timeout'); + assert.equal(timeout.retryable, true); + + const stalled = classifyQueueFailure('wait', 'Deep Research stalled without progress.', { + stalledResearch: true + }); + assert.equal(stalled.class, 'stalled-research'); + assert.equal(stalled.retryable, true); + + const transient = classifyQueueFailure('wait', 'Could not read ChatGPT tab.', { + error: { message: 'Could not read ChatGPT tab.' } + }); + assert.equal(transient.class, 'transient'); + assert.equal(transient.retryable, true); +}); + +test('finite retry backoff increases and respects the cap', () => { + const first = getRetryBackoffDelayMs(0, false); + const second = getRetryBackoffDelayMs(1, false); + const third = getRetryBackoffDelayMs(2, false); + assert.ok(first > 0); + assert.ok(second > first); + assert.ok(third > second); + + const capped = getRetryBackoffDelayMs(20, false); + assert.equal(capped, QUEUE_RETRY_POLICY.backoffMaxMs); + + const unlimited = getRetryBackoffDelayMs(99, true); + assert.equal(unlimited, UNLIMITED_RETRY_DELAY_MS); + assert.ok(unlimited > 0); +}); + +test('success-after-retry requeues the same command and resets after completion', async () => { + const restoreConsole = muteConsole(); + const tabId = 2901; + const job = createJob(tabId); + jobs.set(tabId, job); + + await withRetryPolicy({ backoffBaseMs: 15, backoffMaxMs: 20, sleepSliceMs: 5 }, async () => { + const retried = await retryCurrentCommandIfEnabled( + tabId, + job, + 'wait', + 'ChatGPT showed an error or retry state.', + { state: generationState({ hasError: true, hasTryAgainButton: true }) } + ); + + assert.equal(retried, true); + assert.equal(job.retryAttemptCount, 1); + assert.equal(job.retryMode, 'finite'); + assert.equal(job.queue[0], 'retry-me'); + assert.equal(job.currentMessage, null); + assert.equal(job.queue.filter((message) => message === 'retry-me').length, 1); + + job.currentMessage = job.queue.shift(); + job.currentCommandNumber = 1; + completeCurrentCommand(tabId, job, 2, { recoveredAfterRetry: true }); + assert.equal(job.retryAttemptCount, 0); + assert.equal(job.currentMessage, null); + assert.equal(job.lastRetryableReason, ''); + }); + + jobs.clear(); + restoreConsole(); +}); + +test('finite retry exhausts exactly at the default attempt limit', async () => { + const restoreConsole = muteConsole(); + const tabId = 2902; + const job = createJob(tabId, { queue: [] }); + jobs.set(tabId, job); + + await withRetryPolicy({ backoffBaseMs: 10, backoffMaxMs: 15, sleepSliceMs: 5 }, async () => { + for (let attempt = 0; attempt < QUEUE_RETRY_POLICY.maxAutomaticAttempts; attempt += 1) { + job.currentMessage = 'retry-me'; + job.currentCommandNumber = 1; + const retried = await retryCurrentCommandIfEnabled( + tabId, + job, + 'wait', + 'Timed out waiting for ChatGPT response.', + { failureClass: 'timeout' } + ); + assert.equal(retried, true, `retry ${attempt + 1} should run`); + } + + job.currentMessage = 'retry-me'; + job.currentCommandNumber = 1; + const exhausted = await retryCurrentCommandIfEnabled( + tabId, + job, + 'wait', + 'Timed out waiting for ChatGPT response.', + { failureClass: 'timeout' } + ); + assert.equal(exhausted, false); + assert.equal(job.retryAttemptCount, QUEUE_RETRY_POLICY.maxAutomaticAttempts); + assert.equal(job.retryExhausted, true); + assert.match(job.lastError, /Automatic retry exhausted after 3 attempts/); + assert.match(job.lastError, /Timed out waiting for ChatGPT response/); + }); + + jobs.clear(); + restoreConsole(); +}); + +test('unlimited retry continues past the finite limit and stays interruptible', async () => { + const restoreConsole = muteConsole(); + const tabId = 2903; + mockSyncStorage.queueUnlimitedRetryWait = true; + const job = createJob(tabId, { queue: [] }); + jobs.set(tabId, job); + + await withRetryPolicy({ + backoffBaseMs: 10, + unlimitedDelayMs: 20, + sleepSliceMs: 5 + }, async () => { + job.retryAttemptCount = QUEUE_RETRY_POLICY.maxAutomaticAttempts; + const retried = await retryCurrentCommandIfEnabled( + tabId, + job, + 'wait', + 'ChatGPT showed an error or retry state.', + { state: generationState({ hasError: true }) } + ); + assert.equal(retried, true); + assert.equal(job.retryMode, 'unlimited'); + assert.ok(job.retryAttemptCount > QUEUE_RETRY_POLICY.maxAutomaticAttempts); + + job.currentMessage = 'retry-me'; + job.queue = []; + const retryPromise = retryCurrentCommandIfEnabled( + tabId, + job, + 'wait', + 'ChatGPT showed an error or retry state.', + { state: generationState({ hasError: true }) } + ); + job.isStopped = true; + job.isRunning = false; + const stopped = await retryPromise; + assert.equal(stopped, false); + assert.equal(job.currentMessage, 'retry-me'); + assert.equal(job.queue.includes('retry-me'), false); + }); + + mockSyncStorage.queueUnlimitedRetryWait = false; + jobs.clear(); + restoreConsole(); +}); + +test('queueDeepResearchAware cannot disable all wait limits by itself', { timeout: 4000 }, async () => { + const restoreConsole = muteConsole(); + const tabId = 2904; + const job = createJob(tabId, { currentPhase: 'waiting' }); + jobs.set(tabId, job); + installTab(tabId, () => generationState({ + generating: true, + deepResearchActive: true, + researchStatusPreview: 'Deep research is searching sources' + })); + + const result = await waitContext(tabId, { + queueSettings: { + queueUnlimitedRetryWait: false, + queueDeepResearchAware: true + } + }); + + assert.equal(result.ok, false); + assert.match(result.error, /Deep Research|timed out/i); + assert.equal(result.details.sawDeepResearch, true); + assert.notEqual(result.details.settings.queueUnlimitedRetryWait, true); + + jobs.clear(); + mockTabs.delete(tabId); + restoreConsole(); +}); + +test('continuing Deep Research may wait longer than an ordinary response', { timeout: 4000 }, async () => { + const restoreConsole = muteConsole(); + const tabId = 2905; + const job = createJob(tabId); + jobs.set(tabId, job); + const startedAt = Date.now(); + let polls = 0; + + installTab(tabId, () => { + polls += 1; + const elapsed = Date.now() - startedAt; + const active = elapsed < 140; + return generationState({ + generating: active, + deepResearchActive: active, + researchStatusPreview: active ? `Deep research step ${polls}` : 'done' + }); + }); + + const result = await waitContext(tabId, { + maxWaitMs: 80, + deepResearchMaxWaitMs: 400, + deepResearchStaleMs: 300 + }); + + assert.equal(result.ok, true); + assert.ok(result.details.elapsedMs > 80); + assert.equal(result.details.sawDeepResearch, true); + + jobs.clear(); + mockTabs.delete(tabId); + restoreConsole(); +}); + +test('stalled Deep Research fails with a specific reason under default finite settings', { timeout: 4000 }, async () => { + const restoreConsole = muteConsole(); + const tabId = 2906; + const job = createJob(tabId); + jobs.set(tabId, job); + installTab(tabId, () => generationState({ + generating: false, + deepResearchActive: true, + researchStatusPreview: 'Deep research is searching sources' + })); + + const result = await waitContext(tabId, { + maxWaitMs: 1000, + deepResearchMaxWaitMs: 1000, + deepResearchStaleMs: 70 + }); + + assert.equal(result.ok, false); + assert.match(result.error, /stalled/i); + assert.equal(result.details.stalledResearch, true); + assert.equal(result.details.failureClass, 'stalled-research'); + + jobs.clear(); + mockTabs.delete(tabId); + restoreConsole(); +}); + +test('stop during wait or backoff prevents later submission', { timeout: 4000 }, async () => { + const restoreConsole = muteConsole(); + const tabId = 2907; + const job = createJob(tabId, { queue: ['other'] }); + jobs.set(tabId, job); + installTab(tabId, () => generationState({ + generating: true, + deepResearchActive: true, + researchStatusPreview: `tick ${Date.now()}` + })); + + const waitPromise = waitContext(tabId, { + maxWaitMs: 1000, + deepResearchMaxWaitMs: 1000, + deepResearchStaleMs: 1000 + }); + await new Promise((resolve) => { + setTimeout(resolve, 40); + }); + job.isStopped = true; + job.isRunning = false; + const waitResult = await waitPromise; + assert.equal(waitResult.ok, false); + assert.match(waitResult.error, /stopped/i); + + job.isStopped = false; + job.isRunning = true; + job.currentMessage = 'retry-me'; + job.queue = ['other']; + + const retryPromise = withRetryPolicy({ + backoffBaseMs: 200, + sleepSliceMs: 10 + }, () => retryCurrentCommandIfEnabled( + tabId, + job, + 'wait', + 'Timed out waiting for ChatGPT response.', + { failureClass: 'timeout' } + )); + await new Promise((resolve) => { + setTimeout(resolve, 20); + }); + job.isStopped = true; + job.isRunning = false; + const retried = await retryPromise; + assert.equal(retried, false); + assert.equal(job.currentMessage, 'retry-me'); + assert.deepEqual(job.queue, ['other']); + + jobs.clear(); + mockTabs.delete(tabId); + restoreConsole(); +}); + +test('non-retryable compatibility failures do not enter an automatic retry loop', async () => { + const restoreConsole = muteConsole(); + const tabId = 2908; + const job = createJob(tabId, { queue: [] }); + jobs.set(tabId, job); + + const retried = await retryCurrentCommandIfEnabled( + tabId, + job, + 'send', + 'ChatGPT compatibility failure: required composer signal was not found.', + { compatibilityFailure: 'composer' } + ); + + assert.equal(retried, false); + assert.equal(job.retryAttemptCount, 0); + assert.equal(job.currentMessage, 'retry-me'); + assert.deepEqual(job.queue, []); + assert.equal(classifyQueueFailure( + 'send', + 'ChatGPT compatibility failure: required composer signal was not found.', + { compatibilityFailure: 'composer' } + ).retryable, false); + + jobs.clear(); + restoreConsole(); +}); + +test('durable recovery preserves retry budget and research wait state without duplicating the command', () => { + const tabId = 2909; + jobs.clear(); + jobs.set(tabId, createJob(tabId, { + queue: ['next-command'], + currentMessage: 'retry-me', + currentPhase: 'retry-wait', + retryAttemptCount: 2, + lastRetryableReason: 'Timed out waiting for ChatGPT response.', + retryClass: 'timeout', + retryMode: 'finite', + nextRetryDelayMs: 4000, + nextRetryAt: Date.now() + 4000, + waitStartedAt: 123, + lastResearchProgressAt: 456, + sawDeepResearch: true, + sawGenerating: true + })); + + const durable = getDurableJobsState(); + assert.equal(durable[tabId].retryAttemptCount, 2); + assert.equal(durable[tabId].currentMessage, 'retry-me'); + assert.equal(durable[tabId].currentPhase, 'retry-wait'); + assert.equal(durable[tabId].sawDeepResearch, true); + assert.equal(durable[tabId].waitStartedAt, 123); + + const snapshot = getRunningJobsSnapshot(); + assert.equal(snapshot[tabId].retryAttemptCount, 2); + assert.equal(snapshot[tabId].lastError, ''); + + jobs.clear(); + const restored = restoreDurableJobs(durable); + assert.equal(restored.length, 1); + const job = restored[0]; + assert.equal(job.currentMessage, 'retry-me'); + assert.equal(job.currentPhase, 'retry-wait'); + assert.equal(job.retryAttemptCount, 2); + assert.equal(job.lastRetryableReason, 'Timed out waiting for ChatGPT response.'); + assert.equal(job.sawDeepResearch, true); + assert.equal(job.waitStartedAt, 123); + assert.equal(job.lastResearchProgressAt, 456); + assert.deepEqual(job.queue, ['next-command']); + assert.equal(job.queue.filter((message) => message === 'retry-me').length, 0); + + jobs.clear(); + restoreDurableJobs({ + 2910: { + tabId: 2910, + provider: 'chatgpt', + conversationId: 'waiting', + conversationType: 'existing', + targetKey: 'chatgpt:c:waiting', + queue: [], + currentMessage: 'keep-waiting', + isRunning: true, + currentPhase: 'waiting', + retryAttemptCount: 1, + waitStartedAt: 50, + lastResearchProgressAt: 75, + sawDeepResearch: true + } + }); + const waiting = jobs.get(2910); + assert.equal(waiting.currentMessage, 'keep-waiting'); + assert.equal(waiting.currentPhase, 'waiting'); + assert.equal(waiting.retryAttemptCount, 1); + assert.deepEqual(waiting.queue, []); + + jobs.clear(); +}); + +test('manual retryPausedJob resumes the paused command once', () => { + const restoreConsole = muteConsole(); + const tabId = 2911; + const job = createJob(tabId, { + currentMessage: 'retry-me', + queue: ['next-command'] + }); + jobs.set(tabId, job); + job.isProcessing = true; + pauseJob(tabId, 'Automatic retry exhausted after 3 attempts: Timed out waiting for ChatGPT response.'); + assert.deepEqual(job.queue, ['retry-me', 'next-command']); + assert.equal(job.currentMessage, null); + + let response = null; + handleRetryPausedJob({ tabId }, (result) => { + response = result; + }); + + assert.equal(response.ok, true); + assert.equal(job.isPaused, false); + assert.equal(job.isRunning, true); + assert.deepEqual(job.queue, ['retry-me', 'next-command']); + assert.equal(job.retryAttemptCount, 0); + assert.equal(job.lastError, ''); + + jobs.clear(); + restoreConsole(); +}); + +test('exactly-once sending recovery still rewrites an unconfirmed command once', () => { + jobs.clear(); + restoreDurableJobs({ + 2912: { + tabId: 2912, + provider: 'chatgpt', + conversationId: 'send-once', + conversationType: 'existing', + targetKey: 'chatgpt:c:send-once', + queue: [], + currentMessage: 'only-once', + isRunning: true, + currentPhase: 'sending', + currentCommandNumber: 1, + retryAttemptCount: 1 + } + }); + const job = jobs.get(2912); + assert.deepEqual(job.queue, ['only-once']); + assert.equal(job.currentMessage, null); + assert.equal(job.currentPhase, 'queued'); + assert.equal(job.retryAttemptCount, 1); + jobs.clear(); +}); + +test('recovered wait elapsed time is preserved so Deep Research cannot restart its budget', { timeout: 4000 }, async () => { + const restoreConsole = muteConsole(); + const tabId = 2913; + const job = createJob(tabId, { + waitStartedAt: Date.now() - 250, + lastResearchProgressAt: Date.now() - 250, + sawDeepResearch: true + }); + jobs.set(tabId, job); + installTab(tabId, () => generationState({ + generating: true, + deepResearchActive: true, + researchStatusPreview: `Deep research ${Date.now()}` + })); + + const result = await waitContext(tabId, { + maxWaitMs: 1000, + deepResearchMaxWaitMs: 80, + deepResearchStaleMs: 1000 + }); + + assert.equal(result.ok, false); + assert.match(result.error, /Deep Research|timed out/i); + + jobs.clear(); + mockTabs.delete(tabId); + restoreConsole(); +}); + +test('queue defaults keep Deep Research aware and finite retry/wait', () => { + assert.equal(QUEUE_SETTINGS_DEFAULTS.queueUnlimitedRetryWait, false); + assert.equal(QUEUE_SETTINGS_DEFAULTS.queueDeepResearchAware, true); + assert.equal(QUEUE_RETRY_POLICY.maxAutomaticAttempts, 3); + assert.ok(QUEUE_WAIT_POLICY.deepResearchMaxWaitMs > QUEUE_WAIT_POLICY.responseMaxWaitMs); + assert.ok(QUEUE_WAIT_POLICY.deepResearchStaleMs > 0); +}); diff --git a/test/provider-adapter.test.js b/test/provider-adapter.test.js index 2add063..57853be 100644 --- a/test/provider-adapter.test.js +++ b/test/provider-adapter.test.js @@ -93,6 +93,7 @@ const { refreshChatGPTTab, waitForTabToRecover, waitForTabResponse, + classifyQueueFailure, QUEUE_SETTINGS_DEFAULTS, jobs } = require('../background.js'); @@ -891,6 +892,9 @@ test('ChatGPTAdapter detects delivery timeout distinct from generic errors', () assert.equal(state1.hasError, true); assert.equal(state1.matchedError, 'Message delivery timed out. Please try again.'); assert.equal(state1.generating, false); + const timeoutClass = classifyQueueFailure('wait', state1.matchedError, { state: state1 }); + assert.equal(timeoutClass.class, 'timeout'); + assert.equal(timeoutClass.retryable, true); // 2. Delivery timeout inside the latest turn const { doc: doc2 } = createTestDoc({ @@ -1304,3 +1308,54 @@ test('Durable state and settings preserve deliveryTimeoutAttempts and defaults', jobs.clear(); }); +test('waitForTabResponse keeps Deep Research finite unless unlimited retry is enabled', async () => { + const tabId = 702; + mockTabs.set(tabId, { + id: tabId, + url: 'https://chatgpt.com/c/deep-research-wait', + onMessage: (message) => { + if (message.type === 'CHECK_GENERATION_STATE') { + return { + state: { + generating: false, + deepResearchActive: true, + researchStatusPreview: 'Deep research is searching sources', + hasError: false, + hasTryAgainButton: false + } + }; + } + return { ok: true }; + } + }); + + jobs.set(tabId, { + tabId, + isRunning: true, + isPaused: false, + isStopped: false, + currentPhase: 'waiting' + }); + + const waitResult = await waitForTabResponse(tabId, { + commandNumber: 1, + totalMessages: 1, + queueSettings: { + queueUnlimitedRetryWait: false, + queueDeepResearchAware: true + }, + maxWaitMs: 80, + deepResearchMaxWaitMs: 180, + deepResearchStaleMs: 1000, + checkIntervalMs: 20 + }); + + assert.equal(waitResult.ok, false); + assert.match(waitResult.error, /Deep Research|timed out/i); + assert.equal(waitResult.details.sawDeepResearch, true); + + const classified = classifyQueueFailure('wait', waitResult.error, waitResult.details); + assert.equal(classified.retryable, true); + jobs.clear(); +}); + diff --git a/types/extension.d.ts b/types/extension.d.ts index 950f01d..f851980 100644 --- a/types/extension.d.ts +++ b/types/extension.d.ts @@ -8,6 +8,19 @@ type QueueJobPhase = | 'paused' | 'complete' | string; + +type QueueFailureClass = + | 'transient' + | 'generation-error' + | 'retry-visible' + | 'timeout' + | 'stalled-research' + | 'compatibility' + | 'user-stop' + | 'non-retryable' + | string; + +type QueueRetryMode = 'finite' | 'unlimited' | '' | string; type QueueJobStatus = 'running' | 'paused' | 'stopped' | 'idle' | string; type ScheduledMessageStatus = 'pending' | 'firing' | 'completed' | 'failed' | 'cancelled' | string; @@ -25,6 +38,22 @@ interface QueueSettings { queueDeliveryTimeoutRefresh: boolean; } +interface QueueRetryPolicy { + maxAutomaticAttempts: number; + backoffBaseMs: number; + backoffFactor: number; + backoffMaxMs: number; + unlimitedDelayMs: number; + sleepSliceMs: number; +} + +interface QueueWaitPolicy { + responseMaxWaitMs: number; + deepResearchMaxWaitMs: number; + deepResearchStaleMs: number; + checkIntervalMs: number; +} + interface QueueJob { tabId: number; provider: string; @@ -46,6 +75,17 @@ interface QueueJob { currentCommandNumber: number; currentPhase: QueueJobPhase; deliveryTimeoutAttempts: number; + retryAttemptCount?: number; + lastRetryableReason?: string; + retryClass?: QueueFailureClass; + retryMode?: QueueRetryMode; + nextRetryDelayMs?: number; + nextRetryAt?: number; + retryExhausted?: boolean; + waitStartedAt?: number; + lastResearchProgressAt?: number; + sawDeepResearch?: boolean; + sawGenerating?: boolean; startedAt: number; updatedAt: number; } @@ -71,6 +111,17 @@ interface DurableQueueJob { currentPhase: QueueJobPhase; waitForIdleBeforeSend: boolean; deliveryTimeoutAttempts: number; + retryAttemptCount?: number; + lastRetryableReason?: string; + retryClass?: QueueFailureClass; + retryMode?: QueueRetryMode; + nextRetryDelayMs?: number; + nextRetryAt?: number; + retryExhausted?: boolean; + waitStartedAt?: number; + lastResearchProgressAt?: number; + sawDeepResearch?: boolean; + sawGenerating?: boolean; startedAt: number; updatedAt: number; } @@ -101,6 +152,17 @@ interface RunningJobSnapshot { currentPhase: QueueJobPhase; waitForIdleBeforeSend: boolean; deliveryTimeoutAttempts: number; + retryAttemptCount?: number; + lastRetryableReason?: string; + retryClass?: QueueFailureClass; + retryMode?: QueueRetryMode; + nextRetryDelayMs?: number; + nextRetryAt?: number; + retryExhausted?: boolean; + waitStartedAt?: number; + lastResearchProgressAt?: number; + sawDeepResearch?: boolean; + sawGenerating?: boolean; startedAt: number; updatedAt: number; }