Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -454,9 +454,9 @@ extension SuggestionCoordinator {
// round-trip the speculation existed to skip. Stand down and let it land; `apply`
// validates via the same signature. Any divergence falls through to the normal
// reschedule, whose newer work id retires the speculation automatically.
if let expected = pendingSpeculativeContext,
currentContext?.sessionIdentity == expected.sessionIdentity,
currentContext?.contentSignature == expected.contentSignature {
if let expected = pendingSpeculativeContext, let currentContext,
currentContext.sessionIdentity.continues(expected.sessionIdentity),
currentContext.contentSignature == expected.contentSignature {
logStage(
"speculation-validated",
workID: currentWorkID,
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -863,9 +863,10 @@ extension SuggestionCoordinator {
// not published yet, so its generation predates the live one by construction. When the
// live content now matches the signature the speculation was built against, the bet paid
// off and the result is exactly current.
let isPaidOffSpeculation = pendingSpeculativeContext != nil
&& pendingSpeculativeContext?.sessionIdentity == liveContext.sessionIdentity
&& pendingSpeculativeContext?.contentSignature == liveContext.contentSignature
let isPaidOffSpeculation = pendingSpeculativeContext.map { speculated in
liveContext.sessionIdentity.continues(speculated.sessionIdentity)
&& liveContext.contentSignature == speculated.contentSignature
} ?? false
if isPaidOffSpeculation {
pendingSpeculativeContext = nil
}
Expand Down
28 changes: 28 additions & 0 deletions Cotabby/Models/Focus/FocusModels.swift
Original file line number Diff line number Diff line change
Expand Up @@ -26,6 +26,34 @@ nonisolated struct FocusedInputSessionIdentity: Hashable, Sendable {
let focusedURLString: String?
let windowTitle: String?
let fieldPlaceholder: String?

/// True when this identity, read from a live poll, still describes the writing session
/// `previous` was captured in.
///
/// This is the comparison every consumer of a *live* snapshot must use instead of `==`. The
/// process, bundle and focus sequence are Cotabby's own bookkeeping and always known, so they
/// must match exactly. The three surface facts are bounded AX reads repeated on every poll,
/// each under `AXHelper`'s 50 ms messaging timeout. A host busy processing a burst of synthetic
/// keystrokes (Chromium during rapid Tab accepts) times one out, and the fact arrives as nil.
/// Nil is an unreadable fact, not a navigation: navigation is one known value replaced by a
/// different known value. Reading nil as "changed" tore the active suggestion down mid-burst as
/// a "focused field" change, and the next Tab reached the page as a real Tab.
func continues(_ previous: FocusedInputSessionIdentity) -> Bool {
processIdentifier == previous.processIdentifier
&& bundleIdentifier == previous.bundleIdentifier
&& focusChangeSequence == previous.focusChangeSequence
&& Self.surfaceFactsAgree(focusedURLString, previous.focusedURLString)
&& Self.surfaceFactsAgree(windowTitle, previous.windowTitle)
&& Self.surfaceFactsAgree(fieldPlaceholder, previous.fieldPlaceholder)
}

/// Two reads of one surface fact agree unless both are known and differ. Shared with
/// `FocusedInputPollingSignature` so the tracker's navigation signal and the session identity
/// consumers rely on cannot drift apart on what an unreadable fact means.
static func surfaceFactsAgree(_ lhs: String?, _ rhs: String?) -> Bool {
guard let lhs, let rhs else { return true }
return lhs == rhs
}
}

/// Describes how trustworthy the resolved caret rect is.
Expand Down
7 changes: 5 additions & 2 deletions Cotabby/Services/Focus/FocusTracker.swift
Original file line number Diff line number Diff line change
Expand Up @@ -374,8 +374,11 @@ final class FocusTracker {
let nextSignature = FocusedInputPollingSignature(context: context)
if let lastFocusedInputSignature, nextSignature.continuesField(of: lastFocusedInputSignature) {
// Same field, possibly resized in place. Track its latest frame so later growth is
// compared with the current edges, without opening a new writing session.
self.lastFocusedInputSignature = nextSignature
// compared with the current edges, without opening a new writing session. A surface
// fact this poll failed to read (a title or URL query that hit the AX timeout while the
// host was busy) keeps its last known value, so a later genuinely different value is
// still recognized as navigation rather than compared against a blank.
self.lastFocusedInputSignature = nextSignature.carryingKnownSurfaceFacts(from: lastFocusedInputSignature)
return FocusCaptureResult(snapshot: firstPassSnapshot, didChangeFocusedInput: false)
}

Expand Down
6 changes: 5 additions & 1 deletion Cotabby/Services/Suggestion/State/ContextBuffer.swift
Original file line number Diff line number Diff line change
Expand Up @@ -20,7 +20,11 @@ final class ContextBuffer {

// Identical drafts in two chat tabs are different requests. Session identity includes
// navigation but excludes volatile AX tokens, so a wrapper refresh alone stays harmless.
if snapshot.sessionIdentity != lastSessionIdentity || signature != lastSignature {
// A surface fact one poll failed to read is not navigation either: bumping the generation
// for it would retire an in-flight suggestion for text that never changed. Real navigation
// still bumps, because FocusTracker advances the focus sequence for it.
let continuesSession = lastSessionIdentity.map { snapshot.sessionIdentity.continues($0) } ?? false
if !continuesSession || signature != lastSignature {
nextGeneration &+= 1
}

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -96,7 +96,7 @@ final class SuggestionInteractionState {
return false
}

return currentContext.sessionIdentity != focusedContext.sessionIdentity
return !focusedContext.sessionIdentity.continues(currentContext.sessionIdentity)
}

/// Reconciles the currently active session against the latest AX snapshot and stores the
Expand Down Expand Up @@ -244,7 +244,7 @@ final class SuggestionInteractionState {
sessionForAcceptance = reconciledSession
}
} else {
guard liveContext.sessionIdentity == activeSession.baseContext.sessionIdentity else {
guard liveContext.sessionIdentity.continues(activeSession.baseContext.sessionIdentity) else {
return SessionValidation(session: nil, failureReason: "Key passed through because the focused field changed.")
}

Expand Down
20 changes: 14 additions & 6 deletions Cotabby/Services/Visual/VisualContextCoordinator.swift
Original file line number Diff line number Diff line change
Expand Up @@ -77,7 +77,7 @@ final class VisualContextCoordinator {
// Surface facts can change even when an app reuses its composer and AX handle. Drop the
// old excerpt synchronously; the settle delay below must never expose another chat's text.
if let previous = activeSessionIdentity ?? pendingStartContext?.sessionIdentity,
previous != snapshotContext.sessionIdentity {
!snapshotContext.sessionIdentity.continues(previous) {
cancel(resetState: true)
}
// Coalesce repeated calls for the same field (active or already pending) so a flapping focus
Expand Down Expand Up @@ -172,7 +172,7 @@ final class VisualContextCoordinator {
guard !Task.isCancelled, activeAugmentationSession?.sessionID == session.sessionID else { return }
guard screenRecordingPermissionProvider(), let currentContext,
currentContext.identity == snapshotContext.identity,
currentContext.sessionIdentity == snapshotContext.sessionIdentity, !currentContext.isSecure else {
currentContext.sessionIdentity.continues(snapshotContext.sessionIdentity), !currentContext.isSecure else {
cancel(resetState: true)
return
}
Expand Down Expand Up @@ -205,9 +205,10 @@ final class VisualContextCoordinator {
if let provider = refreshContextProvider {
let liveContext = provider()
guard activeAugmentationSession?.sessionID == session.sessionID else { return }
guard screenRecordingPermissionProvider(), liveContext?.identity == snapshotContext.identity,
liveContext?.sessionIdentity == snapshotContext.sessionIdentity,
liveContext?.isSecure == false else {
guard screenRecordingPermissionProvider(), let liveContext,
liveContext.identity == snapshotContext.identity,
liveContext.sessionIdentity.continues(snapshotContext.sessionIdentity),
!liveContext.isSecure else {
cancel(resetState: true)
return
}
Expand Down Expand Up @@ -253,7 +254,8 @@ final class VisualContextCoordinator {
let context = liveContext,
context.elementIdentifier == session.elementIdentifier,
context.focusChangeSequence == session.focusChangeSequence,
context.sessionIdentity == self.activeSessionIdentity,
let activeSessionIdentity = self.activeSessionIdentity,
context.sessionIdentity.continues(activeSessionIdentity),
!context.isSecure else {
self.cancel(resetState: true)
return
Expand Down Expand Up @@ -291,6 +293,12 @@ final class VisualContextCoordinator {
/// visual-context session still belongs to that same field.
func excerpt(for context: FocusedInputContext) -> String? {
expireExcerptIfNeeded()
// Exact identity here, unlike the session-keeping checks above. Those tolerate a surface
// fact the poll failed to read so a busy host does not cancel the capture; this call hands
// screen text to a prompt, and a chat switch that reuses the composer, frame and URL while
// its new title is momentarily unreadable would otherwise condition the new chat's request
// on the previous chat's excerpt. Withholding the excerpt for that one poll costs a request
// its screen context; the session and its capture survive for the next readable poll.
guard let activeAugmentationSession,
activeSessionIdentity == context.sessionIdentity,

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

P2 Readable facts hide visual excerpts If a visual-context session starts while a title, URL, or placeholder is unreadable, its stored identity keeps that nil value. When the same field becomes readable, this exact comparison rejects the live context. Refresh can keep capturing screenshots, but their excerpts remain unavailable to suggestion requests for the rest of the session, so those requests lose screen context.

Note: If this suggestion doesn't match your team's coding style, reply to this and let me know. I'll remember it for next time!

Fix in Codex Fix in Claude Code

activeAugmentationSession.elementIdentifier == context.elementIdentifier,
Expand Down
53 changes: 51 additions & 2 deletions Cotabby/Support/Focus/FocusedInputPollingSignature.swift
Original file line number Diff line number Diff line change
Expand Up @@ -33,18 +33,67 @@ nonisolated struct FocusedInputPollingSignature: Equatable {
)
}

private init(
bundleIdentifier: String,
processIdentifier: Int32,
role: String,
subrole: String?,
fieldAnchor: FieldAnchor,
windowTitle: String?,
focusedURLString: String?,
fieldPlaceholder: String?
) {
self.bundleIdentifier = bundleIdentifier
self.processIdentifier = processIdentifier
self.role = role
self.subrole = subrole
self.fieldAnchor = fieldAnchor
self.windowTitle = windowTitle
self.focusedURLString = focusedURLString
self.fieldPlaceholder = fieldPlaceholder
}

/// True when this poll still observes the field `previous` described. Every identity fact must
/// match, but the frame may resize in place: a chat composer grows when a line wraps, keeping
/// its left edge, width, and either its top or bottom edge. Treating that as navigation would
/// start a new writing session and discard the visible suggestion on every wrap. Two distinct
/// fields stacked at the same x and width still differ at both edges.
///
/// The surface facts (title, URL, placeholder) are compared with the same rule the session
/// identity uses: a fact that read as nil this poll is unreadable, not different. Each is a
/// bounded AX read under a 50 ms timeout, and a host busy with a burst of synthetic keystrokes
/// drops one now and then. Treating that as navigation advanced the focus sequence and retired
/// the active suggestion in the middle of rapid Tab accepts.
func continuesField(of previous: FocusedInputPollingSignature) -> Bool {
guard bundleIdentifier == previous.bundleIdentifier, processIdentifier == previous.processIdentifier,
role == previous.role, subrole == previous.subrole, windowTitle == previous.windowTitle,
focusedURLString == previous.focusedURLString, fieldPlaceholder == previous.fieldPlaceholder
role == previous.role, subrole == previous.subrole,
FocusedInputSessionIdentity.surfaceFactsAgree(windowTitle, previous.windowTitle),
FocusedInputSessionIdentity.surfaceFactsAgree(focusedURLString, previous.focusedURLString),
FocusedInputSessionIdentity.surfaceFactsAgree(fieldPlaceholder, previous.fieldPlaceholder)
else { return false }
return fieldAnchor.continues(previous.fieldAnchor)
}

/// This poll's signature with any surface fact it failed to read filled in from `previous`.
///
/// The tracker stores this, not the raw poll, as the field's latest signature. Otherwise a
/// navigation that straddles one unreadable poll would go unnoticed: title A, then nil (which
/// continues A), then title B (which would continue nil). Carrying A forward makes the B poll
/// compare against A and read as the navigation it is. The frame is always this poll's own, so
/// in-place growth keeps being measured from the current edges.
func carryingKnownSurfaceFacts(from previous: FocusedInputPollingSignature?) -> FocusedInputPollingSignature {
guard let previous else { return self }
return FocusedInputPollingSignature(
bundleIdentifier: bundleIdentifier,
processIdentifier: processIdentifier,
role: role,
subrole: subrole,
fieldAnchor: fieldAnchor,
windowTitle: windowTitle ?? previous.windowTitle,
focusedURLString: focusedURLString ?? previous.focusedURLString,
fieldPlaceholder: fieldPlaceholder ?? previous.fieldPlaceholder
)
}
}

private extension FocusedInputPollingSignature {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -106,7 +106,7 @@ nonisolated struct SuggestionContinuationPlan: Equatable, Sendable {
/// to web fields with the same real focus sequence and frame; native fields, missing geometry,
/// and legacy snapshots without a sequence continue to require the exact element identifier.
static func sameFocusedField(_ snapshot: FocusedInputSnapshot, context expected: FocusedInputContext) -> Bool {
guard snapshot.sessionIdentity == expected.sessionIdentity,
guard snapshot.sessionIdentity.continues(expected.sessionIdentity),
snapshot.role == expected.role, snapshot.subrole == expected.subrole else { return false }
if snapshot.elementIdentifier == expected.elementIdentifier { return true }
guard snapshot.isWebContentField, expected.isWebContentField,
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -61,7 +61,9 @@ enum SuggestionSessionReconciler {

// Text may be identical in two conversations. Validate the writing session before even
// the post-insertion AX-lag tolerance, which must never authorize a different target.
guard liveContext.sessionIdentity == session.baseContext.sessionIdentity else {
// `continues`, not `==`: a surface fact the live poll failed to read is not a new target
// (see `FocusedInputSessionIdentity.continues`).
guard liveContext.sessionIdentity.continues(session.baseContext.sessionIdentity) else {
return .invalid("Overlay hidden because the focused field changed.")
}

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -38,7 +38,9 @@ final class SuggestionConversationIsolationTests: XCTestCase {
}

func test_speculativeTextMatchCannotOverrideConversationMismatch() async {
let rig = makeCoordinatorRig()
// The speculation was built while the title was readable; a different known title is
// another conversation (a nil one would only be a timed-out read of the same field).
let rig = makeCoordinatorRig(snapshot: CotabbyTestFixtures.focusedInputSnapshot(windowTitle: "This conversation"))
defer { rig.coordinator.stop() }
let source = rig.interactionState.materializeContext(from: rig.focusProvider.snapshot.context!)
rig.coordinator.pendingSpeculativeContext = source
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -159,6 +159,70 @@ final class SuggestionCoordinatorAcceptanceTests: SuggestionCoordinatorRigTestCa
)
}

/// A browser field whose title and URL were readable when the suggestion was offered.
private var browserComposerSnapshot: FocusedInputSnapshot {
CotabbyTestFixtures.focusedInputSnapshot(
precedingText: "Hello", isWebContentField: true,
focusedURLString: "https://chat.example/one", windowTitle: "Chat one"
)
}

private func setLiveSnapshot(_ context: FocusedInputSnapshot, in rig: CoordinatorRig) {
rig.focusProvider.snapshot = FocusSnapshot(
applicationName: context.applicationName,
bundleIdentifier: context.bundleIdentifier,
capability: .supported,
context: context
)
}

func test_rapidTabIsStillConsumedWhenThePreAcceptRefreshCannotReadTheTitleOrURL() {
// Regression: the accept path refreshes the focus snapshot synchronously, inside the event
// tap, while the host is still busy with the previous synthetic insertion. The window title
// and page URL are re-read on that refresh under a 50 ms AX timeout, and a busy Chromium
// answers one of them with nothing. That nil used to fail the session-identity equality as
// a "focused field" change: the session was torn down and Tab reached the page, moving
// focus to its buttons. Slow Tabs never hit it because the host was idle by the next read.
let rig = retained(makeCoordinatorRig(snapshot: browserComposerSnapshot))
startVisibleSession(in: rig, fullText: " hello world again")

XCTAssertTrue(rig.coordinator.acceptCurrentSuggestion())

// The second Tab's refresh reads the same field, but its title and URL time out. AX has not
// published " hello" yet either.
setLiveSnapshot(CotabbyTestFixtures.focusedInputSnapshot(
precedingText: "Hello", isWebContentField: true, focusedURLString: nil, windowTitle: nil
), in: rig)
XCTAssertTrue(rig.coordinator.acceptCurrentSuggestion(), "An unreadable title is not another field")

// The third Tab's refresh reads both facts again.
setLiveSnapshot(browserComposerSnapshot, in: rig)
XCTAssertTrue(rig.coordinator.acceptCurrentSuggestion())

XCTAssertEqual(rig.inserter.insertedChunks, [" hello", " world", " again"])
XCTAssertFalse(
rig.overlayController.hideReasons.contains { $0.contains("focused field changed") },
"A timed-out surface read must not be mistaken for navigation"
)
}

func test_tabStillPassesThroughWhenTheRefreshReadsAnotherConversation() {
// The tolerance is for facts that could not be read, never for facts that read differently:
// identical draft text in another conversation must still hand Tab back.
let rig = retained(makeCoordinatorRig(snapshot: browserComposerSnapshot))
startVisibleSession(in: rig, fullText: " hello world")
XCTAssertTrue(rig.coordinator.acceptCurrentSuggestion())

setLiveSnapshot(CotabbyTestFixtures.focusedInputSnapshot(
precedingText: "Hello", isWebContentField: true,
focusedURLString: "https://chat.example/two", windowTitle: "Chat two"
), in: rig)

XCTAssertFalse(rig.coordinator.acceptCurrentSuggestion())
XCTAssertEqual(rig.inserter.insertedChunks, [" hello"])
XCTAssertNil(rig.interactionState.activeSession)
}

func test_heldPresentLandingKeepsTheRemainingTailAcceptable() {
let rig = retained(makeCoordinatorRig())
startVisibleSession(in: rig, fullText: " hello world again")
Expand Down
Loading
Loading