fix(tandem): harden Mobi connection recovery and prevent workflow lockouts

gradle/libs.versions.toml

@@ -253,7 +253,7 @@ jetbrains-compose-material-icons-extended = { module = "org.jetbrains.compose.ma
 com-jakewharton-timber = { group = "com.jakewharton.timber", name = "timber", version="5.0.1" }
 com-github-weliem-blessed-android = { group = "com.github.weliem", name = "blessed-android", version="2.4.0" }
 io-github-vanpra-compose-dialogs-datetime = { group = "io.github.vanpra.compose-material-dialogs", name = "datetime", version="0.9.0" }
-com-github-jwoglom-pumpx2-android = { group = "com.github.jwoglom.pumpX2", name = "pumpx2-android", version="v1.9.2" }
+com-github-jwoglom-pumpx2-android = { group = "com.github.jwoglom.pumpX2", name = "pumpx2-android", version="v1.9.3" }
 androidx-compose-runtime-livedata = { group = "androidx.compose.runtime", name = "runtime-livedata" }
 androidx-compose-foundation = { group = "androidx.compose.foundation", name = "foundation" }
 

pump/tandem/src/main/kotlin/app/aaps/pump/tandem/common/comm/TandemPumpCommunicationManager.kt

@@ -62,16 +62,26 @@ class TandemPumpCommunicationManager(
     lateinit var peripheral: BluetoothPeripheral
     @Volatile var connected = false
     @Volatile var errorConnecting = false
+
+    // True while the *host* (command queue / user) has intentionally disconnected, e.g. the
+    // idle-disconnect after waitForDisconnectionInSeconds(). Used to veto pumpX2's internal
+    // auto-reconnect for planned teardowns: resurrecting the link creates a zombie connection
+    // that occupies the pump's single connection slot and blocks the next real connectToPump().
+    @Volatile var plannedDisconnect = false
     var commandRequestModeRunning: Boolean = false
         get() { return inFlightRequests.isNotEmpty() }
 
     var dataStore: TandemUiStateWriter = tandemDataStore
 
     var communicationListener : CommunicationListener? = null
-        set(value) {  if (value==null)
-            operationMode=OperationMode.StandardOperation
-        else
-            operationMode=OperationMode.ExternalListenerOperation
+        set(value) {
+            // NOTE: `field = value` must run UNCONDITIONALLY. It previously sat inside the else
+            // branch, so passing null switched operationMode back to StandardOperation but left
+            // the stale listener attached - responses then routed to a dead listener instance
+            // (observed 2026-08-31 22:56: a HistoryLogStatusResponse was silently dropped and the
+            // history download wedged the CommandExecutor for its full timeout).
+            operationMode = if (value==null) OperationMode.StandardOperation
+                           else OperationMode.ExternalListenerOperation
             field = value
         }
 
@@ -94,6 +104,7 @@ class TandemPumpCommunicationManager(
         val COMMAND_TIMEOUT = 5 * 1000  // 5s (in ms) timeout for receiving pump command response
         val HANDSHAKE_TIMEOUT = 30 * 1000L  // 30s (in ms) timeout for handshake (pairing) connecting flow
         val CONNECT_TIMEOUT = 60 * 1000L // 60s (in ms) timeout for complete connecting flow
+        val CONNECT_TOTAL_TIMEOUT = 180 * 1000L // 180s (in ms) hard cap for the whole connect call
     }
 
 
@@ -106,6 +117,13 @@ class TandemPumpCommunicationManager(
         }
 
         connected = false
+        val wasPlannedDisconnect = plannedDisconnect
+        plannedDisconnect = false
+        aapsLogger.info(
+            TAG,
+            "connect(): plannedDisconnect $wasPlannedDisconnect -> false, preventReconnect=${pumpUtil.preventReconnect}, " +
+                "preventQueueExecution=${pumpStatus.preventQueueExecution}"
+        )
         val connectStartTime = System.currentTimeMillis()
         operationMode = OperationMode.ConnectionMode
         bluetoothHandler!!.startScan()
@@ -130,6 +148,15 @@ class TandemPumpCommunicationManager(
                 aapsLogger.error(TAG, "Connection timeout after ${CONNECT_TIMEOUT / 1000}s, forcing disconnect")
                 errorConnecting = true
                 forceDisconnect(onDisconnect = false, tandemError = null)
+            } else if (System.currentTimeMillis() - connectStartTime > CONNECT_TOTAL_TIMEOUT) {
+                // Hard cap independent of handshakingStartTime: when the link drops BEFORE
+                // service discovery completes, handshakingStartTime stays 0 and neither of
+                // the timeouts above can fire - this loop then blocked forever holding
+                // inConnectMode=true, silently swallowing every future connect attempt.
+                aapsLogger.error(TAG, "Connect total deadline (${CONNECT_TOTAL_TIMEOUT / 1000}s) exceeded - aborting connect")
+                handshakingStartTime = 0L
+                errorConnecting = true
+                forceDisconnect(onDisconnect = false, tandemError = null)
             }
         }
 
@@ -155,6 +182,11 @@ class TandemPumpCommunicationManager(
 
         aapsLogger.info(TAG, "disconnect()")
 
+        // Mark as host-initiated BEFORE tearing down, so the pumpX2 disconnect callback knows
+        // not to auto-reconnect (see onPumpDisconnected).
+        plannedDisconnect = true
+        aapsLogger.info(TAG, "disconnect(): plannedDisconnect=true (preventReconnect=${pumpUtil.preventReconnect} before teardown)")
+
         // Fully tear down the BLE handler: cancel any live peripheral
         // connection, stop the central, and null the pumpx2 singleton so the
         // next connect() builds a fresh handler. blessed's close() alone does
@@ -167,6 +199,11 @@ class TandemPumpCommunicationManager(
 
         publishDisconnectedState()
 
+        aapsLogger.info(
+            TAG,
+            "disconnect(): teardown complete, connected=$connected, preventReconnect=${pumpUtil.preventReconnect}, " +
+                "plannedDisconnect=$plannedDisconnect"
+        )
         return connected
     }
 
@@ -383,6 +420,41 @@ class TandemPumpCommunicationManager(
             this.connected = true
             this.operationMode = OperationMode.StandardOperation
 
+            // This handler runs on EVERY (re)connect, including pumpx2's internal auto-reconnect
+            // after an RF drop — a path that bypasses connect(). Publish the connection fact here
+            // so pumpConnectedFlow can never go stale-false while the link is demonstrably up.
+            // (Stale false made PumpAvailabilitySync hold Unknown and fast-fail all delivery ops.)
+            pumpStatus.pumpConnectedFlow.value = true
+            dataStore.postPumpConnected(true)
+            val preventReconnectBefore = pumpUtil.preventReconnect
+            aapsLogger.info(
+                TAG,
+                "Handshake complete: pumpConnectedFlow=true [preventReconnect=$preventReconnectBefore, " +
+                    "preventQueueExecution=${pumpStatus.preventQueueExecution}, errorType=${pumpUtil.errorType}]"
+            )
+            // tryToReconnectToPump() tears the link down through forceResetBluetoothHandler(), which
+            // sets preventReconnect=true. Once the handshake succeeds the link is demonstrably back,
+            // so that "do not reconnect" latch must not outlive it - otherwise isBusy() and
+            // isInPreventConnectMode() stay true while connected, the queue logs "busy" forever and
+            // the scheduled status refresh is suppressed (observed 2026-10-26 16:43:50).
+            if (preventReconnectBefore) {
+                pumpUtil.preventReconnect = false
+                aapsLogger.info(TAG, "Handshake complete: cleared preventReconnect latch (link is up again)")
+            }
+
+            // A successful (re)handshake also invalidates any error state recorded against the
+            // previous link, so a latched error cannot outlive the connection it described.
+            if (pumpUtil.errorType != null || pumpUtil.errorDetails.isNotEmpty()) {
+                val latchedType = pumpUtil.errorType
+                val clearedDetails = pumpUtil.errorDetails
+                pumpUtil.clearErrorDetails()
+                rxBus.send(EventPumpFragmentValuesChanged(PumpUpdateFragmentType.PumpStatus))
+                aapsLogger.info(
+                    TAG,
+                    "Cleared error details after successful reconnect (core errorType latch left as-is: $latchedType) details='${clearedDetails.take(100)}'"
+                )
+            }
+
         } else if (message is PumpVersionResponse) {
             dataStore.postPumpVersionResponse(message)
         }
@@ -488,20 +560,35 @@ class TandemPumpCommunicationManager(
 
     override fun onPumpDisconnected(peripheral: BluetoothPeripheral?, status: HciStatus?): Boolean {
         aapsLogger.error(TAG, "Pump Disconnected: $status")
+        // Veto pumpX2's auto-reconnect when the queue/host intentionally disconnected:
+        // the command queue owns the connection lifecycle, and an autonomous reconnect here
+        // creates a zombie link that blocks the next real connectToPump(). Genuine RF drops
+        // still return true and keep the 250ms auto-reconnect.
+        val shouldReconnect = !plannedDisconnect
+        aapsLogger.info(TAG, "onPumpDisconnected: plannedDisconnect=$plannedDisconnect -> pumpX2 reconnect=$shouldReconnect")
         forceDisconnect(onDisconnect = true,
                         hciStatus = status)
-        return super.onPumpDisconnected(peripheral, status)
+        plannedDisconnect = false
+        return shouldReconnect && super.onPumpDisconnected(peripheral, status)
     }
 
     fun forceDisconnect(onDisconnect: Boolean, hciStatus: HciStatus? = null,  tandemError: TandemError? = null) {
         aapsLogger.error(TAG, "forceDisconnect: onDisconnect=${onDisconnect}" +
-            ", hciStatus=${hciStatus}, tandemError=${tandemError}")
+            ", hciStatus=${hciStatus}, tandemError=${tandemError}, plannedDisconnect=$plannedDisconnect")
         this.pumpStatus.disconnectData = DisconnectDataDto(onDisconnect = onDisconnect,
                                                            hciStatus = hciStatus,
                                                            tandemError = tandemError)
         pumpUtil.driverStatus = PumpDriverState.Disconnected
+        // Unplanned drops must drop the connection flow too: PumpAvailabilitySync maps
+        // (connected=false, running) to Unknown, which is the conservative gate state until
+        // the link (and a status read) come back.
+        publishDisconnectedState()
         rxBus.send(EventPumpFragmentValuesChanged(PumpUpdateFragmentType.PumpStatus))
-        tandemConnectionFixer.startConnectionFix()
+        if (!plannedDisconnect) {
+            // The ConnectionFixer recovers from *unplanned* failures; during a planned
+            // teardown it would just re-connect the link we were asked to close.
+            tandemConnectionFixer.startConnectionFix()
+        }
     }
 
 

pump/tandem/src/main/kotlin/app/aaps/pump/tandem/common/comm/history/HistoryRetriever.kt

@@ -139,17 +139,51 @@ class HistoryRetriever(
         downloadRunning = true
         startDataRetrieval()
 
+        // Progress watchdog: every message the per-download TandemUICommunication receives counts
+        // as progress. A stalled stream (RF drop mid-chunk, or a misrouted response that a stale
+        // listener instance dropped - both observed 2026-08-31/09-01) previously wedged this loop,
+        // and with it the whole CommandExecutor, for the full 60-min timeout. Un-wedge after
+        // WATCHDOG_TIMEOUT_MS of no messages instead.
+        var lastMessageCount = -1
+        var lastProgressMs = System.currentTimeMillis()
+        var firstMessageAtMs = -1L
+        var loopCount = 0
+        aapsLogger.info(TAG, "${historyPrefix}history watchdog: ${WATCHDOG_TIMEOUT_MS / 1000}s without messages aborts, overall timeout ${(timeoutTime - startTime) / 60_000}min")
+
         while (downloadRunning) {
             aapsLogger.debug("${historyPrefix}download running")
             pumpUtil.sleepSeconds(5)
+            loopCount++
+
+            if (communication.messageCount != lastMessageCount) {
+                if (firstMessageAtMs < 0L) {
+                    firstMessageAtMs = System.currentTimeMillis()
+                    aapsLogger.info(TAG, "${historyPrefix}first message after ${firstMessageAtMs - startTime}ms (count=${communication.messageCount})")
+                }
+                lastMessageCount = communication.messageCount
+                lastProgressMs = System.currentTimeMillis()
+                if (loopCount % 12 == 0) {
+                    aapsLogger.info(TAG, "${historyPrefix}progress: messages=$lastMessageCount, elapsed=${System.currentTimeMillis() - startTime}ms")
+                }
+            } else if (System.currentTimeMillis() - lastProgressMs > WATCHDOG_TIMEOUT_MS) {
+                aapsLogger.error(TAG, "${historyPrefix}No messages received for ${WATCHDOG_TIMEOUT_MS / 1000}s (messages=$lastMessageCount) - aborting history download.")
+                downloadRunning = false
+                this.communication.tandemPumpCommunicationManager = null
+                currentRequest = null
+                setSemaphore()
+                endProgress()
+                return false
+            }
 
             if (timeoutTime < System.currentTimeMillis()) {
                 if (communication.messageCount==0) {
                     aapsLogger.error(TAG, "[History] Timeout reached while trying to read history, with no messages read.")
+                    this.communication.tandemPumpCommunicationManager = null
                     return false
                 } else {
                     aapsLogger.error(TAG, "[History] Timeout reached while trying to read history, with ${communication.messageCount} messages read.")
                     downloadRunning = false
+                    this.communication.tandemPumpCommunicationManager = null
                     return false
                 }
             }
@@ -203,9 +237,38 @@ class HistoryRetriever(
         downloadRunning = true
         startDataRetrieval()
 
+        // Same progress watchdog as downloadHistory(): this loop previously had NO timeout at
+        // all, so a stalled silent download wedged the CommandExecutor indefinitely.
+        var lastMessageCount = -1
+        var lastProgressMs = System.currentTimeMillis()
+        var firstMessageAtMs = -1L
+        var loopCount = 0
+        aapsLogger.info(TAG, "${historyPrefix}silent download watchdog: ${WATCHDOG_TIMEOUT_MS / 1000}s without messages aborts")
+
         while(downloadRunning) {
             aapsLogger.debug("${historyPrefix}download running")
             pumpUtil.sleepSeconds(5)
+            loopCount++
+
+            if (communication.messageCount != lastMessageCount) {
+                if (firstMessageAtMs < 0L) {
+                    firstMessageAtMs = System.currentTimeMillis()
+                    aapsLogger.info(TAG, "${historyPrefix}silent download first message after ${firstMessageAtMs - startTime}ms (count=${communication.messageCount})")
+                }
+                lastMessageCount = communication.messageCount
+                lastProgressMs = System.currentTimeMillis()
+                if (loopCount % 12 == 0) {
+                    aapsLogger.info(TAG, "${historyPrefix}silent download progress: messages=$lastMessageCount, elapsed=${System.currentTimeMillis() - startTime}ms")
+                }
+            } else if (System.currentTimeMillis() - lastProgressMs > WATCHDOG_TIMEOUT_MS) {
+                aapsLogger.error(TAG, "${historyPrefix}Silent download: no messages received for ${WATCHDOG_TIMEOUT_MS / 1000}s (messages=$lastMessageCount) - aborting.")
+                downloadRunning = false
+                this.communication.tandemPumpCommunicationManager = null
+                currentRequest = null
+                setSemaphore()
+                endProgress()
+                return mutableListOf()
+            }
         }
 
         this.communication.tandemPumpCommunicationManager = null

pump/tandem/src/main/kotlin/app/aaps/pump/tandem/common/comm/maint/TandemPairingManager.kt

@@ -138,8 +138,17 @@ class TandemPairingManager constructor(
 
     fun stopBluetoothHandler() {
         aapsLogger.info(TAG, "TANDEM-PAIR-DBG: stopBluetoothHandler pairing")
-        tandemPumpUtil.forceResetBluetoothHandler(bluetoothHandler)
+        val handler = bluetoothHandler
+        if (handler == null) {
+            // Nothing owned by the pairing manager. Calling forceReset here would still set
+            // preventReconnect=true and reset the pumpx2 singleton - and it ran AFTER the wizard's
+            // onFinish cleared that flag, so the flag stayed latched until an app restart. The main
+            // TandemPumpCommunicationManager may also own a handler by now; do not touch it.
+            aapsLogger.info(TAG, "TANDEM-PAIR-DBG: stopBluetoothHandler: no handler owned by pairing manager - nothing to tear down")
+            return
+        }
         bluetoothHandler = null
+        tandemPumpUtil.forceResetBluetoothHandler(handler)
     }
 
     // Pair Status

pump/tandem/src/main/kotlin/app/aaps/pump/tandem/common/comm/ui/TandemUICommunication.kt

@@ -87,8 +87,14 @@ class TandemUICommunication(
             return false
         }
 
-        if (!tandemPumpCommunicationManager!!.isListenerEnabled()) {
-            tandemPumpCommunicationManager!!.communicationListener = this
+        // Always (re)bind: the listener from a previous download cycle (or a UI screen) may
+        // still be attached, and stale instances drop history responses (receivedLogStatus etc.
+        // never invoked). Rebinding is idempotent and guarantees the active instance routes
+        // this download's responses into the shared HistoryRetriever.
+        val previousListener = tandemPumpCommunicationManager!!.communicationListener
+        tandemPumpCommunicationManager!!.communicationListener = this
+        if (previousListener != null && previousListener !== this) {
+            aapsLogger.info(TAG, "Rebound communicationListener: previous=${previousListener.javaClass.simpleName} -> new instance (previous listener was still attached)")
         }
 
         if (!tandemPumpCommunicationManager!!.isPumpFullyConnected()) {
@@ -140,6 +146,10 @@ class TandemUICommunication(
                 dataStore.pumpRunningState.value = runningState
                 pumpStatus.pumpRunningState = runningState
 
+                // Ground truth of actual delivery state - command ACKs (e.g. ResumePumping)
+                // can report success while the pump has not actually resumed.
+                dataStore.mirrorBasalStatus.value = message.basalStatusIcon
+
                 pumpStatus.pumpStatusMirror = HomeScreenMirrorDto()
                 pumpStatus.pumpStatusMirror!!.parse(message.cargo)
             }
@@ -226,16 +236,22 @@ class TandemUICommunication(
 
                 if (::historyRetriever.isInitialized) {
                     historyRetriever.receivedStatus(message)
+                } else {
+                    aapsLogger.error(TAG, "HistoryLogStatusResponse dropped: historyRetriever not initialized on this TandemUICommunication instance")
                 }
             }
             is HistoryLogResponse -> {
                 if (::historyRetriever.isInitialized) {
                     historyRetriever.receivedLogResponse(message)
+                } else {
+                    aapsLogger.error(TAG, "HistoryLogResponse dropped: historyRetriever not initialized on this TandemUICommunication instance")
                 }
             }
             is HistoryLogStreamResponse -> {
                 if (::historyRetriever.isInitialized) {
                     historyRetriever.receivedLogStreamResponse(message)
+                } else {
+                    aapsLogger.error(TAG, "HistoryLogStreamResponse dropped: historyRetriever not initialized on this TandemUICommunication instance")
                 }
             }
             is ResumePumpingResponse    -> {