dsd-neo Scanner: power cycle an upstream USB hub when the RTL-SDR wedges

ControlBooth/ControlBooth.sdef

@@ -63,6 +63,26 @@
             <access-group identifier="com.dsward.ControlBooth.pipelines"/>
         </command>
 
+        <command name="usb hub status" code="CBthUbSt"
+                 description="The USB hub port chosen for power cycling a wedged RTL-SDR, the RTL-SDRs on it and the last power cycle's outcome, as JSON.">
+            <cocoa class="UsbHubStatusCommand"/>
+            <access-group identifier="com.dsward.ControlBooth.pipelines"/>
+            <result type="text" description="A JSON object (see USBHubPowerStatus)."/>
+        </command>
+
+        <command name="usb tree" code="CBthUbTr"
+                 description="Like usb hub status, with the USB device tree read fresh.">
+            <cocoa class="UsbTreeCommand"/>
+            <access-group identifier="com.dsward.ControlBooth.pipelines"/>
+            <result type="text" description="A JSON object (see USBHubPowerStatus)."/>
+        </command>
+
+        <command name="power cycle usb hub" code="CBthUbPc"
+                 description="Switch the chosen USB hub port off and on, which powers off every device below it (including RTL-SDRs). Returns at once; the outcome is in usb hub status.">
+            <cocoa class="UsbHubCycleCommand"/>
+            <access-group identifier="com.dsward.ControlBooth.pipelines"/>
+        </command>
+
         <command name="dsd-neo status" code="CBthDnSt"
                  description="The dsd-neo Scanner's state, follow mode, talkgroup being heard and talkgroup lockouts, as JSON.">
             <cocoa class="DsdNeoStatusCommand"/>

ControlBooth/ControlBoothApp.swift

@@ -12,6 +12,7 @@ struct ControlBoothApp: App {
     @State private var airPlaySettingsStore = AirPlaySettingsStore()
     @State private var airPlayReceiverService = AirPlayReceiverService()
     @State private var radioStation = RadioStation()
+    @State private var usbHubPower = USBHubPowerController()
 
     init() {
         LogStore.shared.configure(appName: "ControlBooth")
@@ -27,12 +28,14 @@ struct ControlBoothApp: App {
                 .environment(airPlaySettingsStore)
                 .environment(airPlayReceiverService)
                 .environment(radioStation)
+                .environment(usbHubPower)
                 .background(CloseButtonHider())
                 .onAppear {
                     appDelegate.runner = runner
                     appDelegate.store = store
                     appDelegate.airPlayReceiverService = airPlayReceiverService
                     appDelegate.radioStation = radioStation
+                    appDelegate.usbHubPower = usbHubPower
                     radioStation.configure(runner: runner, airPlay: airPlayReceiverService)
                     // A copy hosting unit tests must leave the real app's world
                     // alone: no database seeding, scheduled starts, or AirPlay
@@ -71,6 +74,8 @@ struct ControlBoothApp: App {
             }
             CommandMenu("Commands") {
                 StopPipelineMenuButton(store: store, runner: runner)
+                Divider()
+                USBHubPowerMenuButtons(controller: usbHubPower)
             }
         }
 
@@ -79,6 +84,12 @@ struct ControlBoothApp: App {
         }
         .windowResizability(.contentSize)
 
+        Window("USB Tree", id: "usb-tree") {
+            USBTreeView()
+                .environment(usbHubPower)
+        }
+        .defaultSize(width: 640, height: 480)
+
         Window("Logs", id: "logs") {
             LogViewerView()
         }
@@ -126,6 +137,23 @@ private struct StopPipelineMenuButton: View {
     }
 }
 
+/// Commands > USB Tree… and Power Cycle USB Hub…: the second opens the USB
+/// Tree window, which asks to confirm before anything loses power.
+private struct USBHubPowerMenuButtons: View {
+    let controller: USBHubPowerController
+    @Environment(\.openWindow) private var openWindow
+
+    var body: some View {
+        Button("USB Tree…") { openWindow(id: "usb-tree") }
+            .keyboardShortcut("u", modifiers: [.command, .shift])
+        Button("Power Cycle USB Hub…") {
+            controller.confirmationRequested = true
+            openWindow(id: "usb-tree")
+        }
+        .disabled(!controller.canCycle)
+    }
+}
+
 private struct RevealRecordingsFolderButton: View {
     var body: some View {
         Button("Show Recordings Folder in Finder") {
@@ -153,6 +181,7 @@ final class AppDelegate: NSObject, NSApplicationDelegate {
     var store: PipelineStore?
     var airPlayReceiverService: AirPlayReceiverService?
     var radioStation: RadioStation?
+    var usbHubPower: USBHubPowerController?
 
     override init() {
         super.init()

ControlBooth/Services/DsdNeo/DsdNeoScanner.swift

@@ -22,6 +22,13 @@ final class DsdNeoScanner {
         case failed(String)
     }
 
+    /// What a hardware recovery (a USB hub power cycle) achieved.
+    enum HardwareRecoveryResult: Sendable, Equatable {
+        /// The RTL-SDR is back, at this USB index.
+        case recovered(index: UInt32)
+        case failed(String)
+    }
+
     private(set) var state: State = .idle
     /// The pipeline this scanner is feeding, while active.
     private(set) var pipelineID: Int64?
@@ -71,6 +78,12 @@ final class DsdNeoScanner {
     /// wedged RTL-SDR or a dead antenna, so after a few it gives up.
     @ObservationIgnored private var restartsWithoutSignal = 0
     static let maxRestartsWithoutSignal = 3
+    /// Power cycles the USB hub the RTL-SDR is on (given its serial) once the
+    /// restarts above have failed, when the user has set that up. It reports
+    /// the dongle's new USB index, which can differ from before.
+    @ObservationIgnored var hardwareRecovery: (@Sendable (String) async -> HardwareRecoveryResult)?
+    /// One power cycle per run: a second wouldn't fix what the first didn't.
+    @ObservationIgnored private var hardwareRecoveryTried = false
     @ObservationIgnored private var pendingGrant: (talkgroup: Int, encrypted: Bool)?
     @ObservationIgnored private var listNames: [Int: String] = [:]
     @ObservationIgnored private var sitesDirty = false
@@ -128,6 +141,7 @@ final class DsdNeoScanner {
         lastRestartReason = nil
         budget = DsdNeoRestartBudget()
         restartsWithoutSignal = 0
+        hardwareRecoveryTried = false
         nowPlayingTalkgroup = nil
         nowPlayingText = nil
         terminalBacklog.removeAll()
@@ -191,6 +205,7 @@ final class DsdNeoScanner {
         self.run = run
         budget = DsdNeoRestartBudget()
         restartsWithoutSignal = 0
+        hardwareRecoveryTried = false
         restartCount = 0
         lastRestartReason = nil
         nowPlayingTalkgroup = nil
@@ -448,9 +463,15 @@ final class DsdNeoScanner {
     private func restart(because reason: String) {
         guard run != nil else { return }
         guard restartsWithoutSignal < Self.maxRestartsWithoutSignal else {
+            if let hardwareRecovery, !hardwareRecoveryTried, let serial = run?.settings.rtlSerial, !serial.isEmpty {
+                hardwareRecoveryTried = true
+                powerCycleHub(serial: serial, recovery: hardwareRecovery, reason: reason)
+                return
+            }
             let device = run.map { $0.settings.rtlSerial.isEmpty ? "" : " (\($0.settings.rtlSerial))" } ?? ""
             let type = run?.settings.effectiveSystemType ?? .p25Trunked
             fail("dsd-neo has had no \(type.protocolName) signal through \(Self.maxRestartsWithoutSignal) restarts. "
+                 + (hardwareRecoveryTried ? "Power cycling its USB hub didn't help. " : "")
                  + "The RTL-SDR\(device) may have stopped delivering samples: unplug it, plug it back in, "
                  + "and start the scanner again. If that doesn't help, check the antenna and the "
                  + "\(type.channelTitle.lowercased()) frequency.")
@@ -485,6 +506,46 @@ final class DsdNeoScanner {
         }
     }
 
+    /// Stops dsd-neo, power cycles the RTL-SDR's USB hub, and launches dsd-neo
+    /// again on the dongle's new index. Failing here stops the pipeline.
+    private func powerCycleHub(serial: String,
+                               recovery: @escaping @Sendable (String) async -> HardwareRecoveryResult,
+                               reason: String) {
+        LogStore.shared.log(.warning, source: "DsdNeoScanner",
+                            "\(reason); power cycling the USB hub of RTL-SDR \(serial)")
+        lastRestartReason = reason
+        nowPlayingTalkgroup = nil
+        nowPlayingText = nil
+        state = .restarting("power cycling the USB hub")
+        generation += 1
+        let current = generation
+        process?.terminateAndWait()
+        process = nil
+        readEventLog()
+        closeStderrLog()
+        Task { [weak self] in
+            let result = await recovery(serial)
+            guard let self, self.generation == current, self.run != nil,
+                  case .restarting = self.state else { return }
+            switch result {
+            case .recovered(let index):
+                LogStore.shared.log(.info, source: "DsdNeoScanner",
+                                    "RTL-SDR \(serial) is back at USB index \(index); restarting dsd-neo")
+                self.run?.rtlIndex = index
+                self.restartsWithoutSignal = 0
+                self.budget = DsdNeoRestartBudget()
+                do {
+                    try self.launch()
+                } catch {
+                    self.fail("could not restart dsd-neo: \(error)")
+                }
+            case .failed(let message):
+                self.fail("Power cycling the USB hub failed: \(message) Unplug the RTL-SDR (\(serial)), "
+                          + "plug it back in, and start the scanner again.")
+            }
+        }
+    }
+
     private func fail(_ message: String) {
         LogStore.shared.log(.error, source: "DsdNeoScanner", message)
         stop()

ControlBooth/Services/PipelineRunner.swift

@@ -452,6 +452,10 @@ final class PipelineRunner {
             saved.systemID = id
             saved.save()
         }
+        // A wedged RTL-SDR only recovers when it loses power; with an
+        // upstream hub chosen for that, the scanner may cycle it before giving up.
+        let hubPower = USBHubPowerSettings.load()
+        dsdNeoScanner.hardwareRecovery = USBHubRecovery.recovery(for: hubPower)
         dsdNeoScanner.onFatal = { [weak self] _ in
             guard let self, let manager = self.managers[id] else { return }
             manager.terminateAndWait()

ControlBooth/Services/ScriptingCommands.swift

@@ -313,3 +313,66 @@ nonisolated final class StopRadioCommand: NSScriptCommand {
         }
     }
 }
+
+
+// MARK: USB hub power
+
+@MainActor
+private func usbHubPower(reportingTo command: NSScriptCommand) -> USBHubPowerController? {
+    guard let controller = AppDelegate.shared?.usbHubPower else {
+        command.scriptErrorNumber = NSInternalScriptError
+        command.scriptErrorString = "ControlBooth is still launching; try again."
+        return nil
+    }
+    return controller
+}
+
+@MainActor
+private func usbHubPowerJSON(_ controller: USBHubPowerController, refreshing: Bool) -> String {
+    if refreshing { controller.refresh() }
+    let encoder = JSONEncoder()
+    encoder.outputFormatting = [.sortedKeys]
+    let data = (try? encoder.encode(controller.status(includingTree: refreshing))) ?? Data("{}".utf8)
+    return String(decoding: data, as: UTF8.self)
+}
+
+/// 'usb hub status': the chosen power-cycle port and the last outcome, as JSON.
+@objc(UsbHubStatusCommand)
+nonisolated final class UsbHubStatusCommand: NSScriptCommand {
+    override func performDefaultImplementation() -> Any? {
+        MainActor.assumeIsolated {
+            guard let controller = usbHubPower(reportingTo: self) else { return nil }
+            return usbHubPowerJSON(controller, refreshing: false)
+        }
+    }
+}
+
+/// 'usb tree': the same, with the USB tree read fresh.
+@objc(UsbTreeCommand)
+nonisolated final class UsbTreeCommand: NSScriptCommand {
+    override func performDefaultImplementation() -> Any? {
+        MainActor.assumeIsolated {
+            guard let controller = usbHubPower(reportingTo: self) else { return nil }
+            return usbHubPowerJSON(controller, refreshing: true)
+        }
+    }
+}
+
+/// 'power cycle usb hub': starts cycling the chosen port and returns at once
+/// (the cycle takes several seconds); the outcome is in 'usb hub status'.
+@objc(UsbHubCycleCommand)
+nonisolated final class UsbHubCycleCommand: NSScriptCommand {
+    override func performDefaultImplementation() -> Any? {
+        MainActor.assumeIsolated {
+            guard let controller = usbHubPower(reportingTo: self) else { return nil }
+            controller.refresh()
+            do {
+                try controller.startCycle()
+            } catch {
+                scriptErrorNumber = NSInternalScriptError
+                scriptErrorString = "\(error)"
+            }
+            return nil
+        }
+    }
+}