Base: the five upstream latency and robustness fixes

app/src/main/AndroidManifest.xml

@@ -2,6 +2,11 @@
 <manifest xmlns:android="http://schemas.android.com/apk/res/android">
 
     <uses-feature android:name="android.hardware.usb.host" />
+    <!-- Ground stations without a touchscreen (headsets, TV boxes, kiosk displays) are
+         driven with a pointer or a gamepad, so do not make a touchscreen a requirement. -->
+    <uses-feature
+        android:name="android.hardware.touchscreen"
+        android:required="false" />
 
     <uses-permission android:name="android.permission.ACCESS_NETWORK_STATE" />
     <uses-permission android:name="android.permission.ACCESS_WIFI_STATE" />
@@ -46,8 +51,10 @@
 
         <activity
             android:name=".VideoActivity"
+            android:configChanges="orientation|screenSize|smallestScreenSize|screenLayout|keyboard|keyboardHidden|navigation"
             android:exported="true"
             android:launchMode="singleInstance"
+            android:resizeableActivity="true"
             android:screenOrientation="sensorLandscape"
             android:theme="@style/Theme.FPVueMat">
 

app/src/main/java/com/openipc/pixelpilot/VideoActivity.java

@@ -134,6 +134,9 @@ public void run() {
     private Timer recordTimer = null;
     private int seconds = 0;
     private boolean isVRMode = false;
+    // Which view the main video is rendered into. SurfaceView is the low latency
+    // default; the TextureView is only used when object detection needs getBitmap().
+    private boolean videoUsesTextureView = false;
     private ConstraintLayout constraintLayout;
     private ConstraintSet constraintSet;
     private WfbNgLink wfbLink;
@@ -165,6 +168,11 @@ public static int getChannel(Context context) {
                 Context.MODE_PRIVATE).getInt("wifi-channel", 161);
     }
 
+    public static boolean getLowLatencySetting(Context context) {
+        return context.getSharedPreferences("general",
+                Context.MODE_PRIVATE).getBoolean("low_latency_decoder", true);
+    }
+
     public static int getBandwidth(Context context) {
         return context.getSharedPreferences("general",
                 Context.MODE_PRIVATE).getInt("bandwidth", 20);
@@ -353,6 +361,7 @@ private void initializeWfbNg() {
     private void initializeVideoPlayers() {
         videoPlayer = new VideoPlayer(this);
         videoPlayer.setIVideoParamsChanged(this);
+        videoPlayer.setLowLatency(getLowLatencySetting(this));
 
         isVRMode = getVRSetting();
 
@@ -368,6 +377,7 @@ private void initializeVideoPlayers() {
      */
     private void setupVRVideoPlayers() {
         binding.mainVideo.setVisibility(View.GONE);
+        binding.mainVideoSurface.setVisibility(View.GONE);
         binding.surfaceViewLeft.getHolder().addCallback(videoPlayer.configure1(0));
         binding.surfaceViewRight.getHolder().addCallback(videoPlayer.configure1(1));
     }
@@ -378,7 +388,22 @@ private void setupVRVideoPlayers() {
     private void setupStandardVideoPlayer() {
         binding.surfaceViewRight.setVisibility(View.GONE);
         binding.surfaceViewLeft.setVisibility(View.GONE);
-        binding.mainVideo.setSurfaceTextureListener(videoPlayer.configureTextureView(0));
+
+        // Object detection reads frames back with TextureView.getBitmap(), which forces
+        // the video through the view hierarchy's GPU composition. Without it a
+        // SurfaceView is used so the video stays on a hardware overlay plane.
+        videoUsesTextureView = getSharedPreferences("general", MODE_PRIVATE)
+                .getBoolean("od_enabled", false);
+
+        if (videoUsesTextureView) {
+            binding.mainVideoSurface.setVisibility(View.GONE);
+            binding.mainVideo.setVisibility(View.VISIBLE);
+            binding.mainVideo.setSurfaceTextureListener(videoPlayer.configureTextureView(0));
+        } else {
+            binding.mainVideo.setVisibility(View.GONE);
+            binding.mainVideoSurface.setVisibility(View.VISIBLE);
+            binding.mainVideoSurface.getHolder().addCallback(videoPlayer.configure1(0));
+        }
     }
 
     // ----------------------------------------------------------------------------
@@ -587,6 +612,9 @@ private void showSettingsMenu(View anchor) {
         // Bandwidth submenu
         setupBandwidthSubMenu(popup);
 
+        // Video submenu
+        setupVideoSubMenu(popup);
+
         // OSD submenu
         setupOSDSubMenu(popup);
 
@@ -671,6 +699,32 @@ private void setupBandwidthSubMenu(PopupMenu popup) {
         }
     }
 
+    /**
+     * Submenu for video decoder options.
+     * "Low latency" sets the MediaCodec low-latency and realtime-priority keys. It is on
+     * by default; decoders that misbehave with those keys can be put back on the stock
+     * pipeline here.
+     */
+    private void setupVideoSubMenu(PopupMenu popup) {
+        SubMenu videoMenu = popup.getMenu().addSubMenu("Video");
+
+        MenuItem lowLatencyItem = videoMenu.add("Low latency");
+        lowLatencyItem.setCheckable(true);
+        lowLatencyItem.setChecked(getLowLatencySetting(this));
+        lowLatencyItem.setOnMenuItemClickListener(item -> {
+            boolean enabled = !item.isChecked();
+            item.setChecked(enabled);
+            getSharedPreferences("general", MODE_PRIVATE).edit()
+                    .putBoolean("low_latency_decoder", enabled).apply();
+            videoPlayer.setLowLatency(enabled);
+            Toast.makeText(this, "Low latency " + (enabled ? "enabled" : "disabled")
+                    + ", applies on next video start.", Toast.LENGTH_SHORT).show();
+            item.setShowAsAction(MenuItem.SHOW_AS_ACTION_COLLAPSE_ACTION_VIEW);
+            item.setActionView(new View(this));
+            return false;
+        });
+    }
+
     /**
      * Submenu handling OSD toggles and locks.
      */
@@ -1541,6 +1595,7 @@ public void onVideoRatioChanged(final int videoW, final int videoH) {
         Log.d(TAG, "Set resolution: " + videoW + "x" + videoH);
 
         updateViewRatio(R.id.mainVideo, lastVideoW, lastVideoH);
+        updateViewRatio(R.id.mainVideoSurface, lastVideoW, lastVideoH);
         updateViewRatio(R.id.surfaceViewLeft, lastVideoW, lastVideoH);
         updateViewRatio(R.id.surfaceViewRight, lastVideoW, lastVideoH);
     }
@@ -1984,6 +2039,15 @@ private void setObjectDetectionEnabled(boolean enabled) {
         isObjectDetectionEnabled = enabled;
         prefs.edit().putBoolean("od_enabled", enabled).apply();
 
+        // Enabling / disabling detection swaps the main video renderer. Handing the
+        // decoder a different surface at runtime would need the receiver lifecycle in
+        // VideoPlayer reworked, so restart instead - same as the VR mode toggle does.
+        if (!isVRMode && enabled != videoUsesTextureView) {
+            Toast.makeText(this, "Restarting to switch video renderer...", Toast.LENGTH_SHORT).show();
+            resetApp();
+            return;
+        }
+
         if (enabled) {
             binding.detectionOverlay.setVisibility(View.VISIBLE);
             startObjectDetectionLoop();
@@ -2003,6 +2067,7 @@ private void restartObjectDetector() {
 
     private void startObjectDetectionLoop() {
         if (isVRMode) return; // Standard mode only
+        if (!videoUsesTextureView) return; // getBitmap() needs the TextureView renderer
         if (objectDetectionExecutor == null) {
             objectDetectionExecutor = Executors.newSingleThreadExecutor();
         }

app/src/main/java/com/openipc/pixelpilot/WfbLinkManager.java

@@ -120,6 +120,10 @@ public Map<String, UsbDevice> getAttachedAdapters() {
 
     public synchronized void refreshAdapters() {
         Map<String, UsbDevice> attachedAdapters = getAttachedAdapters();
+        if (attachedAdapters == null) {
+            Log.e(TAG, "Could not read the usb device filter, skipping adapter refresh.");
+            return;
+        }
 
         boolean missingPermissions = false;
         android.hardware.usb.UsbManager usbManager =
@@ -128,8 +132,13 @@ public synchronized void refreshAdapters() {
             if (!usbManager.hasPermission(entry.getValue())) {
                 binding.tvMessage.setVisibility(View.VISIBLE);
                 binding.tvMessage.setText("No permission for wifi adapter(s) " + entry.getValue().getDeviceName());
+                // Android 14 refuses to deliver a PendingIntent built from an implicit
+                // intent to a runtime registered receiver, so the permission result never
+                // arrives unless the package is set explicitly.
+                Intent permissionIntent = new Intent(WfbLinkManager.ACTION_USB_PERMISSION);
+                permissionIntent.setPackage(context.getPackageName());
                 PendingIntent pendingIntent = PendingIntent.getBroadcast(context, 0,
-                        new Intent(WfbLinkManager.ACTION_USB_PERMISSION), PendingIntent.FLAG_IMMUTABLE);
+                        permissionIntent, PendingIntent.FLAG_IMMUTABLE);
                 usbManager.requestPermission(entry.getValue(), pendingIntent);
                 missingPermissions = true;
             }
@@ -155,8 +164,11 @@ public synchronized void refreshAdapters() {
             if (activeWifiAdapters.containsKey(entry.getKey())) {
                 continue;
             }
-            startAdapter(entry.getValue());
-            activeWifiAdapters.put(entry.getKey(), entry.getValue());
+            // Only track it as active if it actually came up, otherwise a failed adapter
+            // is never retried on the next refresh.
+            if (startAdapter(entry.getValue())) {
+                activeWifiAdapters.put(entry.getKey(), entry.getValue());
+            }
         }
 
         if (activeWifiAdapters.isEmpty()) {
@@ -204,7 +216,10 @@ public synchronized boolean startAdapter(UsbDevice dev) {
         String text = "Starting wfb-ng channel " + wifiChannel + " with " + String.format(
                 "[%04X", dev.getVendorId()) + ":" + String.format("%04X]", dev.getProductId());
         binding.tvMessage.setText(text);
-        wfbLink.start(wifiChannel, bandWidth.getValue(), dev);
+        if (!wfbLink.start(wifiChannel, bandWidth.getValue(), dev)) {
+            binding.tvMessage.setText("Could not open wifi adapter " + dev.getDeviceName());
+            return false;
+        }
         return true;
     }
 }

app/src/main/res/layout/activity_video.xml

@@ -5,10 +5,24 @@
     android:layout_width="match_parent"
     android:layout_height="match_parent">
 
+    <SurfaceView
+        android:id="@+id/mainVideoSurface"
+        android:layout_width="0dp"
+        android:layout_height="0dp"
+        app:layout_constraintBottom_toBottomOf="parent"
+        app:layout_constraintDimensionRatio="16:9"
+        app:layout_constraintEnd_toEndOf="parent"
+        app:layout_constraintStart_toStartOf="parent"
+        app:layout_constraintTop_toTopOf="parent" />
+
+    <!-- Only used when object detection is enabled: TextureView.getBitmap() is how
+         frames are handed to MediaPipe. It costs a GPU composition pass and an extra
+         frame of latency compared to mainVideoSurface, so it stays hidden otherwise. -->
     <TextureView
         android:id="@+id/mainVideo"
         android:layout_width="0dp"
         android:layout_height="0dp"
+        android:visibility="gone"
         app:layout_constraintBottom_toBottomOf="parent"
         app:layout_constraintDimensionRatio="16:9"
         app:layout_constraintEnd_toEndOf="parent"

app/videonative/src/main/cpp/VideoDecoder.cpp

@@ -135,15 +135,6 @@ void VideoDecoder::configureStartDecoder(int idx)
     AMediaFormat* format = AMediaFormat_new();
     AMediaFormat_setString(format, AMEDIAFORMAT_KEY_MIME, MIME.c_str());
 
-    // AMediaFormat_setInt32(format, "low-latency", 1);
-    // AMediaFormat_setInt32(format, "vendor.low-latency.enable", 1);
-    // AMediaFormat_setInt32(format, "vendor.qti-ext-dec-low-latency.enable", 1);
-    // AMediaFormat_setInt32(format, "vendor.hisi-ext-low-latency-video-dec.video-scene-for-low-latency-req", 1);
-    // AMediaFormat_setInt32(format, "vendor.rtc-ext-dec-low-latency.enable", 1);
-
-    // MediaCodec supports two priorities: 0 - realtime, 1 - best effort
-    // AMediaFormat_setInt32(format, "priority", 0);
-
     if (IS_H265)
     {
         h265_configureAMediaFormat(mKeyFrameFinder, format);
@@ -153,6 +144,11 @@ void VideoDecoder::configureStartDecoder(int idx)
         h264_configureAMediaFormat(mKeyFrameFinder, format);
     }
 
+    if (mLowLatency)
+    {
+        writeAndroidPerformanceParams(format);
+    }
+
     MLOGD << "Configuring decoder:" << AMediaFormat_toString(format);
 
     auto status = AMediaCodec_configure(decoder.codec[idx], format, decoder.window[idx], nullptr, 0);

app/videonative/src/main/cpp/VideoDecoder.h

@@ -86,6 +86,10 @@ class VideoDecoder
 
     void registerOnDecodingInfoChangedCallback(DECODING_INFO_CHANGED_CALLBACK decodingInfoChangedCallback);
 
+    // Enable / disable the low-latency and realtime-priority AMediaFormat keys.
+    // Applied the next time the decoder is configured, not to a running decoder.
+    void setLowLatency(bool enabled) { mLowLatency = enabled; }
+
     // If the decoder has been configured, feed NALU. Else search for configuration data and
     // configure as soon as possible
     //  If the input pipe was closed (surface has been removed or is not set yet), only buffer key frames
@@ -109,6 +113,8 @@ class VideoDecoder
 
     std::unique_ptr<std::thread> mCheckOutputThread[2]  = {nullptr, nullptr};
     bool                         USE_SW_DECODER_INSTEAD = false;
+    // Some decoders misbehave with the low-latency keys, so it stays user switchable.
+    std::atomic<bool>            mLowLatency            = true;
     // Holds the AMediaCodec instance, as well as the state (configured or not configured)
     Decoder      decoder{};
     DecodingInfo decodingInfo;

app/videonative/src/main/cpp/VideoPlayer.cpp

@@ -61,16 +61,15 @@ void VideoPlayer::processQueue()
         }
         if (!naluQueue.empty())
         {
-            NALU nalu = naluQueue.front();
             if (framerate == 0)
             {
                 if (latestDecodingInfo.currentFPS <= 0)
                 {
                     continue;
                 }
+                const bool is_h265 = naluQueue.front().is_h265;
                 if (MP4E_STATUS_OK !=
-                    mp4_h26x_write_init(
-                        &mp4wr, mux, latestVideoRatio.width, latestVideoRatio.height, nalu.IS_H265_PACKET))
+                    mp4_h26x_write_init(&mp4wr, mux, latestVideoRatio.width, latestVideoRatio.height, is_h265))
                 {
                     __android_log_print(ANDROID_LOG_DEBUG, TAG, "error: mp4_h26x_write_init failed");
                 }
@@ -82,12 +81,13 @@ void VideoPlayer::processQueue()
                     framerate,
                     latestVideoRatio.width,
                     latestVideoRatio.height,
-                    nalu.IS_H265_PACKET);
+                    is_h265);
             }
+            DvrNalu nalu = std::move(naluQueue.front());
             naluQueue.pop();
             lock.unlock();
             // Process the NALU
-            auto res = mp4_h26x_write_nal(&mp4wr, nalu.getData(), nalu.getSize(), 90000 / framerate);
+            auto res = mp4_h26x_write_nal(&mp4wr, nalu.data.data(), (int) nalu.data.size(), 90000 / framerate);
             if (MP4E_STATUS_OK != res)
             {
                 __android_log_print(ANDROID_LOG_DEBUG, TAG, "mp4_h26x_write_nal failed with %d", res);
@@ -144,11 +144,9 @@ void VideoPlayer::onNewNALU(const NALU& nalu)
     {
         return;
     }
-    // Copy data to write if from a different thread.
-    uint8_t* m_data_copy = new uint8_t[nalu.getSize()];
-    memcpy(m_data_copy, nalu.getData(), nalu.getSize());
-    NALU nalu_(m_data_copy, nalu.getSize(), nalu.IS_H265_PACKET);
-    enqueueNALU(nalu_);
+    // The writer thread outlives this call, so hand it an owning copy.
+    enqueueNALU(DvrNalu{std::vector<uint8_t>(nalu.getData(), nalu.getData() + nalu.getSize()),
+                        nalu.IS_H265_PACKET});
 }
 
 void VideoPlayer::setVideoSurface(JNIEnv* env, jobject surface, jint i)
@@ -318,6 +316,16 @@ extern "C"
         }
     }
 
+    JNI_METHOD(void, nativeSetLowLatency)
+    (JNIEnv* env, jclass jclass1, jlong nativeInstance, jboolean enabled)
+    {
+        VideoPlayer* p = native(nativeInstance);
+        if (p)
+        {
+            p->setLowLatency(enabled);
+        }
+    }
+
     JNI_METHOD(void, nativeSetVideoSurface)
     (JNIEnv* env, jclass jclass1, jlong videoPlayerN, jobject surface, jint index)
     {