diff --git a/app/src/main/AndroidManifest.xml b/app/src/main/AndroidManifest.xml index b13e729a..1ad6bc61 100644 --- a/app/src/main/AndroidManifest.xml +++ b/app/src/main/AndroidManifest.xml @@ -2,6 +2,11 @@ + + @@ -46,8 +51,10 @@ diff --git a/app/src/main/java/com/openipc/pixelpilot/VideoActivity.java b/app/src/main/java/com/openipc/pixelpilot/VideoActivity.java index cacfc1ba..e126fa2e 100644 --- a/app/src/main/java/com/openipc/pixelpilot/VideoActivity.java +++ b/app/src/main/java/com/openipc/pixelpilot/VideoActivity.java @@ -134,6 +134,9 @@ public void run() { private Timer recordTimer = null; private int seconds = 0; private boolean isVRMode = false; + // Which view the main video is rendered into. SurfaceView is the low latency + // default; the TextureView is only used when object detection needs getBitmap(). + private boolean videoUsesTextureView = false; private ConstraintLayout constraintLayout; private ConstraintSet constraintSet; private WfbNgLink wfbLink; @@ -165,6 +168,11 @@ public static int getChannel(Context context) { Context.MODE_PRIVATE).getInt("wifi-channel", 161); } + public static boolean getLowLatencySetting(Context context) { + return context.getSharedPreferences("general", + Context.MODE_PRIVATE).getBoolean("low_latency_decoder", true); + } + public static int getBandwidth(Context context) { return context.getSharedPreferences("general", Context.MODE_PRIVATE).getInt("bandwidth", 20); @@ -353,6 +361,7 @@ private void initializeWfbNg() { private void initializeVideoPlayers() { videoPlayer = new VideoPlayer(this); videoPlayer.setIVideoParamsChanged(this); + videoPlayer.setLowLatency(getLowLatencySetting(this)); isVRMode = getVRSetting(); @@ -368,6 +377,7 @@ private void initializeVideoPlayers() { */ private void setupVRVideoPlayers() { binding.mainVideo.setVisibility(View.GONE); + binding.mainVideoSurface.setVisibility(View.GONE); binding.surfaceViewLeft.getHolder().addCallback(videoPlayer.configure1(0)); binding.surfaceViewRight.getHolder().addCallback(videoPlayer.configure1(1)); } @@ -378,7 +388,22 @@ private void setupVRVideoPlayers() { private void setupStandardVideoPlayer() { binding.surfaceViewRight.setVisibility(View.GONE); binding.surfaceViewLeft.setVisibility(View.GONE); - binding.mainVideo.setSurfaceTextureListener(videoPlayer.configureTextureView(0)); + + // Object detection reads frames back with TextureView.getBitmap(), which forces + // the video through the view hierarchy's GPU composition. Without it a + // SurfaceView is used so the video stays on a hardware overlay plane. + videoUsesTextureView = getSharedPreferences("general", MODE_PRIVATE) + .getBoolean("od_enabled", false); + + if (videoUsesTextureView) { + binding.mainVideoSurface.setVisibility(View.GONE); + binding.mainVideo.setVisibility(View.VISIBLE); + binding.mainVideo.setSurfaceTextureListener(videoPlayer.configureTextureView(0)); + } else { + binding.mainVideo.setVisibility(View.GONE); + binding.mainVideoSurface.setVisibility(View.VISIBLE); + binding.mainVideoSurface.getHolder().addCallback(videoPlayer.configure1(0)); + } } // ---------------------------------------------------------------------------- @@ -587,6 +612,9 @@ private void showSettingsMenu(View anchor) { // Bandwidth submenu setupBandwidthSubMenu(popup); + // Video submenu + setupVideoSubMenu(popup); + // OSD submenu setupOSDSubMenu(popup); @@ -671,6 +699,32 @@ private void setupBandwidthSubMenu(PopupMenu popup) { } } + /** + * Submenu for video decoder options. + * "Low latency" sets the MediaCodec low-latency and realtime-priority keys. It is on + * by default; decoders that misbehave with those keys can be put back on the stock + * pipeline here. + */ + private void setupVideoSubMenu(PopupMenu popup) { + SubMenu videoMenu = popup.getMenu().addSubMenu("Video"); + + MenuItem lowLatencyItem = videoMenu.add("Low latency"); + lowLatencyItem.setCheckable(true); + lowLatencyItem.setChecked(getLowLatencySetting(this)); + lowLatencyItem.setOnMenuItemClickListener(item -> { + boolean enabled = !item.isChecked(); + item.setChecked(enabled); + getSharedPreferences("general", MODE_PRIVATE).edit() + .putBoolean("low_latency_decoder", enabled).apply(); + videoPlayer.setLowLatency(enabled); + Toast.makeText(this, "Low latency " + (enabled ? "enabled" : "disabled") + + ", applies on next video start.", Toast.LENGTH_SHORT).show(); + item.setShowAsAction(MenuItem.SHOW_AS_ACTION_COLLAPSE_ACTION_VIEW); + item.setActionView(new View(this)); + return false; + }); + } + /** * Submenu handling OSD toggles and locks. */ @@ -1541,6 +1595,7 @@ public void onVideoRatioChanged(final int videoW, final int videoH) { Log.d(TAG, "Set resolution: " + videoW + "x" + videoH); updateViewRatio(R.id.mainVideo, lastVideoW, lastVideoH); + updateViewRatio(R.id.mainVideoSurface, lastVideoW, lastVideoH); updateViewRatio(R.id.surfaceViewLeft, lastVideoW, lastVideoH); updateViewRatio(R.id.surfaceViewRight, lastVideoW, lastVideoH); } @@ -1984,6 +2039,15 @@ private void setObjectDetectionEnabled(boolean enabled) { isObjectDetectionEnabled = enabled; prefs.edit().putBoolean("od_enabled", enabled).apply(); + // Enabling / disabling detection swaps the main video renderer. Handing the + // decoder a different surface at runtime would need the receiver lifecycle in + // VideoPlayer reworked, so restart instead - same as the VR mode toggle does. + if (!isVRMode && enabled != videoUsesTextureView) { + Toast.makeText(this, "Restarting to switch video renderer...", Toast.LENGTH_SHORT).show(); + resetApp(); + return; + } + if (enabled) { binding.detectionOverlay.setVisibility(View.VISIBLE); startObjectDetectionLoop(); @@ -2003,6 +2067,7 @@ private void restartObjectDetector() { private void startObjectDetectionLoop() { if (isVRMode) return; // Standard mode only + if (!videoUsesTextureView) return; // getBitmap() needs the TextureView renderer if (objectDetectionExecutor == null) { objectDetectionExecutor = Executors.newSingleThreadExecutor(); } diff --git a/app/src/main/java/com/openipc/pixelpilot/WfbLinkManager.java b/app/src/main/java/com/openipc/pixelpilot/WfbLinkManager.java index 8580be69..0c0390e0 100644 --- a/app/src/main/java/com/openipc/pixelpilot/WfbLinkManager.java +++ b/app/src/main/java/com/openipc/pixelpilot/WfbLinkManager.java @@ -120,6 +120,10 @@ public Map getAttachedAdapters() { public synchronized void refreshAdapters() { Map attachedAdapters = getAttachedAdapters(); + if (attachedAdapters == null) { + Log.e(TAG, "Could not read the usb device filter, skipping adapter refresh."); + return; + } boolean missingPermissions = false; android.hardware.usb.UsbManager usbManager = @@ -128,8 +132,13 @@ public synchronized void refreshAdapters() { if (!usbManager.hasPermission(entry.getValue())) { binding.tvMessage.setVisibility(View.VISIBLE); binding.tvMessage.setText("No permission for wifi adapter(s) " + entry.getValue().getDeviceName()); + // Android 14 refuses to deliver a PendingIntent built from an implicit + // intent to a runtime registered receiver, so the permission result never + // arrives unless the package is set explicitly. + Intent permissionIntent = new Intent(WfbLinkManager.ACTION_USB_PERMISSION); + permissionIntent.setPackage(context.getPackageName()); PendingIntent pendingIntent = PendingIntent.getBroadcast(context, 0, - new Intent(WfbLinkManager.ACTION_USB_PERMISSION), PendingIntent.FLAG_IMMUTABLE); + permissionIntent, PendingIntent.FLAG_IMMUTABLE); usbManager.requestPermission(entry.getValue(), pendingIntent); missingPermissions = true; } @@ -155,8 +164,11 @@ public synchronized void refreshAdapters() { if (activeWifiAdapters.containsKey(entry.getKey())) { continue; } - startAdapter(entry.getValue()); - activeWifiAdapters.put(entry.getKey(), entry.getValue()); + // Only track it as active if it actually came up, otherwise a failed adapter + // is never retried on the next refresh. + if (startAdapter(entry.getValue())) { + activeWifiAdapters.put(entry.getKey(), entry.getValue()); + } } if (activeWifiAdapters.isEmpty()) { @@ -204,7 +216,10 @@ public synchronized boolean startAdapter(UsbDevice dev) { String text = "Starting wfb-ng channel " + wifiChannel + " with " + String.format( "[%04X", dev.getVendorId()) + ":" + String.format("%04X]", dev.getProductId()); binding.tvMessage.setText(text); - wfbLink.start(wifiChannel, bandWidth.getValue(), dev); + if (!wfbLink.start(wifiChannel, bandWidth.getValue(), dev)) { + binding.tvMessage.setText("Could not open wifi adapter " + dev.getDeviceName()); + return false; + } return true; } } diff --git a/app/src/main/res/layout/activity_video.xml b/app/src/main/res/layout/activity_video.xml index cabe29e6..8aa7e519 100644 --- a/app/src/main/res/layout/activity_video.xml +++ b/app/src/main/res/layout/activity_video.xml @@ -5,10 +5,24 @@ android:layout_width="match_parent" android:layout_height="match_parent"> + + + mCheckOutputThread[2] = {nullptr, nullptr}; bool USE_SW_DECODER_INSTEAD = false; + // Some decoders misbehave with the low-latency keys, so it stays user switchable. + std::atomic mLowLatency = true; // Holds the AMediaCodec instance, as well as the state (configured or not configured) Decoder decoder{}; DecodingInfo decodingInfo; diff --git a/app/videonative/src/main/cpp/VideoPlayer.cpp b/app/videonative/src/main/cpp/VideoPlayer.cpp index 6175d585..832383f7 100644 --- a/app/videonative/src/main/cpp/VideoPlayer.cpp +++ b/app/videonative/src/main/cpp/VideoPlayer.cpp @@ -61,16 +61,15 @@ void VideoPlayer::processQueue() } if (!naluQueue.empty()) { - NALU nalu = naluQueue.front(); if (framerate == 0) { if (latestDecodingInfo.currentFPS <= 0) { continue; } + const bool is_h265 = naluQueue.front().is_h265; if (MP4E_STATUS_OK != - mp4_h26x_write_init( - &mp4wr, mux, latestVideoRatio.width, latestVideoRatio.height, nalu.IS_H265_PACKET)) + mp4_h26x_write_init(&mp4wr, mux, latestVideoRatio.width, latestVideoRatio.height, is_h265)) { __android_log_print(ANDROID_LOG_DEBUG, TAG, "error: mp4_h26x_write_init failed"); } @@ -82,12 +81,13 @@ void VideoPlayer::processQueue() framerate, latestVideoRatio.width, latestVideoRatio.height, - nalu.IS_H265_PACKET); + is_h265); } + DvrNalu nalu = std::move(naluQueue.front()); naluQueue.pop(); lock.unlock(); // Process the NALU - auto res = mp4_h26x_write_nal(&mp4wr, nalu.getData(), nalu.getSize(), 90000 / framerate); + auto res = mp4_h26x_write_nal(&mp4wr, nalu.data.data(), (int) nalu.data.size(), 90000 / framerate); if (MP4E_STATUS_OK != res) { __android_log_print(ANDROID_LOG_DEBUG, TAG, "mp4_h26x_write_nal failed with %d", res); @@ -144,11 +144,9 @@ void VideoPlayer::onNewNALU(const NALU& nalu) { return; } - // Copy data to write if from a different thread. - uint8_t* m_data_copy = new uint8_t[nalu.getSize()]; - memcpy(m_data_copy, nalu.getData(), nalu.getSize()); - NALU nalu_(m_data_copy, nalu.getSize(), nalu.IS_H265_PACKET); - enqueueNALU(nalu_); + // The writer thread outlives this call, so hand it an owning copy. + enqueueNALU(DvrNalu{std::vector(nalu.getData(), nalu.getData() + nalu.getSize()), + nalu.IS_H265_PACKET}); } void VideoPlayer::setVideoSurface(JNIEnv* env, jobject surface, jint i) @@ -318,6 +316,16 @@ extern "C" } } + JNI_METHOD(void, nativeSetLowLatency) + (JNIEnv* env, jclass jclass1, jlong nativeInstance, jboolean enabled) + { + VideoPlayer* p = native(nativeInstance); + if (p) + { + p->setLowLatency(enabled); + } + } + JNI_METHOD(void, nativeSetVideoSurface) (JNIEnv* env, jclass jclass1, jlong videoPlayerN, jobject surface, jint index) { diff --git a/app/videonative/src/main/cpp/VideoPlayer.h b/app/videonative/src/main/cpp/VideoPlayer.h index 830d423e..ee67ed22 100644 --- a/app/videonative/src/main/cpp/VideoPlayer.h +++ b/app/videonative/src/main/cpp/VideoPlayer.h @@ -9,6 +9,8 @@ #include #include #include +#include +#include #include "AudioDecoder.h" #include "BufferedPacketQueue.h" #include "UdpReceiver.h" @@ -54,6 +56,8 @@ class VideoPlayer void setForwarding(const std::string& ip, int port, bool enabled); + void setLowLatency(bool enabled) { videoDecoder.setLowLatency(enabled); } + private: void onNewNALU(const NALU& nalu); @@ -74,9 +78,18 @@ class VideoPlayer H26XParser mParser; BufferedPacketQueue mBufferedPacketQueueVideo, mBufferedPacketQueueAudio; + // A NALU is a non-owning view onto the parser's buffer (see NALU.hpp), which is + // reused for the next packet. The DVR writer runs on its own thread, so what gets + // handed over has to own its bytes. + struct DvrNalu + { + std::vector data; + bool is_h265 = false; + }; + // DVR attributes int dvr_fd; - std::queue naluQueue; + std::queue naluQueue; std::mutex mtx; std::condition_variable cv; bool stopFlag = false; @@ -84,11 +97,11 @@ class VideoPlayer int dvr_mp4_fragmentation = 0; uint64_t last_dvr_write = 0; - void enqueueNALU(const NALU& nalu) + void enqueueNALU(DvrNalu&& nalu) { { std::lock_guard lock(mtx); - naluQueue.push(nalu); + naluQueue.push(std::move(nalu)); } cv.notify_one(); } diff --git a/app/videonative/src/main/cpp/helper/AndroidMediaFormatHelper.h b/app/videonative/src/main/cpp/helper/AndroidMediaFormatHelper.h index 28a05897..bc3ffa9d 100644 --- a/app/videonative/src/main/cpp/helper/AndroidMediaFormatHelper.h +++ b/app/videonative/src/main/cpp/helper/AndroidMediaFormatHelper.h @@ -5,26 +5,22 @@ #include #include "../NALU/KeyFrameFinder.hpp" -// Some of these params are only supported on the latest Android versions -// However,writing them has no negative affect on devices with older Android versions -// Note that for example the low-latency key cannot fix any issues like the 'VUI' issue -void writeAndroidPerformanceParams(AMediaFormat* format) +// Decoder tuning that trades pipeline depth for latency. Unknown keys are ignored by +// MediaCodec, so writing all of them is safe on every device / Android version. +static void writeAndroidPerformanceParams(AMediaFormat* format) { - // I think: KEY_LOW_LATENCY is for decoder. But it doesn't really make a difference anyways - static const auto PARAMETER_KEY_LOW_LATENCY = "low-latency"; - AMediaFormat_setInt32(format, PARAMETER_KEY_LOW_LATENCY, 1); - // Lower values mean higher priority - // Works on pixel 3 (look at output format description) - static const auto AMEDIAFORMAT_KEY_PRIORITY = "priority"; - AMediaFormat_setInt32(format, AMEDIAFORMAT_KEY_PRIORITY, 0); - // set operating rate ? - doesn't make a difference - // static const auto AMEDIAFORMAT_KEY_OPERATING_RATE="operating-rate"; - // AMediaFormat_setInt32(format,AMEDIAFORMAT_KEY_OPERATING_RATE,60); - // - // AMEDIAFORMAT_KEY_LOW_LATENCY; - // AMEDIAFORMAT_KEY_LATENCY; - // AMediaFormat_setInt32(format,AMEDIAFORMAT_KEY_LATENCY,0); - // AMediaFormat_setInt32(format,AMEDIAFORMAT_KEY_OPERATING_RATE,0); + // AMEDIAFORMAT_KEY_LOW_LATENCY (API 30+). Tells the decoder to output a frame as soon + // as it is decoded instead of keeping a reorder/output queue. For a live stream that + // never uses B-frames the queue only adds latency. + AMediaFormat_setInt32(format, "low-latency", 1); + // Vendor equivalents for SoCs whose codec does not pick up the AOSP key. Qualcomm is + // the relevant one for most phones and for the Snapdragon XR2 headsets. + AMediaFormat_setInt32(format, "vendor.low-latency.enable", 1); + AMediaFormat_setInt32(format, "vendor.qti-ext-dec-low-latency.enable", 1); + AMediaFormat_setInt32(format, "vendor.hisi-ext-low-latency-video-dec.video-scene-for-low-latency-req", 1); + AMediaFormat_setInt32(format, "vendor.rtc-ext-dec-low-latency.enable", 1); + // MediaCodec knows two priorities: 0 - realtime, 1 - best effort. Lower is higher. + AMediaFormat_setInt32(format, "priority", 0); } static void h264_configureAMediaFormat(KeyFrameFinder& kff, AMediaFormat* format) @@ -42,7 +38,6 @@ static void h264_configureAMediaFormat(KeyFrameFinder& kff, AMediaFormat* format // AVCProfileBaseline==1 // AMediaFormat_setInt32(decoder.format,AMEDIAFORMAT_KEY_PROFILE,1); // AMediaFormat_setInt32(decoder.format,AMEDIAFORMAT_KEY_PRIORITY,0); - // writeAndroidPerformanceParams(format); } static void h265_configureAMediaFormat(KeyFrameFinder& kff, AMediaFormat* format) @@ -60,7 +55,6 @@ static void h265_configureAMediaFormat(KeyFrameFinder& kff, AMediaFormat* format AMediaFormat_setInt32(format, AMEDIAFORMAT_KEY_HEIGHT, videoWH[1]); AMediaFormat_setBuffer(format, "csd-0", buff.data(), buff.size()); MLOGD << "Video WH:" << videoWH[0] << " H:" << videoWH[1]; - // writeAndroidPerformanceParams(format); } #endif // FPVUE_ANDROIDMEDIAFORMATHELPER_H diff --git a/app/videonative/src/main/java/com/openipc/videonative/VideoPlayer.java b/app/videonative/src/main/java/com/openipc/videonative/VideoPlayer.java index 7330eb8b..e1c49d0f 100644 --- a/app/videonative/src/main/java/com/openipc/videonative/VideoPlayer.java +++ b/app/videonative/src/main/java/com/openipc/videonative/VideoPlayer.java @@ -52,6 +52,8 @@ public VideoPlayer(final AppCompatActivity parent) { public static native void nativeSetUdpForwarding(long nativeInstance, String ip, int port, boolean enabled); + public static native void nativeSetLowLatency(long nativeInstance, boolean enabled); + public static native void nativeStartDvr(long nativeInstance, int fd, int fmp4_enabled); public static native void nativeStopDvr(long nativeInstance); @@ -126,6 +128,14 @@ public boolean isRunning() { return timer != null; } + /** + * Enable/disable the low latency + realtime priority MediaCodec keys. + * Takes effect the next time the decoder is configured. + */ + public void setLowLatency(boolean enabled) { + nativeSetLowLatency(nativeVideoPlayer, enabled); + } + public void setUdpForwarding(String ip, int port, boolean enabled) { verifyApplicationThread(); nativeSetUdpForwarding(nativeVideoPlayer, ip, port, enabled); diff --git a/app/wfbngrtl8812/src/main/cpp/WfbngLink.cpp b/app/wfbngrtl8812/src/main/cpp/WfbngLink.cpp index e8d9c14c..a3046b07 100644 --- a/app/wfbngrtl8812/src/main/cpp/WfbngLink.cpp +++ b/app/wfbngrtl8812/src/main/cpp/WfbngLink.cpp @@ -34,12 +34,6 @@ #undef TAG #define TAG "pixelpilot" -#define CRASH() \ - do { \ - int *i = 0; \ - *i = 42; \ - } while (0) - std::string generate_random_string(size_t length) { const std::string characters = "abcdefghijklmnopqrstuvwxyz"; std::random_device rd; @@ -283,8 +277,9 @@ int WfbngLink::run(JNIEnv *env, jobject context, jint wifiChannel, jint bw, jint void WfbngLink::stop(JNIEnv *env, jobject context, jint fd) { if (rtl_devices.find(fd) == rtl_devices.end()) { - __android_log_print(ANDROID_LOG_ERROR, TAG, "rtl_devices.find(%d) == rtl_devices.end()", fd); - CRASH(); + // Happens when the adapter was already gone by the time the stop arrived, e.g. it + // was unplugged or the hub re-enumerated it. Nothing left to stop. + __android_log_print(ANDROID_LOG_WARN, TAG, "stop: no rtl device for fd=%d, already gone", fd); return; } auto dev = rtl_devices.at(fd).get(); diff --git a/app/wfbngrtl8812/src/main/java/com/openipc/wfbngrtl8812/WfbNgLink.java b/app/wfbngrtl8812/src/main/java/com/openipc/wfbngrtl8812/WfbNgLink.java index ea0347de..39dca17d 100644 --- a/app/wfbngrtl8812/src/main/java/com/openipc/wfbngrtl8812/WfbNgLink.java +++ b/app/wfbngrtl8812/src/main/java/com/openipc/wfbngrtl8812/WfbNgLink.java @@ -92,17 +92,35 @@ public void nativeSetUseStbc(int use) { nativeSetUseStbc(nativeWfbngLink, use); } - public synchronized void start(int wifiChannel, int bandWidth, UsbDevice usbDevice) { + public synchronized boolean start(int wifiChannel, int bandWidth, UsbDevice usbDevice) { Log.d(TAG, "wfb-ng monitoring on " + usbDevice.getDeviceName() + " using wifi channel " + wifiChannel); UsbManager usbManager = (UsbManager) context.getSystemService(Context.USB_SERVICE); + // Returns null when the permission was revoked or the device disappeared between + // the permission check and here, which is easy to hit on a re-enumerating hub. UsbDeviceConnection usbDeviceConnection = usbManager.openDevice(usbDevice); + if (usbDeviceConnection == null) { + Log.e(TAG, "Could not open " + usbDevice.getDeviceName() + " (no permission or already gone)"); + return false; + } int fd = usbDeviceConnection.getFileDescriptor(); + if (fd < 0) { + Log.e(TAG, "Invalid file descriptor for " + usbDevice.getDeviceName()); + usbDeviceConnection.close(); + return false; + } Thread t = new Thread(() -> nativeRun(nativeWfbngLink, context, wifiChannel, bandWidth, fd)); - t.setName("wfb-" + usbDevice.getDeviceName().split("/dev/bus/usb/")[1]); + t.setName(threadNameFor(usbDevice)); linkThreads.put(usbDevice, t); linkConns.put(usbDevice, usbDeviceConnection); - linkThreads.get(usbDevice).start(); + t.start(); Log.d(TAG, "wfb-ng thread on " + usbDevice.getDeviceName() + " started."); + return true; + } + + private static String threadNameFor(UsbDevice usbDevice) { + String name = usbDevice.getDeviceName(); + String[] parts = name.split("/dev/bus/usb/"); + return "wfb-" + (parts.length > 1 ? parts[1] : name); } public synchronized void stopAll() throws InterruptedException { @@ -114,9 +132,13 @@ public synchronized void stopAll() throws InterruptedException { if (t != null) { t.join(); } + // The connection holds a dup of the usbfs fd. Without close() every + // attach/detach cycle leaks one, until the process runs out. + entry.getValue().close(); Log.d(TAG, "wfb-ng thread on " + entry.getKey().getDeviceName() + " done."); } linkThreads.clear(); + linkConns.clear(); } public synchronized void stop(UsbDevice dev) throws InterruptedException { @@ -131,6 +153,8 @@ public synchronized void stop(UsbDevice dev) throws InterruptedException { t.join(); } linkThreads.remove(dev); + linkConns.remove(dev); + conn.close(); } public void SetWfbNGStatsChanged(final WfbNGStatsChanged callback) {