fix(linux): retry PipeWire capture with the shared-memory format alone

electron/native/pipewire-capture/csrc/pw_shim.c

@@ -1754,7 +1754,7 @@ static const struct pw_stream_events osc_stream_events = {
 };
 
 struct osc_pw_session *osc_pw_start(int fd, uint32_t node_id, int want_video,
-                                    int prefer_dmabuf,
+                                    int prefer_dmabuf, int shm_only,
                                     const struct osc_pw_callbacks *callbacks, char *err,
                                     size_t err_len)
 {
@@ -1764,6 +1764,7 @@ struct osc_pw_session *osc_pw_start(int fd, uint32_t node_id, int want_video,
     uint8_t buffer[2048];
     struct spa_pod_builder builder = SPA_POD_BUILDER_INIT(buffer, sizeof(buffer));
     const struct spa_pod *params[2];
+    uint32_t n_params = 2;
     int result;
 
     if (api_handle == NULL) {
@@ -1849,13 +1850,22 @@ struct osc_pw_session *osc_pw_start(int fd, uint32_t node_id, int want_video,
      * Without either, the ordering is unchanged — shm first — so nothing moves on
      * a build or driver without the VAAPI import.
      */
-    if (session->prefer_dmabuf || getenv("OPENSCREEN_PIPEWIRE_FORCE_DMABUF") != NULL) {
+    if (shm_only) {
+        /*
+         * The shared-memory retry after "error alloc buffers" (issue #1445).
+         * xdg-desktop-portal-cosmic accepts the modifier-carrying object even in
+         * second place, yet only offers MemFd for it: a DmaBuf-only Buffers param
+         * then intersects to nothing, and MemFd on a modifier format arrives
+         * black. Dropping the object is the only way to get the plain shm format.
+         */
+        n_params = 1;
+    } else if (session->prefer_dmabuf || getenv("OPENSCREEN_PIPEWIRE_FORCE_DMABUF") != NULL) {
         const struct spa_pod *shm = params[0];
         params[0] = params[1];
         params[1] = shm;
     }
 
-    if (params[0] == NULL || params[1] == NULL) {
+    if (params[0] == NULL || (n_params == 2 && params[1] == NULL)) {
         osc_set_error(err, err_len, "the EnumFormat PODs did not fit their builder");
         goto fail;
     }
@@ -1897,7 +1907,7 @@ struct osc_pw_session *osc_pw_start(int fd, uint32_t node_id, int want_video,
                                 want_video ? (PW_STREAM_FLAG_AUTOCONNECT |
                                               PW_STREAM_FLAG_MAP_BUFFERS)
                                            : PW_STREAM_FLAG_AUTOCONNECT,
-                                params, SPA_N_ELEMENTS(params));
+                                params, n_params);
     if (result < 0) {
         osc_set_error(err, err_len, "pw_stream_connect failed: %s", spa_strerror(result));
         goto fail;

electron/native/pipewire-capture/csrc/pw_shim.h

@@ -272,10 +272,14 @@ const char *osc_pw_library_version(void);
  * on the GPU rather than copied through throttled shm (issue #507); set it only
  * when the VAAPI import pipeline is available. shm remains the fallback.
  *
+ * `shm_only` offers the shm EnumFormat alone, without the DMA-BUF object: the
+ * retry after a failed buffer allocation (issue #1445). It overrides
+ * `prefer_dmabuf`.
+ *
  * Returns NULL on failure, with a message in `err`.
  */
 struct osc_pw_session *osc_pw_start(int fd, uint32_t node_id, int want_video,
-                                    int prefer_dmabuf,
+                                    int prefer_dmabuf, int shm_only,
                                     const struct osc_pw_callbacks *callbacks, char *err,
                                     size_t err_len);
 

electron/native/pipewire-capture/src/main.rs

@@ -554,20 +554,19 @@ fn begin_stream<W: Write>(
     granted_kind: &mut Option<portal::SourceKind>,
     stream: portal::PortalStream,
     prefer_dmabuf: bool,
+    shared_memory_only: bool,
     shared_memory_fallback: &mut Option<SharedMemoryFallback>,
 ) -> Result<(), ()> {
     let portal::PortalStream { fd, fallback_fd, node_id, position, size, source_kind } = stream;
-    *shared_memory_fallback = if prefer_dmabuf {
-        fallback_fd.map(|fd| SharedMemoryFallback {
-            fd,
-            node_id,
-            position,
-            size,
-            source_kind,
-        })
-    } else {
-        None
-    };
+    // Armed even without `prefer_dmabuf`: the DMA-BUF object is still offered in
+    // second place, and xdg-desktop-portal-cosmic picks it there too (issue #1445).
+    *shared_memory_fallback = fallback_fd.map(|fd| SharedMemoryFallback {
+        fd,
+        node_id,
+        position,
+        size,
+        source_kind,
+    });
     let forward = sender.clone();
     match shim::Session::start(
         fd,
@@ -577,6 +576,7 @@ fn begin_stream<W: Write>(
         }),
         frames.clone(),
         prefer_dmabuf,
+        shared_memory_only,
     ) {
         Ok(started) => {
             *session = Some(started);
@@ -1056,6 +1056,7 @@ fn run<W: Write>(
                         &mut granted_kind,
                         stream,
                         prefer_dmabuf,
+                        false,
                         &mut shared_memory_fallback,
                     ) {
                         exit_code = 1;
@@ -1117,6 +1118,7 @@ fn run<W: Write>(
                         &mut granted_kind,
                         stream,
                         prefer_dmabuf,
+                        false,
                         &mut shared_memory_fallback,
                     ) {
                         exit_code = 1;
@@ -1269,12 +1271,12 @@ fn run<W: Write>(
                     let fallback = shared_memory_fallback.take().expect("checked above");
                     // Drop and join the failed stream before reconnecting with
                     // the portal's independent remote. The second attempt
-                    // prefers shared memory; it will not arm another retry.
+                    // offers the shm format alone; it will not arm another retry.
                     drop(session.take());
                     portal_stream = None;
                     let _ = emitter.emit(&Event::Warning {
                         code: "capture-retry-shared-memory".to_owned(),
-                        message: "PipeWire buffer allocation failed with DMA-BUF; retrying screen capture with shared-memory buffers."
+                        message: "PipeWire buffer allocation failed; retrying screen capture with shared-memory buffers."
                             .to_owned(),
                     });
                     let retry = portal::PortalStream {
@@ -1294,6 +1296,7 @@ fn run<W: Write>(
                         &mut granted_kind,
                         retry,
                         false,
+                        true,
                         &mut shared_memory_fallback,
                     ) {
                         exit_code = 1;

electron/native/pipewire-capture/src/shim.rs

@@ -160,6 +160,7 @@ extern "C" {
         node_id: u32,
         want_video: i32,
         prefer_dmabuf: i32,
+        shm_only: i32,
         callbacks: *const RawCallbacks,
         err: *mut c_char,
         err_len: usize,
@@ -1013,6 +1014,9 @@ impl Session {
         // Offer dmabuf before shm for a whole-monitor GPU import (issue #507).
         // Only honoured for a video session; ignored for cursor-only.
         prefer_dmabuf: bool,
+        // Offer the shm format alone: the retry after a failed buffer
+        // allocation (issue #1445). Overrides `prefer_dmabuf`.
+        shared_memory_only: bool,
     ) -> Result<Self, String> {
         let want_video = i32::from(frames.is_some());
         let prefer_dmabuf = i32::from(want_video != 0 && prefer_dmabuf);
@@ -1037,6 +1041,7 @@ impl Session {
                 node_id,
                 want_video,
                 prefer_dmabuf,
+                i32::from(shared_memory_only),
                 &callbacks,
                 err.as_mut_ptr(),
                 ERR_LEN,
@@ -1692,6 +1697,7 @@ mod tests {
             None,
             // Cursor-only, so dmabuf preference is irrelevant.
             false,
+            false,
         )
         .expect("stream must connect");