Reuse and close Android barcode scanners without leaking shared images

android/build.gradle

@@ -14,12 +14,14 @@ if (isNewArchitectureEnabled()) {
 }
 
 android {
+    testOptions { unitTests.includeAndroidResources = true }
     def agpVersion = com.android.Version.ANDROID_GRADLE_PLUGIN_VERSION
     if (agpVersion.tokenize('.')[0].toInteger() >= 7) {
       namespace "com.rncamerakit"
     }
     compileSdkVersion = 34
     defaultConfig {
+        testInstrumentationRunner "androidx.test.runner.AndroidJUnitRunner"
         minSdkVersion = 24
         // noinspection ExpiredTargetSdkVersion
         targetSdkVersion = 34
@@ -46,6 +48,12 @@ android {
 }
 
 dependencies {
+    testImplementation 'junit:junit:4.13.2'
+    testImplementation 'org.robolectric:robolectric:4.14.1'
+    testImplementation 'org.mockito:mockito-core:5.18.0'
+    androidTestImplementation 'androidx.test:runner:1.6.2'
+    androidTestImplementation 'androidx.test.ext:junit:1.2.1'
+    androidTestImplementation 'com.google.zxing:core:3.5.3'
     implementation 'com.facebook.react:react-native:+'
     implementation group: 'com.drewnoakes', name: 'metadata-extractor', version: '2.18.0'
     implementation "org.jetbrains.kotlin:kotlin-stdlib-jdk7:$kotlinVersion"

android/src/androidTest/java/com/rncamerakit/BarcodeNativeLifecycleTest.kt

@@ -0,0 +1,150 @@
+package com.rncamerakit
+
+import android.annotation.SuppressLint
+import android.graphics.ImageFormat
+import android.graphics.Matrix
+import android.graphics.Rect
+import android.media.Image
+import android.media.ImageReader
+import android.media.ImageWriter
+import android.os.Debug
+import android.os.Handler
+import android.os.Looper
+import android.util.Log
+import androidx.camera.core.ImageInfo
+import androidx.camera.core.ImageProxy
+import androidx.camera.core.impl.TagBundle
+import androidx.camera.core.impl.utils.ExifData
+import androidx.test.ext.junit.runners.AndroidJUnit4
+import androidx.test.platform.app.InstrumentationRegistry
+import com.google.android.gms.tasks.Task
+import com.google.mlkit.vision.barcode.BarcodeScanner
+import com.google.mlkit.vision.barcode.BarcodeScanning
+import com.google.mlkit.vision.barcode.common.Barcode
+import com.google.mlkit.vision.common.InputImage
+import com.google.zxing.BarcodeFormat
+import com.google.zxing.qrcode.QRCodeWriter
+import org.junit.Assert.*
+import org.junit.Test
+import org.junit.runner.RunWith
+import java.nio.ByteBuffer
+import java.util.concurrent.CountDownLatch
+import java.util.concurrent.TimeUnit
+import java.util.concurrent.atomic.AtomicInteger
+
+/** Real bundled ML Kit clients; authored Android YUV image; no camera/HAL claim. */
+@RunWith(AndroidJUnit4::class)
+@SuppressLint("RestrictedApi", "UnsafeOptInUsageError")
+class BarcodeNativeLifecycleTest {
+    @Test
+    fun repeatedGenerationsCloseClientsAndImagesExactlyOnce() {
+        val instrumentation = InstrumentationRegistry.getInstrumentation()
+        val ready = CountDownLatch(1)
+        val reader = ImageReader.newInstance(320, 240, ImageFormat.YUV_420_888, 2)
+        val writer = ImageWriter.newInstance(reader.surface, 2)
+        reader.setOnImageAvailableListener({ ready.countDown() }, Handler(Looper.getMainLooper()))
+        val writable = writer.dequeueInputImage()
+        val qr = QRCodeWriter().encode("LIFECYCLE", BarcodeFormat.QR_CODE, 160, 160)
+        for ((index, plane) in writable.planes.withIndex()) {
+            val width = if (index == 0) 320 else 160
+            val height = if (index == 0) 240 else 120
+            for (y in 0 until height) for (x in 0 until width) {
+                val value = if (index != 0) 128 else if (x in 80 until 240 && y in 40 until 200 && qr[x - 80, y - 40]) 0 else 255
+                plane.buffer.put(y * plane.rowStride + x * plane.pixelStride, value.toByte())
+            }
+        }
+        writer.queueInputImage(writable)
+        assertTrue(ready.await(10, TimeUnit.SECONDS))
+        val media = requireNotNull(reader.acquireNextImage())
+        val created = AtomicInteger()
+        val closed = AtomicInteger()
+        val processed = AtomicInteger()
+        val imageCloses = AtomicInteger()
+        val events = AtomicInteger()
+        fun report(generation: Int) {
+            val memory = Debug.MemoryInfo().also { Debug.getMemoryInfo(it) }
+            Log.i("CameraKitLifecycle", "generation=$generation created=${created.get()} closed=${closed.get()} processed=${processed.get()} imageCloses=${imageCloses.get()} events=${events.get()} nativeHeapBytes=${Debug.getNativeHeapAllocatedSize()} totalPssKb=${memory.totalPss} fixture=authored-yuv")
+        }
+        report(0)
+        try {
+            repeat(20) { generation ->
+                var latest: Task<List<Barcode>>? = null
+                val analyzer = QRCodeAnalyzer({ codes, _ ->
+                    assertEquals(listOf("LIFECYCLE"), codes.map { it.rawValue })
+                    events.incrementAndGet()
+                }, 0) {
+                    created.incrementAndGet()
+                    val client = BarcodeScanning.getClient()
+                    object : BarcodeScanner by client {
+                        override fun process(image: InputImage): Task<List<Barcode>> {
+                            processed.incrementAndGet()
+                            return client.process(image).also { latest = it }
+                        }
+                        override fun close() {
+                            closed.incrementAndGet()
+                            client.close()
+                        }
+                    }
+                }
+                try {
+                    repeat(3) { frame ->
+                        val proxy = SceneImage(media, imageCloses)
+                        instrumentation.runOnMainSync {
+                            analyzer.analyze(proxy)
+                            // Close before queued success callbacks can run for the final frame.
+                            if (frame == 2) analyzer.close()
+                        }
+                        val done = CountDownLatch(1)
+                        val task = requireNotNull(latest)
+                        task.addOnCompleteListener { done.countDown() }
+                        assertTrue(done.await(30, TimeUnit.SECONDS))
+                        assertTrue("Real barcode inference failed: ${task.exception}", task.isSuccessful)
+                        assertEquals(listOf("LIFECYCLE"), task.result.map { it.rawValue })
+                        // The shared-task barrier schedules its own completion callback.
+                        assertTrue("Shared image was not released", proxy.closed.await(5, TimeUnit.SECONDS))
+                        assertEquals(1, proxy.closeCount)
+                    }
+                } finally {
+                    analyzer.close()
+                }
+                assertEquals(generation + 1, created.get())
+                assertEquals(generation + 1, closed.get())
+                assertEquals((generation + 1) * 3, imageCloses.get())
+                assertEquals((generation + 1) * 2, events.get())
+                if ((generation + 1) % 5 == 0) report(generation + 1)
+            }
+            assertEquals(60, processed.get())
+        } finally {
+            media.close()
+            writer.close()
+            reader.close()
+        }
+    }
+
+    private class SceneImage(private val media: Image, private val totalCloses: AtomicInteger) : ImageProxy {
+        val closed = CountDownLatch(1)
+        var closeCount = 0
+        override fun close() { closeCount++; totalCloses.incrementAndGet(); closed.countDown() }
+        // Media storage belongs to the fixture and is reused only after each task completes.
+        override fun getImage(): Image = media
+        override fun getWidth() = media.width
+        override fun getHeight() = media.height
+        override fun getFormat() = media.format
+        override fun getCropRect() = Rect(0, 0, width, height)
+        override fun setCropRect(rect: Rect?) = Unit
+        override fun getPlanes(): Array<ImageProxy.PlaneProxy> = media.planes.map { plane ->
+            object : ImageProxy.PlaneProxy {
+                override fun getBuffer(): ByteBuffer = plane.buffer
+                override fun getRowStride() = plane.rowStride
+                override fun getPixelStride() = plane.pixelStride
+            }
+        }.toTypedArray()
+        override fun getImageInfo(): ImageInfo = object : ImageInfo {
+            override fun getRotationDegrees() = 0
+            override fun getTimestamp() = media.timestamp
+            override fun getTagBundle(): TagBundle = TagBundle.emptyBundle()
+            override fun getSensorToBufferTransformMatrix() = Matrix()
+            override fun populateExifData(builder: ExifData.Builder) = Unit
+        }
+    }
+}

android/src/main/java/com/rncamerakit/CKCamera.kt

@@ -45,8 +45,6 @@ import android.graphics.Rect
 import android.graphics.RectF
 import android.util.Size
 import com.facebook.react.uimanager.UIManagerHelper
-import com.google.android.gms.tasks.Task
-import com.google.android.gms.tasks.Tasks
 import com.google.mlkit.vision.barcode.common.Barcode
 import com.rncamerakit.events.*
 
@@ -86,6 +84,7 @@ class CKCamera(context: ThemedReactContext) : FrameLayout(context), LifecycleObs
     private var imageCapture: ImageCapture? = null
     private var imageAnalyzer: ImageAnalysis? = null
     private var faceAnalyzer: FaceAnalyzer? = null
+    private var barcodeAnalyzer: QRCodeAnalyzer? = null
     private var orientationListener: OrientationEventListener? = null
     private var viewFinder: PreviewView = PreviewView(context)
     private var rectOverlay: RectOverlay = RectOverlay(context)
@@ -149,6 +148,8 @@ class CKCamera(context: ThemedReactContext) : FrameLayout(context), LifecycleObs
     override fun onDetachedFromWindow() {
         super.onDetachedFromWindow()
 
+        barcodeAnalyzer?.close()
+        barcodeAnalyzer = null
         faceAnalyzer?.close()
         faceAnalyzer = null
         cameraExecutor.shutdown()
@@ -358,10 +359,12 @@ class CKCamera(context: ThemedReactContext) : FrameLayout(context), LifecycleObs
 
         val useCases = mutableListOf(preview, imageCapture)
 
+        barcodeAnalyzer?.close()
+        barcodeAnalyzer = null
         faceAnalyzer?.close()
         faceAnalyzer = null
 
-        val barcodeAnalyzer: QRCodeAnalyzer? = if (scanBarcode) {
+        barcodeAnalyzer = if (scanBarcode) {
             QRCodeAnalyzer({ barcodes, imageSize ->
                 if (barcodes.isEmpty()) return@QRCodeAnalyzer
 
@@ -422,14 +425,14 @@ class CKCamera(context: ThemedReactContext) : FrameLayout(context), LifecycleObs
             )
         } else null
 
+        val activeBarcodeAnalyzer = barcodeAnalyzer
         val activeFaceAnalyzer = faceAnalyzer
-        if (barcodeAnalyzer != null || activeFaceAnalyzer != null) {
+        if (activeBarcodeAnalyzer != null || activeFaceAnalyzer != null) {
             imageAnalyzer!!.setAnalyzer(cameraExecutor) { image ->
-                val tasks = mutableListOf<Task<*>>()
-                barcodeAnalyzer?.analyzeWithoutClosing(image)?.let { tasks.add(it) }
-                activeFaceAnalyzer?.analyzeWithoutClosing(image)?.let { tasks.add(it) }
-                if (tasks.isEmpty()) image.close()
-                else Tasks.whenAllComplete(tasks).addOnCompleteListener { image.close() }
+                analyzeSharedImage(image,
+                    { activeBarcodeAnalyzer?.analyzeWithoutClosing(it) },
+                    { activeFaceAnalyzer?.analyzeWithoutClosing(it) },
+                )
             }
             useCases.add(imageAnalyzer)
         }

android/src/main/java/com/rncamerakit/QRCodeAnalyzer.kt

@@ -6,27 +6,50 @@ import androidx.camera.core.ExperimentalGetImage
 import androidx.camera.core.ImageAnalysis
 import androidx.camera.core.ImageProxy
 import com.google.android.gms.tasks.Task
+import com.google.mlkit.vision.barcode.BarcodeScanner
 import com.google.mlkit.vision.barcode.BarcodeScanning
 import com.google.mlkit.vision.barcode.common.Barcode
 import com.google.mlkit.vision.common.InputImage
 
-class QRCodeAnalyzer (
+class QRCodeAnalyzer internal constructor(
     private val onQRCodesDetected: (qrCodes: List<Barcode>, imageSize: Size) -> Unit,
-    private val scanThrottleDelay: Long = 0L
-) : ImageAnalysis.Analyzer {
+    private val scanThrottleDelay: Long,
+    private val scannerFactory: () -> BarcodeScanner,
+) : ImageAnalysis.Analyzer, AutoCloseable {
+    constructor(
+        onQRCodesDetected: (qrCodes: List<Barcode>, imageSize: Size) -> Unit,
+        scanThrottleDelay: Long = 0L,
+    ) : this(onQRCodesDetected, scanThrottleDelay, { BarcodeScanning.getClient() })
+
+    private enum class State { OPEN, CLOSING, CLOSED }
+    private val lock = Any()
+    private var state = State.OPEN
+    private var scanner: BarcodeScanner? = null
+    private var inFlight = 0
+
     // Time in milliseconds of the last time we dispatched detected barcodes
     private var lastBarcodeDetectedTime: Long = 0L
 
     @SuppressLint("UnsafeExperimentalUsageError")
     @ExperimentalGetImage
-    fun analyzeWithoutClosing(image: ImageProxy): Task<*>? {
+    fun analyzeWithoutClosing(image: ImageProxy): Task<*>? = synchronized(lock) {
+        if (state != State.OPEN) return null
         val mediaImage = image.image ?: return null
-
+        val imageSize = Size(image.width, image.height)
         val inputImage = InputImage.fromMediaImage(mediaImage, image.imageInfo.rotationDegrees)
-
-        val scanner = BarcodeScanning.getClient()
-        return scanner.process(inputImage)
-            .addOnSuccessListener { barcodes ->
+        val activeScanner = scanner ?: scannerFactory().also { scanner = it }
+        // Register the consumer before process() can race with close().
+        inFlight++
+        val task = try {
+            activeScanner.process(inputImage)
+        } catch (e: Exception) {
+            inFlight--
+            closeIfIdle()
+            throw e
+        }
+        task.addOnSuccessListener { barcodes ->
+            synchronized(lock) {
+                if (state != State.OPEN) return@addOnSuccessListener
                 // Throttle callback invocations based on scanThrottleDelay (ms)
                 val now = System.currentTimeMillis()
                 if (scanThrottleDelay > 0 && (now - lastBarcodeDetectedTime) < scanThrottleDelay) {
@@ -35,19 +58,37 @@ class QRCodeAnalyzer (
 
                 if (barcodes.isNotEmpty()) {
                     lastBarcodeDetectedTime = now
-                    onQRCodesDetected(barcodes, Size(image.width, image.height))
+                    onQRCodesDetected(barcodes, imageSize)
                 }
             }
+        }
+        task.addOnCompleteListener {
+            synchronized(lock) {
+                inFlight--
+                closeIfIdle()
+            }
+        }
+        task
     }
 
     @SuppressLint("UnsafeExperimentalUsageError")
     @ExperimentalGetImage
     override fun analyze(image: ImageProxy) {
-        val task = analyzeWithoutClosing(image)
-        if (task == null) {
-            image.close()
-            return
+        analyzeSharedImage(image, ::analyzeWithoutClosing)
+    }
+
+    override fun close() = synchronized(lock) {
+        if (state == State.OPEN) state = State.CLOSING
+        closeIfIdle()
+    }
+
+    // All state transitions and outward callbacks are serialized by lock.
+    private fun closeIfIdle() {
+        if (state == State.CLOSING && inFlight == 0) {
+            state = State.CLOSED
+            val closingScanner = scanner
+            scanner = null
+            closingScanner?.close()
         }
-        task.addOnCompleteListener { image.close() }
     }
 }

android/src/main/java/com/rncamerakit/SharedImageAnalyzer.kt

@@ -0,0 +1,20 @@
+package com.rncamerakit
+
+import android.util.Log
+import androidx.camera.core.ImageProxy
+import com.google.android.gms.tasks.Task
+import com.google.android.gms.tasks.Tasks
+
+/** Every successfully launched consumer owns the image until its task settles. */
+internal fun analyzeSharedImage(image: ImageProxy, vararg consumers: (ImageProxy) -> Task<*>?) {
+    val tasks = mutableListOf<Task<*>>()
+    for (consumer in consumers) {
+        try {
+            consumer(image)?.let { tasks.add(it) }
+        } catch (e: Exception) {
+            Log.w("CameraKit", "Image analyzer failed to start", e)
+        }
+    }
+    if (tasks.isEmpty()) image.close()
+    else Tasks.whenAllComplete(tasks).addOnCompleteListener { image.close() }
+}