Map Android barcode bounds into the visible scan frame

android/build.gradle

@@ -14,12 +14,14 @@ if (isNewArchitectureEnabled()) {
 }
 
 android {
+    testOptions { unitTests.includeAndroidResources = true }
     def agpVersion = com.android.Version.ANDROID_GRADLE_PLUGIN_VERSION
     if (agpVersion.tokenize('.')[0].toInteger() >= 7) {
       namespace "com.rncamerakit"
     }
     compileSdkVersion = 34
     defaultConfig {
+        testInstrumentationRunner "androidx.test.runner.AndroidJUnitRunner"
         minSdkVersion = 24
         // noinspection ExpiredTargetSdkVersion
         targetSdkVersion = 34
@@ -46,6 +48,13 @@ android {
 }
 
 dependencies {
+    testImplementation 'junit:junit:4.13.2'
+    testImplementation 'org.robolectric:robolectric:4.14.1'
+    testImplementation 'org.mockito:mockito-core:5.18.0'
+    androidTestImplementation 'androidx.test:runner:1.6.2'
+    androidTestImplementation 'androidx.test.ext:junit:1.2.1'
+    androidTestImplementation 'com.google.zxing:core:3.5.3'
+
     implementation 'com.facebook.react:react-native:+'
     implementation group: 'com.drewnoakes', name: 'metadata-extractor', version: '2.18.0'
     implementation "org.jetbrains.kotlin:kotlin-stdlib-jdk7:$kotlinVersion"

android/src/androidTest/java/com/rncamerakit/BarcodeNativeSceneTest.kt

@@ -0,0 +1,143 @@
+package com.rncamerakit
+
+import android.annotation.SuppressLint
+import android.graphics.ImageFormat
+import android.graphics.Matrix
+import android.graphics.Rect
+import android.media.Image
+import android.media.ImageReader
+import android.media.ImageWriter
+import android.os.Handler
+import android.os.Looper
+import android.util.Log
+import android.util.Size
+import androidx.camera.core.ImageInfo
+import androidx.camera.core.ImageProxy
+import androidx.camera.core.impl.TagBundle
+import androidx.camera.core.impl.utils.ExifData
+import androidx.camera.view.PreviewView
+import androidx.test.ext.junit.runners.AndroidJUnit4
+import androidx.test.platform.app.InstrumentationRegistry
+import com.facebook.react.uimanager.ThemedReactContext
+import com.google.android.gms.tasks.Tasks
+import com.google.zxing.BarcodeFormat
+import com.google.zxing.EncodeHintType
+import com.google.zxing.qrcode.QRCodeWriter
+import com.rncamerakit.barcode.BarcodeFrame
+import org.junit.Assert.*
+import org.junit.Test
+import org.junit.runner.RunWith
+import java.nio.ByteBuffer
+import java.util.concurrent.CountDownLatch
+import java.util.concurrent.TimeUnit
+
+/** Real Android media images -> bundled ML Kit -> CKCamera -> native event IDs.
+ * Sensor/preview transforms are authored fixtures, not sensor/HAL measurements.
+ * No JavaScript, physical camera, scanner mock or screenshot oracle is used.
+ */
+@RunWith(AndroidJUnit4::class)
+@SuppressLint("RestrictedApi", "UnsafeOptInUsageError")
+class BarcodeNativeSceneTest {
+    @Test
+    fun authoredTwoCodeSceneSelectsOnlyInsideAcrossAllMlKitRotations() {
+        val instrumentation = InstrumentationRegistry.getInstrumentation()
+        val ready = CountDownLatch(1)
+        val reader = ImageReader.newInstance(1280, 720, ImageFormat.YUV_420_888, 2)
+        val writer = ImageWriter.newInstance(reader.surface, 2)
+        reader.setOnImageAvailableListener({ ready.countDown() }, Handler(Looper.getMainLooper()))
+        val image = writer.dequeueInputImage()
+        val pixels = ByteArray(1280 * 720) { 255.toByte() }
+        fun code(value: String, x: Int, y: Int, size: Int) {
+            val bits = QRCodeWriter().encode(value, BarcodeFormat.QR_CODE, size, size,
+                mapOf(EncodeHintType.MARGIN to 1))
+            for (row in 0 until size) for (column in 0 until size) {
+                pixels[(y + row) * 1280 + x + column] = if (bits[column, row]) 0 else 255.toByte()
+            }
+        }
+        code("INSIDE", 600, 300, 100)
+        code("OUTSIDE", 300, 400, 90)
+        for ((index, plane) in image.planes.withIndex()) {
+            val width = if (index == 0) 1280 else 640
+            val height = if (index == 0) 720 else 360
+            for (y in 0 until height) for (x in 0 until width) {
+                plane.buffer.put(y * plane.rowStride + x * plane.pixelStride,
+                    if (index == 0) pixels[y * 1280 + x] else 128.toByte())
+            }
+        }
+        writer.queueInputImage(image)
+        assertTrue("Authored Android image did not arrive", ready.await(10, TimeUnit.SECONDS))
+        val media = requireNotNull(reader.acquireNextImage())
+        try {
+            for (rotation in listOf(0, 90, 180, 270)) {
+                lateinit var react: RecordingReactContext
+                lateinit var camera: CKCamera
+                lateinit var preview: BarcodePreviewGeometry
+                instrumentation.runOnMainSync {
+                    react = RecordingReactContext(instrumentation.targetContext)
+                    val context = ThemedReactContext(react, instrumentation.targetContext, "native-scene", 1)
+                    camera = CKCamera(context).apply { id = 42 }
+                    val fixturePreview = PreviewView(context).apply { layout(0, 0, 1280, 720) }
+                    // Seed CameraX's real preview transform with authored surface metadata.
+                    val transform = PreviewView::class.java.getDeclaredField("mPreviewTransform")
+                        .apply { isAccessible = true }.get(fixturePreview)
+                    for ((name, value) in mapOf(
+                        "mResolution" to Size(1280, 720),
+                        "mSurfaceCropRect" to Rect(0, 0, 1280, 720),
+                        "mSensorToBufferTransform" to Matrix(),
+                    )) {
+                        transform.javaClass.getDeclaredField(name).apply { isAccessible = true }
+                            .set(transform, value)
+                    }
+                    assertTrue(requireNotNull(fixturePreview.sensorToViewTransform).isIdentity)
+                    val frame = BarcodeFrame(context).apply { frameRect = Rect(400, 180, 880, 540) }
+                    CKCamera::class.java.getDeclaredField("viewFinder").apply { isAccessible = true }
+                        .set(camera, fixturePreview)
+                    CKCamera::class.java.getDeclaredField("barcodeFrame").apply { isAccessible = true }
+                        .set(camera, frame)
+                    preview = camera.snapshotBarcodePreview()
+                }
+                val callback = CountDownLatch(1)
+                val decoded = mutableSetOf<String?>()
+                val analyzer = QRCodeAnalyzer.withFrameGeometry({ barcodes, geometry ->
+                    decoded.addAll(barcodes.map { it.rawValue })
+                    camera.onBarcodesDetected(barcodes, geometry)
+                    callback.countDown()
+                }, 0)
+                val task = requireNotNull(analyzer.analyzeWithoutClosing(SceneImage(media, rotation), preview))
+                Tasks.await(task, 30, TimeUnit.SECONDS)
+                assertTrue("Native ML Kit success callback did not arrive", callback.await(10, TimeUnit.SECONDS))
+                assertEquals(setOf("INSIDE", "OUTSIDE"), decoded)
+                assertEquals(listOf("INSIDE"), react.values)
+                Log.i("CameraKitScene", "rotation=$rotation buffer=1280x720 crop=(0,0,1280,720) decoded=$decoded emitted=${react.values} transforms=authored")
+            }
+        } finally {
+            media.close()
+            writer.close()
+            reader.close()
+        }
+    }
+
+    private class SceneImage(private val media: Image, private val rotation: Int) : ImageProxy {
+        override fun getImage(): Image = media
+        override fun getWidth() = media.width
+        override fun getHeight() = media.height
+        override fun getFormat() = media.format
+        override fun getCropRect() = Rect(0, 0, width, height)
+        override fun setCropRect(rect: Rect?) = Unit
+        override fun close() = Unit // The fixture owns and reuses this completed image.
+        override fun getPlanes(): Array<ImageProxy.PlaneProxy> = media.planes.map { plane ->
+            object : ImageProxy.PlaneProxy {
+                override fun getBuffer(): ByteBuffer = plane.buffer
+                override fun getRowStride() = plane.rowStride
+                override fun getPixelStride() = plane.pixelStride
+            }
+        }.toTypedArray()
+        override fun getImageInfo(): ImageInfo = object : ImageInfo {
+            override fun getRotationDegrees() = rotation
+            override fun getTimestamp() = media.timestamp
+            override fun getTagBundle(): TagBundle = TagBundle.emptyBundle()
+            override fun getSensorToBufferTransformMatrix() = Matrix()
+            override fun populateExifData(builder: ExifData.Builder) = Unit
+        }
+    }
+}

android/src/androidTest/java/com/rncamerakit/RecordingReactContext.java

@@ -0,0 +1,33 @@
+package com.rncamerakit;
+
+import android.content.Context;
+import com.facebook.react.bridge.BridgeReactContext;
+import com.facebook.react.uimanager.events.EventDispatcher;
+import com.facebook.react.uimanager.events.EventDispatcherProvider;
+import com.rncamerakit.events.ReadCodeEvent;
+import java.lang.reflect.Field;
+import java.lang.reflect.Proxy;
+import java.util.ArrayList;
+import java.util.List;
+
+/** Records the real native event before the JavaScript bridge boundary. */
+public final class RecordingReactContext extends BridgeReactContext implements EventDispatcherProvider {
+    public final List<String> values = new ArrayList<>();
+    private final EventDispatcher dispatcher = (EventDispatcher) Proxy.newProxyInstance(
+        EventDispatcher.class.getClassLoader(), new Class<?>[] {EventDispatcher.class},
+        (proxy, method, arguments) -> {
+            if (method.getName().equals("dispatchEvent")) {
+                ReadCodeEvent event = (ReadCodeEvent) arguments[0];
+                if (!event.getEventName().equals("topReadCode")) throw new AssertionError(event.getEventName());
+                Field value = ReadCodeEvent.class.getDeclaredField("codeStringValue");
+                value.setAccessible(true);
+                values.add((String) value.get(event));
+            }
+            return null;
+        }
+    );
+
+    public RecordingReactContext(Context context) { super(context); }
+    @Override public boolean isBridgeless() { return true; }
+    @Override public EventDispatcher getEventDispatcher() { return dispatcher; }
+}

android/src/main/java/com/rncamerakit/BarcodeCoordinateMapper.kt

@@ -0,0 +1,58 @@
+package com.rncamerakit
+
+import android.graphics.Matrix
+import android.graphics.Rect
+import android.graphics.RectF
+import android.util.Size
+
+/** Immutable snapshot of the visible selection at frame submission. */
+internal class BarcodePreviewGeometry(
+    private val generation: Long,
+    sensorToFrame: Matrix?,
+    frame: RectF?,
+) {
+    private val sensorToFrame = sensorToFrame?.let { Matrix(it) }
+    private val frame = frame?.let { RectF(it) }
+    val hasFrame: Boolean get() = frame != null
+
+    fun matches(other: BarcodePreviewGeometry): Boolean =
+        generation == other.generation && sensorToFrame == other.sensorToFrame && frame == other.frame
+
+    fun contains(bounds: Rect, image: BarcodeFrameGeometry): Boolean {
+        val selection = frame ?: return true
+        val transform = sensorToFrame ?: return false
+        val mapped = image.mapBounds(RectF(bounds), transform) ?: return false
+        return selection.contains(mapped)
+    }
+}
+
+/** ML Kit reports boxes after applying InputImage's rotation, in the full image.
+ * Its coordinates do not start at ImageProxy.cropRect's top-left. Map back to
+ * the sensor, then through PreviewView's crop/scale/mirror transform exactly once.
+ * The sensor matrices do not require changing ImageCapture's ViewPort or crop.
+ */
+internal class BarcodeFrameGeometry(
+    private val width: Int,
+    private val height: Int,
+    private val rotationDegrees: Int,
+    sensorToBuffer: Matrix,
+    val preview: BarcodePreviewGeometry? = null,
+) {
+    private val sensorToBuffer = Matrix(sensorToBuffer)
+    val imageSize = Size(width, height)
+
+    fun mapBounds(bounds: RectF, sensorToView: Matrix): RectF? {
+        if (width <= 0 || height <= 0 || rotationDegrees !in listOf(0, 90, 180, 270)) return null
+        val bufferToDetection = Matrix().apply { setRotate(rotationDegrees.toFloat()) }
+        val rotatedBuffer = RectF(0f, 0f, width.toFloat(), height.toFloat())
+        bufferToDetection.mapRect(rotatedBuffer)
+        bufferToDetection.postTranslate(-rotatedBuffer.left, -rotatedBuffer.top)
+        val sensorToDetection = Matrix().apply {
+            setConcat(bufferToDetection, sensorToBuffer)
+        }
+        val detectionToSensor = Matrix()
+        if (!sensorToDetection.invert(detectionToSensor)) return null
+        val detectionToView = Matrix().apply { setConcat(sensorToView, detectionToSensor) }
+        return RectF(bounds).also { detectionToView.mapRect(it) }
+    }
+}