From d21863507221f0df9fddad9080cc02e16d09e7d9 Mon Sep 17 00:00:00 2001 From: Zakhar Pashkin <112114097+zack-dev-cm@users.noreply.github.com> Date: Mon, 14 Sep 2026 20:15:10 +0400 Subject: [PATCH] Map Android barcode bounds into the displayed preview frame --- android/build.gradle | 9 ++ .../com/rncamerakit/BarcodeNativeSceneTest.kt | 143 ++++++++++++++++++ .../rncamerakit/RecordingReactContext.java | 33 ++++ .../rncamerakit/BarcodeCoordinateMapper.kt | 58 +++++++ .../src/main/java/com/rncamerakit/CKCamera.kt | 120 ++++++++------- .../java/com/rncamerakit/QRCodeAnalyzer.kt | 35 ++++- .../BarcodeCoordinateMapperTest.kt | 116 ++++++++++++++ .../com/rncamerakit/BarcodeSelectionTest.kt | 143 ++++++++++++++++++ 8 files changed, 597 insertions(+), 60 deletions(-) create mode 100644 android/src/androidTest/java/com/rncamerakit/BarcodeNativeSceneTest.kt create mode 100644 android/src/androidTest/java/com/rncamerakit/RecordingReactContext.java create mode 100644 android/src/main/java/com/rncamerakit/BarcodeCoordinateMapper.kt create mode 100644 android/src/test/java/com/rncamerakit/BarcodeCoordinateMapperTest.kt create mode 100644 android/src/test/java/com/rncamerakit/BarcodeSelectionTest.kt diff --git a/android/build.gradle b/android/build.gradle index 145733b842..1ac088ec79 100644 --- a/android/build.gradle +++ b/android/build.gradle @@ -14,12 +14,14 @@ if (isNewArchitectureEnabled()) { } android { + testOptions { unitTests.includeAndroidResources = true } def agpVersion = com.android.Version.ANDROID_GRADLE_PLUGIN_VERSION if (agpVersion.tokenize('.')[0].toInteger() >= 7) { namespace "com.rncamerakit" } compileSdkVersion = 34 defaultConfig { + testInstrumentationRunner "androidx.test.runner.AndroidJUnitRunner" minSdkVersion = 24 // noinspection ExpiredTargetSdkVersion targetSdkVersion = 34 @@ -46,6 +48,13 @@ android { } dependencies { + testImplementation 'junit:junit:4.13.2' + testImplementation 'org.robolectric:robolectric:4.14.1' + testImplementation 'org.mockito:mockito-core:5.18.0' + androidTestImplementation 'androidx.test:runner:1.6.2' + androidTestImplementation 'androidx.test.ext:junit:1.2.1' + androidTestImplementation 'com.google.zxing:core:3.5.3' + implementation 'com.facebook.react:react-native:+' implementation group: 'com.drewnoakes', name: 'metadata-extractor', version: '2.18.0' implementation "org.jetbrains.kotlin:kotlin-stdlib-jdk7:$kotlinVersion" diff --git a/android/src/androidTest/java/com/rncamerakit/BarcodeNativeSceneTest.kt b/android/src/androidTest/java/com/rncamerakit/BarcodeNativeSceneTest.kt new file mode 100644 index 0000000000..bc1204a02f --- /dev/null +++ b/android/src/androidTest/java/com/rncamerakit/BarcodeNativeSceneTest.kt @@ -0,0 +1,143 @@ +package com.rncamerakit + +import android.annotation.SuppressLint +import android.graphics.ImageFormat +import android.graphics.Matrix +import android.graphics.Rect +import android.media.Image +import android.media.ImageReader +import android.media.ImageWriter +import android.os.Handler +import android.os.Looper +import android.util.Log +import android.util.Size +import androidx.camera.core.ImageInfo +import androidx.camera.core.ImageProxy +import androidx.camera.core.impl.TagBundle +import androidx.camera.core.impl.utils.ExifData +import androidx.camera.view.PreviewView +import androidx.test.ext.junit.runners.AndroidJUnit4 +import androidx.test.platform.app.InstrumentationRegistry +import com.facebook.react.uimanager.ThemedReactContext +import com.google.android.gms.tasks.Tasks +import com.google.zxing.BarcodeFormat +import com.google.zxing.EncodeHintType +import com.google.zxing.qrcode.QRCodeWriter +import com.rncamerakit.barcode.BarcodeFrame +import org.junit.Assert.* +import org.junit.Test +import org.junit.runner.RunWith +import java.nio.ByteBuffer +import java.util.concurrent.CountDownLatch +import java.util.concurrent.TimeUnit + +/** Real Android media images -> bundled ML Kit -> CKCamera -> native event IDs. + * Sensor/preview transforms are authored fixtures, not sensor/HAL measurements. + * No JavaScript, physical camera, scanner mock or screenshot oracle is used. + */ +@RunWith(AndroidJUnit4::class) +@SuppressLint("RestrictedApi", "UnsafeOptInUsageError") +class BarcodeNativeSceneTest { + @Test + fun authoredTwoCodeSceneSelectsOnlyInsideAcrossAllMlKitRotations() { + val instrumentation = InstrumentationRegistry.getInstrumentation() + val ready = CountDownLatch(1) + val reader = ImageReader.newInstance(1280, 720, ImageFormat.YUV_420_888, 2) + val writer = ImageWriter.newInstance(reader.surface, 2) + reader.setOnImageAvailableListener({ ready.countDown() }, Handler(Looper.getMainLooper())) + val image = writer.dequeueInputImage() + val pixels = ByteArray(1280 * 720) { 255.toByte() } + fun code(value: String, x: Int, y: Int, size: Int) { + val bits = QRCodeWriter().encode(value, BarcodeFormat.QR_CODE, size, size, + mapOf(EncodeHintType.MARGIN to 1)) + for (row in 0 until size) for (column in 0 until size) { + pixels[(y + row) * 1280 + x + column] = if (bits[column, row]) 0 else 255.toByte() + } + } + code("INSIDE", 600, 300, 100) + code("OUTSIDE", 300, 400, 90) + for ((index, plane) in image.planes.withIndex()) { + val width = if (index == 0) 1280 else 640 + val height = if (index == 0) 720 else 360 + for (y in 0 until height) for (x in 0 until width) { + plane.buffer.put(y * plane.rowStride + x * plane.pixelStride, + if (index == 0) pixels[y * 1280 + x] else 128.toByte()) + } + } + writer.queueInputImage(image) + assertTrue("Authored Android image did not arrive", ready.await(10, TimeUnit.SECONDS)) + val media = requireNotNull(reader.acquireNextImage()) + try { + for (rotation in listOf(0, 90, 180, 270)) { + lateinit var react: RecordingReactContext + lateinit var camera: CKCamera + lateinit var preview: BarcodePreviewGeometry + instrumentation.runOnMainSync { + react = RecordingReactContext(instrumentation.targetContext) + val context = ThemedReactContext(react, instrumentation.targetContext, "native-scene", 1) + camera = CKCamera(context).apply { id = 42 } + val fixturePreview = PreviewView(context).apply { layout(0, 0, 1280, 720) } + // Seed CameraX's real preview transform with authored surface metadata. + val transform = PreviewView::class.java.getDeclaredField("mPreviewTransform") + .apply { isAccessible = true }.get(fixturePreview) + for ((name, value) in mapOf( + "mResolution" to Size(1280, 720), + "mSurfaceCropRect" to Rect(0, 0, 1280, 720), + "mSensorToBufferTransform" to Matrix(), + )) { + transform.javaClass.getDeclaredField(name).apply { isAccessible = true } + .set(transform, value) + } + assertTrue(requireNotNull(fixturePreview.sensorToViewTransform).isIdentity) + val frame = BarcodeFrame(context).apply { frameRect = Rect(400, 180, 880, 540) } + CKCamera::class.java.getDeclaredField("viewFinder").apply { isAccessible = true } + .set(camera, fixturePreview) + CKCamera::class.java.getDeclaredField("barcodeFrame").apply { isAccessible = true } + .set(camera, frame) + preview = camera.snapshotBarcodePreview() + } + val callback = CountDownLatch(1) + val decoded = mutableSetOf() + val analyzer = QRCodeAnalyzer.withFrameGeometry({ barcodes, geometry -> + decoded.addAll(barcodes.map { it.rawValue }) + camera.onBarcodesDetected(barcodes, geometry) + callback.countDown() + }, 0) + val task = requireNotNull(analyzer.analyzeWithoutClosing(SceneImage(media, rotation), preview)) + Tasks.await(task, 30, TimeUnit.SECONDS) + assertTrue("Native ML Kit success callback did not arrive", callback.await(10, TimeUnit.SECONDS)) + assertEquals(setOf("INSIDE", "OUTSIDE"), decoded) + assertEquals(listOf("INSIDE"), react.values) + Log.i("CameraKitScene", "rotation=$rotation buffer=1280x720 crop=(0,0,1280,720) decoded=$decoded emitted=${react.values} transforms=authored") + } + } finally { + media.close() + writer.close() + reader.close() + } + } + + private class SceneImage(private val media: Image, private val rotation: Int) : ImageProxy { + override fun getImage(): Image = media + override fun getWidth() = media.width + override fun getHeight() = media.height + override fun getFormat() = media.format + override fun getCropRect() = Rect(0, 0, width, height) + override fun setCropRect(rect: Rect?) = Unit + override fun close() = Unit // The fixture owns and reuses this completed image. + override fun getPlanes(): Array = media.planes.map { plane -> + object : ImageProxy.PlaneProxy { + override fun getBuffer(): ByteBuffer = plane.buffer + override fun getRowStride() = plane.rowStride + override fun getPixelStride() = plane.pixelStride + } + }.toTypedArray() + override fun getImageInfo(): ImageInfo = object : ImageInfo { + override fun getRotationDegrees() = rotation + override fun getTimestamp() = media.timestamp + override fun getTagBundle(): TagBundle = TagBundle.emptyBundle() + override fun getSensorToBufferTransformMatrix() = Matrix() + override fun populateExifData(builder: ExifData.Builder) = Unit + } + } +} diff --git a/android/src/androidTest/java/com/rncamerakit/RecordingReactContext.java b/android/src/androidTest/java/com/rncamerakit/RecordingReactContext.java new file mode 100644 index 0000000000..a44b7eb7e7 --- /dev/null +++ b/android/src/androidTest/java/com/rncamerakit/RecordingReactContext.java @@ -0,0 +1,33 @@ +package com.rncamerakit; + +import android.content.Context; +import com.facebook.react.bridge.BridgeReactContext; +import com.facebook.react.uimanager.events.EventDispatcher; +import com.facebook.react.uimanager.events.EventDispatcherProvider; +import com.rncamerakit.events.ReadCodeEvent; +import java.lang.reflect.Field; +import java.lang.reflect.Proxy; +import java.util.ArrayList; +import java.util.List; + +/** Records the real native event before the JavaScript bridge boundary. */ +public final class RecordingReactContext extends BridgeReactContext implements EventDispatcherProvider { + public final List values = new ArrayList<>(); + private final EventDispatcher dispatcher = (EventDispatcher) Proxy.newProxyInstance( + EventDispatcher.class.getClassLoader(), new Class[] {EventDispatcher.class}, + (proxy, method, arguments) -> { + if (method.getName().equals("dispatchEvent")) { + ReadCodeEvent event = (ReadCodeEvent) arguments[0]; + if (!event.getEventName().equals("topReadCode")) throw new AssertionError(event.getEventName()); + Field value = ReadCodeEvent.class.getDeclaredField("codeStringValue"); + value.setAccessible(true); + values.add((String) value.get(event)); + } + return null; + } + ); + + public RecordingReactContext(Context context) { super(context); } + @Override public boolean isBridgeless() { return true; } + @Override public EventDispatcher getEventDispatcher() { return dispatcher; } +} diff --git a/android/src/main/java/com/rncamerakit/BarcodeCoordinateMapper.kt b/android/src/main/java/com/rncamerakit/BarcodeCoordinateMapper.kt new file mode 100644 index 0000000000..991cbcb74d --- /dev/null +++ b/android/src/main/java/com/rncamerakit/BarcodeCoordinateMapper.kt @@ -0,0 +1,58 @@ +package com.rncamerakit + +import android.graphics.Matrix +import android.graphics.Rect +import android.graphics.RectF +import android.util.Size + +/** Immutable snapshot of the visible selection at frame submission. */ +internal class BarcodePreviewGeometry( + private val generation: Long, + sensorToFrame: Matrix?, + frame: RectF?, +) { + private val sensorToFrame = sensorToFrame?.let { Matrix(it) } + private val frame = frame?.let { RectF(it) } + val hasFrame: Boolean get() = frame != null + + fun matches(other: BarcodePreviewGeometry): Boolean = + generation == other.generation && sensorToFrame == other.sensorToFrame && frame == other.frame + + fun contains(bounds: Rect, image: BarcodeFrameGeometry): Boolean { + val selection = frame ?: return true + val transform = sensorToFrame ?: return false + val mapped = image.mapBounds(RectF(bounds), transform) ?: return false + return selection.contains(mapped) + } +} + +/** ML Kit reports boxes after applying InputImage's rotation, in the full image. + * Its coordinates do not start at ImageProxy.cropRect's top-left. Map back to + * the sensor, then through PreviewView's crop/scale/mirror transform exactly once. + * The sensor matrices do not require changing ImageCapture's ViewPort or crop. + */ +internal class BarcodeFrameGeometry( + private val width: Int, + private val height: Int, + private val rotationDegrees: Int, + sensorToBuffer: Matrix, + val preview: BarcodePreviewGeometry? = null, +) { + private val sensorToBuffer = Matrix(sensorToBuffer) + val imageSize = Size(width, height) + + fun mapBounds(bounds: RectF, sensorToView: Matrix): RectF? { + if (width <= 0 || height <= 0 || rotationDegrees !in listOf(0, 90, 180, 270)) return null + val bufferToDetection = Matrix().apply { setRotate(rotationDegrees.toFloat()) } + val rotatedBuffer = RectF(0f, 0f, width.toFloat(), height.toFloat()) + bufferToDetection.mapRect(rotatedBuffer) + bufferToDetection.postTranslate(-rotatedBuffer.left, -rotatedBuffer.top) + val sensorToDetection = Matrix().apply { + setConcat(bufferToDetection, sensorToBuffer) + } + val detectionToSensor = Matrix() + if (!sensorToDetection.invert(detectionToSensor)) return null + val detectionToView = Matrix().apply { setConcat(sensorToView, detectionToSensor) } + return RectF(bounds).also { detectionToView.mapRect(it) } + } +} diff --git a/android/src/main/java/com/rncamerakit/CKCamera.kt b/android/src/main/java/com/rncamerakit/CKCamera.kt index bf8d9e7f44..b19916969e 100644 --- a/android/src/main/java/com/rncamerakit/CKCamera.kt +++ b/android/src/main/java/com/rncamerakit/CKCamera.kt @@ -41,7 +41,7 @@ import kotlin.math.max import kotlin.math.min import android.graphics.Canvas import android.graphics.Paint -import android.graphics.Rect +import android.graphics.Matrix import android.graphics.RectF import android.util.Size import com.facebook.react.uimanager.UIManagerHelper @@ -49,6 +49,7 @@ import com.google.android.gms.tasks.Task import com.google.android.gms.tasks.Tasks import com.google.mlkit.vision.barcode.common.Barcode import com.rncamerakit.events.* +import java.util.concurrent.RejectedExecutionException class RectOverlay constructor(context: Context) : View(context) { @@ -114,6 +115,7 @@ class CKCamera(context: ThemedReactContext) : FrameLayout(context), LifecycleObs private var laserColor = Color.RED private var barcodeFrameSize: Size? = null private var allowedBarcodeTypes: Array? = null + private var barcodeBindingGeneration: Long = 0 // Face detection props private var faceDetectionEnabled: Boolean = false @@ -147,6 +149,7 @@ class CKCamera(context: ThemedReactContext) : FrameLayout(context), LifecycleObs } override fun onDetachedFromWindow() { + barcodeBindingGeneration++ super.onDetachedFromWindow() faceAnalyzer?.close() @@ -314,6 +317,7 @@ class CKCamera(context: ThemedReactContext) : FrameLayout(context), LifecycleObs private fun bindCameraUseCases() { if (viewFinder.display == null) return + val bindingGeneration = ++barcodeBindingGeneration val previewWidth = viewFinder.getWidth(); val previewHeight = viewFinder.getHeight(); @@ -362,54 +366,7 @@ class CKCamera(context: ThemedReactContext) : FrameLayout(context), LifecycleObs faceAnalyzer = null val barcodeAnalyzer: QRCodeAnalyzer? = if (scanBarcode) { - QRCodeAnalyzer({ barcodes, imageSize -> - if (barcodes.isEmpty()) return@QRCodeAnalyzer - - // 1. Filter by allowed barcode formats - val allowedTypes = convertAllowedBarcodeTypes() - val filteredByType = if (allowedTypes.isEmpty()) { - barcodes - } else { - barcodes.filter { barcode -> - barcode.format in allowedTypes - } - } - - if (filteredByType.isEmpty()) return@QRCodeAnalyzer - - val barcodeFrame = barcodeFrame - val vf = viewFinder - - // 2. No frame? → behave like original code - if (barcodeFrame == null) { - onBarcodeRead(filteredByType) - return@QRCodeAnalyzer - } - - val frameRect = barcodeFrame.frameRect - - // 3. Calculate scaling factors (image is always rotated by 90 degrees) - val scaleX = vf.width.toFloat() / imageSize.height - val scaleY = vf.height.toFloat() / imageSize.width - - // 4. filter barcodes inside the frame - val filteredBarcodes = filteredByType.filter { barcode -> - val barcodeBoundingBox = barcode.boundingBox ?: return@filter false - - val scaledBarcodeBoundingBox = Rect( - (barcodeBoundingBox.left * scaleX).toInt(), - (barcodeBoundingBox.top * scaleY).toInt(), - (barcodeBoundingBox.right * scaleX).toInt(), - (barcodeBoundingBox.bottom * scaleY).toInt() - ) - frameRect.contains(scaledBarcodeBoundingBox) - } - - // 5. Emit if any left - if (filteredBarcodes.isNotEmpty()) { - onBarcodeRead(filteredBarcodes) - } - }, scanThrottleDelay) + QRCodeAnalyzer.withFrameGeometry(::onBarcodesDetected, scanThrottleDelay) } else null faceAnalyzer = if (faceDetectionEnabled) { @@ -424,12 +381,25 @@ class CKCamera(context: ThemedReactContext) : FrameLayout(context), LifecycleObs val activeFaceAnalyzer = faceAnalyzer if (barcodeAnalyzer != null || activeFaceAnalyzer != null) { - imageAnalyzer!!.setAnalyzer(cameraExecutor) { image -> - val tasks = mutableListOf>() - barcodeAnalyzer?.analyzeWithoutClosing(image)?.let { tasks.add(it) } - activeFaceAnalyzer?.analyzeWithoutClosing(image)?.let { tasks.add(it) } - if (tasks.isEmpty()) image.close() - else Tasks.whenAllComplete(tasks).addOnCompleteListener { image.close() } + // PreviewView's transform is a UI-thread API. Snapshot it before + // dispatching inference to the existing camera executor. + imageAnalyzer!!.setAnalyzer(ContextCompat.getMainExecutor(context)) { image -> + if (bindingGeneration != barcodeBindingGeneration) { + image.close() + return@setAnalyzer + } + val geometry = snapshotBarcodePreview() + try { + cameraExecutor.execute { + val tasks = mutableListOf>() + barcodeAnalyzer?.analyzeWithoutClosing(image, geometry)?.let { tasks.add(it) } + activeFaceAnalyzer?.analyzeWithoutClosing(image)?.let { tasks.add(it) } + if (tasks.isEmpty()) image.close() + else Tasks.whenAllComplete(tasks).addOnCompleteListener { image.close() } + } + } catch (_: RejectedExecutionException) { + image.close() + } } useCases.add(imageAnalyzer) } @@ -579,6 +549,46 @@ class CKCamera(context: ThemedReactContext) : FrameLayout(context), LifecycleObs rectOverlay.drawRectBounds(focusRects) } + internal fun snapshotBarcodePreview(): BarcodePreviewGeometry { + val frame = barcodeFrame + ?: return BarcodePreviewGeometry(barcodeBindingGeneration, null, null) + val selection = RectF(frame.frameRect) + val sensorToView = viewFinder.sensorToViewTransform + ?: return BarcodePreviewGeometry(barcodeBindingGeneration, null, selection) + + // PreviewView and BarcodeFrame are siblings. Include their offsets and + // local view matrices so the comparison happens in BarcodeFrame space. + val previewToParent = Matrix(viewFinder.matrix).apply { + postTranslate(viewFinder.left.toFloat(), viewFinder.top.toFloat()) + } + val frameToParent = Matrix(frame.matrix).apply { + postTranslate(frame.left.toFloat(), frame.top.toFloat()) + } + val parentToFrame = Matrix() + if (!frameToParent.invert(parentToFrame)) { + return BarcodePreviewGeometry(barcodeBindingGeneration, null, selection) + } + val sensorToFrame = Matrix().apply { + setConcat(previewToParent, sensorToView) + postConcat(parentToFrame) + } + return BarcodePreviewGeometry(barcodeBindingGeneration, sensorToFrame, selection) + } + + internal fun onBarcodesDetected(barcodes: List, geometry: BarcodeFrameGeometry) { + val submittedPreview = geometry.preview ?: return + // A rebind, unmount or layout change invalidates an in-flight selection. + if (!submittedPreview.matches(snapshotBarcodePreview())) return + val allowedTypes = convertAllowedBarcodeTypes() + val selected = barcodes.filter { barcode -> + (allowedTypes.isEmpty() || barcode.format in allowedTypes) && + (!submittedPreview.hasFrame || barcode.boundingBox?.let { + submittedPreview.contains(it, geometry) + } == true) + } + if (selected.isNotEmpty()) onBarcodeRead(selected) + } + private fun onBarcodeRead(barcodes: List) { val codeFormat = CodeFormat.fromBarcodeType(barcodes.first().format); val surfaceId = UIManagerHelper.getSurfaceId(currentContext) diff --git a/android/src/main/java/com/rncamerakit/QRCodeAnalyzer.kt b/android/src/main/java/com/rncamerakit/QRCodeAnalyzer.kt index 9faf67bd7c..0fdf622a8c 100644 --- a/android/src/main/java/com/rncamerakit/QRCodeAnalyzer.kt +++ b/android/src/main/java/com/rncamerakit/QRCodeAnalyzer.kt @@ -10,18 +10,43 @@ import com.google.mlkit.vision.barcode.BarcodeScanning import com.google.mlkit.vision.barcode.common.Barcode import com.google.mlkit.vision.common.InputImage -class QRCodeAnalyzer ( - private val onQRCodesDetected: (qrCodes: List, imageSize: Size) -> Unit, - private val scanThrottleDelay: Long = 0L +class QRCodeAnalyzer private constructor( + private val onQRCodesDetected: (qrCodes: List, geometry: BarcodeFrameGeometry) -> Unit, + private val scanThrottleDelay: Long, + @Suppress("UNUSED_PARAMETER") frameCoordinates: Unit, ) : ImageAnalysis.Analyzer { + // Preserve the existing native constructor and raw-buffer size callback. + constructor( + onQRCodesDetected: (qrCodes: List, imageSize: Size) -> Unit, + scanThrottleDelay: Long = 0L, + ) : this({ codes, geometry -> onQRCodesDetected(codes, geometry.imageSize) }, scanThrottleDelay, Unit) + + companion object { + internal fun withFrameGeometry( + onDetected: (List, BarcodeFrameGeometry) -> Unit, + scanThrottleDelay: Long, + ) = QRCodeAnalyzer(onDetected, scanThrottleDelay, Unit) + } + // Time in milliseconds of the last time we dispatched detected barcodes private var lastBarcodeDetectedTime: Long = 0L @SuppressLint("UnsafeExperimentalUsageError") @ExperimentalGetImage - fun analyzeWithoutClosing(image: ImageProxy): Task<*>? { + fun analyzeWithoutClosing(image: ImageProxy): Task<*>? = analyzeWithoutClosing(image, null) + + @SuppressLint("UnsafeExperimentalUsageError") + @ExperimentalGetImage + internal fun analyzeWithoutClosing( + image: ImageProxy, + preview: BarcodePreviewGeometry?, + ): Task<*>? { val mediaImage = image.image ?: return null + val geometry = BarcodeFrameGeometry( + image.width, image.height, image.imageInfo.rotationDegrees, + image.imageInfo.sensorToBufferTransformMatrix, preview, + ) val inputImage = InputImage.fromMediaImage(mediaImage, image.imageInfo.rotationDegrees) val scanner = BarcodeScanning.getClient() @@ -35,7 +60,7 @@ class QRCodeAnalyzer ( if (barcodes.isNotEmpty()) { lastBarcodeDetectedTime = now - onQRCodesDetected(barcodes, Size(image.width, image.height)) + onQRCodesDetected(barcodes, geometry) } } } diff --git a/android/src/test/java/com/rncamerakit/BarcodeCoordinateMapperTest.kt b/android/src/test/java/com/rncamerakit/BarcodeCoordinateMapperTest.kt new file mode 100644 index 0000000000..a35b78da8f --- /dev/null +++ b/android/src/test/java/com/rncamerakit/BarcodeCoordinateMapperTest.kt @@ -0,0 +1,116 @@ +package com.rncamerakit + +import android.graphics.Matrix +import android.graphics.Rect +import android.graphics.RectF +import org.junit.Assert.* +import org.junit.Test +import org.junit.runner.RunWith +import org.robolectric.RobolectricTestRunner +import org.robolectric.annotation.Config + +@RunWith(RobolectricTestRunner::class) +@Config(sdk = [34]) +class BarcodeCoordinateMapperTest { + private fun matrix(vararg values: Float) = Matrix().apply { setValues(values) } + + private fun assertRect(expected: RectF, actual: RectF?) { + assertNotNull(actual) + assertEquals(expected.left, actual!!.left, 0.001f) + assertEquals(expected.top, actual.top, 0.001f) + assertEquals(expected.right, actual.right, 0.001f) + assertEquals(expected.bottom, actual.bottom, 0.001f) + } + + @Test + fun mlKitRotationsAreUndoneExactlyOnce() { + // Authored rotations of buffer box (600,300)-(700,400), not mapper output. + val detections = listOf( + 0 to RectF(600f, 300f, 700f, 400f), + 90 to RectF(320f, 600f, 420f, 700f), + 180 to RectF(580f, 320f, 680f, 420f), + 270 to RectF(300f, 580f, 400f, 680f), + ) + for ((rotation, bounds) in detections) { + val geometry = BarcodeFrameGeometry(1280, 720, rotation, Matrix()) + assertRect(RectF(600f, 300f, 700f, 400f), geometry.mapBounds(bounds, Matrix())) + } + } + + @Test + fun portraitPreviewAndFrontMirrorComeFromThePreviewMatrix() { + val geometry = BarcodeFrameGeometry(1280, 720, 90, Matrix()) + val detection = RectF(320f, 600f, 420f, 700f) + val back = matrix(0f, -1f, 720f, 1f, 0f, 0f, 0f, 0f, 1f) + val front = matrix(0f, 1f, 0f, 1f, 0f, 0f, 0f, 0f, 1f) + assertRect(RectF(320f, 600f, 420f, 700f), geometry.mapBounds(detection, back)) + assertRect(RectF(300f, 600f, 400f, 700f), geometry.mapBounds(detection, front)) + } + + @Test + fun centerCropUsesOffsetsInsteadOfStretchingTheBuffer() { + val geometry = BarcodeFrameGeometry(400, 200, 0, Matrix()) + val crop = matrix(1f, 0f, -100f, 0f, 1f, 0f, 0f, 0f, 1f) + assertRect(RectF(0f, 50f, 100f, 150f), geometry.mapBounds(RectF(100f, 50f, 200f, 150f), crop)) + } + + @Test + fun analysisSensorCropAndPreviewCropMayDiffer() { + val sensorToBuffer = matrix(0.5f, 0f, -50f, 0f, 0.5f, -25f, 0f, 0f, 1f) + val sensorToView = matrix(1f, 0f, -300f, 0f, 1f, -50f, 0f, 0f, 1f) + val geometry = BarcodeFrameGeometry(400, 200, 0, sensorToBuffer) + assertRect(RectF(0f, 50f, 100f, 100f), geometry.mapBounds(RectF(100f, 25f, 150f, 50f), sensorToView)) + } + + @Test + fun fullBoxContainmentIncludesEdgesAndRejectsPartialAndOutsideBoxes() { + val selection = BarcodePreviewGeometry(1, Matrix(), RectF(10f, 20f, 110f, 120f)) + val geometry = BarcodeFrameGeometry(200, 200, 0, Matrix()) + assertTrue(selection.contains(Rect(20, 30, 30, 40), geometry)) + assertTrue(selection.contains(Rect(10, 20, 110, 120), geometry)) + assertFalse(selection.contains(Rect(9, 30, 30, 40), geometry)) + assertFalse(selection.contains(Rect(120, 30, 130, 40), geometry)) + } + + @Test + fun fractionalOutsideEdgeIsNotTruncatedIntoTheFrame() { + val shifted = matrix(1f, 0f, 0.25f, 0f, 1f, 0f, 0f, 0f, 1f) + val selection = BarcodePreviewGeometry(1, shifted, RectF(0f, 0f, 10f, 10f)) + val geometry = BarcodeFrameGeometry(20, 20, 0, Matrix()) + assertFalse(selection.contains(Rect(0, 0, 10, 10), geometry)) + val small = matrix(0.1f, 0f, 0f, 0f, 0.1f, 0f, 0f, 0f, 1f) + assertTrue(BarcodePreviewGeometry(1, small, RectF(10f, 10f, 10.1f, 10.1f)) + .contains(Rect(100, 100, 101, 101), geometry)) + } + + @Test + fun unavailableAndNoninvertibleTransformsCannotSelectAFramedBarcode() { + val geometry = BarcodeFrameGeometry(200, 200, 0, Matrix()) + assertFalse(BarcodePreviewGeometry(1, null, RectF(0f, 0f, 200f, 200f)) + .contains(Rect(1, 1, 10, 10), geometry)) + val singular = BarcodeFrameGeometry(200, 200, 0, Matrix().apply { setScale(0f, 0f) }) + assertNull(singular.mapBounds(RectF(1f, 1f, 10f, 10f), Matrix())) + } + + @Test + fun snapshotsCopyMutableMatricesAndRectangles() { + val transform = Matrix() + val frame = RectF(0f, 0f, 100f, 100f) + val selection = BarcodePreviewGeometry(1, transform, frame) + val geometry = BarcodeFrameGeometry(200, 200, 0, transform) + transform.setScale(0f, 0f) + frame.setEmpty() + assertTrue(selection.matches(BarcodePreviewGeometry(1, Matrix(), RectF(0f, 0f, 100f, 100f)))) + assertTrue(selection.contains(Rect(1, 1, 10, 10), geometry)) + } + + @Test + fun rebindResizeAndFrameMovementInvalidateInFlightSelections() { + val frame = RectF(0f, 0f, 100f, 100f) + val selection = BarcodePreviewGeometry(1, Matrix(), frame) + assertFalse(selection.matches(BarcodePreviewGeometry(2, Matrix(), frame))) + assertFalse(selection.matches(BarcodePreviewGeometry(1, Matrix().apply { setScale(2f, 2f) }, frame))) + assertFalse(selection.matches(BarcodePreviewGeometry(1, Matrix(), RectF(1f, 0f, 101f, 100f)))) + assertFalse(selection.matches(BarcodePreviewGeometry(1, null, null))) + } +} diff --git a/android/src/test/java/com/rncamerakit/BarcodeSelectionTest.kt b/android/src/test/java/com/rncamerakit/BarcodeSelectionTest.kt new file mode 100644 index 0000000000..339e79d81c --- /dev/null +++ b/android/src/test/java/com/rncamerakit/BarcodeSelectionTest.kt @@ -0,0 +1,143 @@ +package com.rncamerakit + +import android.graphics.Matrix +import android.graphics.Rect +import androidx.camera.view.PreviewView +import com.facebook.react.bridge.ReactApplicationContext +import com.facebook.react.uimanager.ThemedReactContext +import com.facebook.react.uimanager.UIManagerHelper +import com.facebook.react.uimanager.events.EventDispatcher +import com.google.mlkit.vision.barcode.common.Barcode +import com.rncamerakit.barcode.BarcodeFrame +import com.rncamerakit.events.ReadCodeEvent +import org.junit.Assert.assertEquals +import org.junit.Test +import org.junit.runner.RunWith +import org.mockito.Mockito.* +import org.mockito.stubbing.Answer +import org.robolectric.RobolectricTestRunner +import org.robolectric.RuntimeEnvironment +import org.robolectric.annotation.Config + +@RunWith(RobolectricTestRunner::class) +@Config(sdk = [34]) +class BarcodeSelectionTest { + private class Fixture { + val context = ThemedReactContext( + mock(ReactApplicationContext::class.java), RuntimeEnvironment.getApplication(), "selection", 1, + ) + val camera = CKCamera(context).apply { id = 42 } + val preview = spy(PreviewView(context)).apply { layout(0, 0, 1280, 720) } + val frame = BarcodeFrame(context).apply { frameRect = Rect(400, 180, 880, 540) } + + init { + set("viewFinder", preview) + set("barcodeFrame", frame) + `when`(preview.sensorToViewTransform).thenReturn(Matrix()) + } + + fun set(name: String, value: Any?) { + CKCamera::class.java.getDeclaredField(name).apply { isAccessible = true }.set(camera, value) + } + + fun geometry() = BarcodeFrameGeometry(1280, 720, 0, Matrix(), camera.snapshotBarcodePreview()) + + fun emit(codes: List, geometry: BarcodeFrameGeometry = geometry()): List { + val values = mutableListOf() + val dispatcher = mock(EventDispatcher::class.java, Answer { invocation -> + if (invocation.method.name == "dispatchEvent") { + val event = invocation.arguments[0] as ReadCodeEvent + assertEquals("topReadCode", event.eventName) + values.add(ReadCodeEvent::class.java.getDeclaredField("codeStringValue").apply { + isAccessible = true + }.get(event) as String?) + } + null + }) + mockStatic(UIManagerHelper::class.java).use { ui -> + ui.`when` { UIManagerHelper.getSurfaceId(context) }.thenReturn(1) + ui.`when` { + UIManagerHelper.getEventDispatcherForReactTag(context, camera.id) + }.thenReturn(dispatcher) + camera.onBarcodesDetected(codes, geometry) + } + return values + } + } + + private fun barcode(value: String, bounds: Rect?, format: Int = Barcode.FORMAT_QR_CODE): Barcode { + val barcode = mock(Barcode::class.java) + `when`(barcode.rawValue).thenReturn(value) + `when`(barcode.boundingBox).thenReturn(bounds) + `when`(barcode.format).thenReturn(format) + return barcode + } + + private fun scene() = listOf( + barcode("INSIDE", Rect(600, 300, 700, 400)), + barcode("OUTSIDE", Rect(300, 400, 390, 500)), + ) + + @Test + fun zeroRotationEmitsInsideBarcodeOnly() { + assertEquals(listOf("INSIDE"), Fixture().emit(scene())) + } + + @Test + fun unavailablePreviewTransformDefersFramedEvents() { + val fixture = Fixture() + `when`(fixture.preview.sensorToViewTransform).thenReturn(null) + assertEquals(emptyList(), fixture.emit(scene())) + } + + @Test + fun noFramePreservesDetectionsWithoutBoundingBoxes() { + val fixture = Fixture() + fixture.set("barcodeFrame", null) + `when`(fixture.preview.sensorToViewTransform).thenReturn(null) + assertEquals(listOf("UNFRAMED"), fixture.emit(listOf(barcode("UNFRAMED", null)))) + } + + @Test + fun formatFilterStillPrecedesTheFirstEmittedEvent() { + val fixture = Fixture() + fixture.set("barcodeFrame", null) + fixture.set("allowedBarcodeTypes", arrayOf(CodeFormat.QR)) + assertEquals(listOf("QR_ONLY"), fixture.emit(listOf( + barcode("CODE128", null, Barcode.FORMAT_CODE_128), barcode("QR_ONLY", null), + ))) + } + + @Test + fun lateDetectionCannotUseANewPreviewTransform() { + val fixture = Fixture() + val submitted = fixture.geometry() + `when`(fixture.preview.sensorToViewTransform).thenReturn(Matrix().apply { setScale(2f, 2f) }) + assertEquals(emptyList(), fixture.emit(scene(), submitted)) + } + + @Test + fun cameraSwitchOrUnmountInvalidatesTheSubmittedGeneration() { + val fixture = Fixture() + val submitted = fixture.geometry() + fixture.set("barcodeBindingGeneration", 1L) + assertEquals(emptyList(), fixture.emit(scene(), submitted)) + } + + @Test + fun previewAndFrameOffsetsAreIncluded() { + val fixture = Fixture() + fixture.preview.layout(20, 40, 1300, 760) + fixture.frame.layout(5, 7, 1285, 727) + fixture.frame.frameRect = Rect(615, 333, 715, 433) + assertEquals(listOf("INSIDE"), fixture.emit(scene())) + } + + @Test + fun movingTheScanFrameInvalidatesAPendingSelection() { + val fixture = Fixture() + val submitted = fixture.geometry() + fixture.frame.frameRect = Rect(500, 200, 900, 600) + assertEquals(emptyList(), fixture.emit(scene(), submitted)) + } +}