diff --git a/package.json b/package.json index de1baad38..cfdfeb724 100644 --- a/package.json +++ b/package.json @@ -71,6 +71,7 @@ "stream-browserify": "^3.0.0", "stream-http": "^3.2.0", "url": "^0.11.3", + "utif2": "4.1.0", "util": "^0.12.5", "uuid": "^14.0.0", "web-vitals": "^0.2.4", diff --git a/src/app/drive/services/image-preview.service/embeddedJpeg.node.test.ts b/src/app/drive/services/image-preview.service/embeddedJpeg.node.test.ts new file mode 100644 index 000000000..9eb4f8976 --- /dev/null +++ b/src/app/drive/services/image-preview.service/embeddedJpeg.node.test.ts @@ -0,0 +1,56 @@ +import { buildJpeg, concatBytes, JPEG_FRAME } from 'testUtils/imageBuilders'; +import { describe, expect, test } from 'vitest'; +import { findLargestEmbeddedJpeg } from './embeddedJpeg'; + +const noise = (length: number): Uint8Array => Uint8Array.from({ length }, (_, index) => (index * 37 + 11) & 0xff); + +describe('findLargestEmbeddedJpeg', () => { + test('when the bytes contain no JPEG, then it returns null', () => { + expect(findLargestEmbeddedJpeg(noise(4096))).toBeNull(); + expect(findLargestEmbeddedJpeg(new Uint8Array(0))).toBeNull(); + }); + + test('when a JPEG is surrounded by other bytes, then it returns exactly the JPEG bytes', () => { + const jpeg = buildJpeg({ width: 640, height: 480 }); + + expect(findLargestEmbeddedJpeg(concatBytes(noise(300), jpeg, noise(200)))).toEqual(jpeg); + }); + + test('when several JPEGs are embedded, then it returns the one with most pixels', () => { + const small = buildJpeg({ width: 160, height: 120 }); + const large = buildJpeg({ width: 1920, height: 1280 }); + const medium = buildJpeg({ width: 800, height: 600 }); + + expect(findLargestEmbeddedJpeg(concatBytes(small, noise(50), large, noise(50), medium))).toEqual(large); + }); + + test('when larger streams are not browser-decodable colour JPEGs, then they are ignored', () => { + const preview = buildJpeg({ width: 1024, height: 768 }); + const losslessSensorData = buildJpeg({ width: 6000, height: 4000, frameMarker: JPEG_FRAME.lossless }); + const twelveBitTile = buildJpeg({ + width: 1952, + height: 552, + frameMarker: JPEG_FRAME.extendedSequential, + precision: 12, + }); + const grayscaleTile = buildJpeg({ width: 4000, height: 3000, components: 1 }); + + const bytes = concatBytes(losslessSensorData, twelveBitTile, preview, grayscaleTile); + + expect(findLargestEmbeddedJpeg(bytes)).toEqual(preview); + }); + + test('when a stream is malformed or truncated, then it is skipped in favour of a complete one', () => { + const truncated = buildJpeg({ width: 4000, height: 3000, hasEndOfImage: false }); + const interruptedByNewImage = concatBytes(truncated, buildJpeg({ width: 320, height: 240 })); + const brokenSegment = Uint8Array.from([0xff, 0xd8, 0xff, 0xe0, 0x00, 0x10, 0x00]); + + expect(findLargestEmbeddedJpeg(truncated)).toBeNull(); + expect(findLargestEmbeddedJpeg(interruptedByNewImage)).toEqual(buildJpeg({ width: 320, height: 240 })); + expect(findLargestEmbeddedJpeg(brokenSegment)).toBeNull(); + }); + + test('when the only stream is smaller than 64px, then it returns null', () => { + expect(findLargestEmbeddedJpeg(buildJpeg({ width: 32, height: 24 }))).toBeNull(); + }); +}); diff --git a/src/app/drive/services/image-preview.service/embeddedJpeg.ts b/src/app/drive/services/image-preview.service/embeddedJpeg.ts new file mode 100644 index 000000000..758e175cd --- /dev/null +++ b/src/app/drive/services/image-preview.service/embeddedJpeg.ts @@ -0,0 +1,118 @@ +const MARKER_PREFIX = 0xff; +const START_OF_IMAGE = 0xd8; +const END_OF_IMAGE = 0xd9; +const START_OF_SCAN = 0xda; +const BROWSER_DECODABLE_FRAMES = new Set([0xc0, 0xc1, 0xc2]); +const OTHER_FRAME_MARKERS = new Set([0xc3, 0xc5, 0xc6, 0xc7, 0xc9, 0xca, 0xcb, 0xcd, 0xce, 0xcf]); +const STANDALONE_MARKERS = new Set([0x01, 0xd0, 0xd1, 0xd2, 0xd3, 0xd4, 0xd5, 0xd6, 0xd7]); +const EIGHT_BIT_PRECISION = 8; +const COLOR_COMPONENT_COUNT = 3; +const MIN_PREVIEW_LONG_SIDE = 64; +const MARKER_SIZE = 2; +const MARKER_CODE_OFFSET = 1; +const FILL_BYTE_SIZE = 1; +const SEGMENT_LENGTH_SIZE = 2; +const START_OF_IMAGE_SIGNATURE_SIZE = 3; +const FRAME_HEADER_OFFSET = { precision: 4, height: 5, width: 7, componentCount: 9 }; + +interface JpegCandidate { + start: number; + end: number; + pixelCount: number; +} + +interface FrameDimensions { + width: number; + height: number; +} + +const readUint16 = (bytes: Uint8Array, offset: number): number => (bytes[offset] << 8) | bytes[offset + 1]; + +const isStartOfImageAt = (bytes: Uint8Array, position: number): boolean => + bytes[position] === MARKER_PREFIX && + bytes[position + MARKER_CODE_OFFSET] === START_OF_IMAGE && + bytes[position + MARKER_SIZE] === MARKER_PREFIX; + +const findEndOfImage = (bytes: Uint8Array, scanStart: number): number | null => { + for (let position = scanStart; position + MARKER_SIZE <= bytes.length; position++) { + if (bytes[position] !== MARKER_PREFIX) continue; + if (bytes[position + MARKER_CODE_OFFSET] === END_OF_IMAGE) return position + MARKER_SIZE; + if (bytes[position + MARKER_CODE_OFFSET] === START_OF_IMAGE) return null; + } + return null; +}; + +/** + * Reads the dimensions of the frame header starting at `position`. Returns null when the + * frame is not 8-bit / 3-component or is too small to serve as a preview. + */ +const readFrameHeader = (bytes: Uint8Array, position: number): FrameDimensions | null => { + const isEightBitColor = + bytes[position + FRAME_HEADER_OFFSET.precision] === EIGHT_BIT_PRECISION && + bytes[position + FRAME_HEADER_OFFSET.componentCount] === COLOR_COMPONENT_COUNT; + if (!isEightBitColor) return null; + + const height = readUint16(bytes, position + FRAME_HEADER_OFFSET.height); + const width = readUint16(bytes, position + FRAME_HEADER_OFFSET.width); + if (Math.max(width, height) < MIN_PREVIEW_LONG_SIDE) return null; + + return { width, height }; +}; + +/** + * Walks the JPEG segments starting at `start`. Returns null when the stream is malformed + * or is not an 8-bit colour JPEG browsers can decode (lossless or 12-bit sensor tiles). + */ +const parseJpegAt = (bytes: Uint8Array, start: number): JpegCandidate | null => { + let position = start + MARKER_SIZE; + let pixelCount = 0; + + while (position + MARKER_SIZE + SEGMENT_LENGTH_SIZE <= bytes.length) { + if (bytes[position] !== MARKER_PREFIX) return null; + const marker = bytes[position + MARKER_CODE_OFFSET]; + + if (marker === MARKER_PREFIX || STANDALONE_MARKERS.has(marker)) { + position += marker === MARKER_PREFIX ? FILL_BYTE_SIZE : MARKER_SIZE; + continue; + } + if (marker === START_OF_IMAGE || marker === END_OF_IMAGE || OTHER_FRAME_MARKERS.has(marker)) return null; + + if (BROWSER_DECODABLE_FRAMES.has(marker)) { + const frame = readFrameHeader(bytes, position); + if (!frame) return null; + pixelCount = frame.width * frame.height; + } + + position += MARKER_SIZE + readUint16(bytes, position + MARKER_SIZE); + + if (marker === START_OF_SCAN) { + const end = pixelCount > 0 ? findEndOfImage(bytes, position) : null; + return end === null ? null : { start, end, pixelCount }; + } + } + + return null; +}; + +/** + * Returns the largest browser-decodable JPEG embedded in a binary file (camera RAW + * files carry one as preview), or null when there is none. + */ +export const findLargestEmbeddedJpeg = (bytes: Uint8Array): Uint8Array | null => { + let largest: JpegCandidate | null = null; + let position = 0; + + while (position + START_OF_IMAGE_SIGNATURE_SIZE < bytes.length) { + const candidate = isStartOfImageAt(bytes, position) ? parseJpegAt(bytes, position) : null; + + if (!candidate) { + position += 1; + continue; + } + + if (!largest || candidate.pixelCount > largest.pixelCount) largest = candidate; + position = candidate.end; + } + + return largest ? bytes.slice(largest.start, largest.end) : null; +}; diff --git a/src/testUtils/imageBuilders.ts b/src/testUtils/imageBuilders.ts new file mode 100644 index 000000000..1f89bd697 --- /dev/null +++ b/src/testUtils/imageBuilders.ts @@ -0,0 +1,157 @@ +/** + * Builders for tiny synthetic JPEG and TIFF files used by the image preview tests, + * so no binary fixtures need to live in the repository. + */ + +export const JPEG_FRAME = { baseline: 0xc0, extendedSequential: 0xc1, lossless: 0xc3 }; +export const TIFF_PHOTOMETRIC = { rgb: 2, colorFilterArray: 32803 }; +export const TIFF_COMPRESSION = { none: 1, nikon: 34713 }; +/** Pixels written by `buildTiff`: red grows from 0 to 255 left to right, green and blue are fixed. */ +export const TIFF_PIXEL = { green: 20, blue: 200 }; + +const RGB_CHANNELS = 3; +const JPEG_MARKER_PREFIX = 0xff; + +const uint16 = (value: number): number[] => [value >> 8, value & 0xff]; + +const jpegSegment = (marker: number, payload: number[]): number[] => [ + JPEG_MARKER_PREFIX, + marker, + ...uint16(payload.length + 2), + ...payload, +]; + +export const concatBytes = (...chunks: Uint8Array[]): Uint8Array => { + const result = new Uint8Array(chunks.reduce((total, chunk) => total + chunk.length, 0)); + let offset = 0; + for (const chunk of chunks) { + result.set(chunk, offset); + offset += chunk.length; + } + return result; +}; + +interface JpegOptions { + width: number; + height: number; + frameMarker?: number; + precision?: number; + components?: number; + hasEndOfImage?: boolean; +} + +/** Builds a structurally valid JPEG (markers only, dummy scan data) with the given frame header. */ +export const buildJpeg = ({ + width, + height, + frameMarker = JPEG_FRAME.baseline, + precision = 8, + components = 3, + hasEndOfImage = true, +}: JpegOptions): Uint8Array => { + const componentSpecs = Array.from({ length: components }, (_, index) => [index + 1, 0x11, 0]).flat(); + const scanData = [0x12, 0x34, 0xff, 0x00, 0x56, 0xff, 0xd0, 0x78, 0x9a]; + + return Uint8Array.from([ + 0xff, + 0xd8, + ...jpegSegment(0xe0, [0x4a, 0x46, 0x49, 0x46, 0x00]), + ...jpegSegment(frameMarker, [precision, ...uint16(height), ...uint16(width), components, ...componentSpecs]), + ...jpegSegment(0xda, [components, 1, 0, 0, 63, 0]), + ...scanData, + ...(hasEndOfImage ? [0xff, 0xd9] : []), + ]); +}; + +/** Reads width and height from the first frame header of a JPEG. */ +export const readJpegDimensions = (bytes: Uint8Array): { width: number; height: number } | null => { + for ( + let position = 2; + position + 9 < bytes.length; + position += 2 + ((bytes[position + 2] << 8) | bytes[position + 3]) + ) { + const isFrameHeader = [JPEG_FRAME.baseline, JPEG_FRAME.extendedSequential, 0xc2].includes(bytes[position + 1]); + if (isFrameHeader) { + return { + height: (bytes[position + 5] << 8) | bytes[position + 6], + width: (bytes[position + 7] << 8) | bytes[position + 8], + }; + } + } + return null; +}; + +export interface TiffPage { + width: number; + height: number; + photometric?: number; + compression?: number; +} + +const TIFF_ENTRY_COUNT = 9; +const TIFF_ENTRY_SIZE = 12; +const TIFF_IFD_SIZE = 2 + TIFF_ENTRY_COUNT * TIFF_ENTRY_SIZE + 4; +const TIFF_BITS_PER_SAMPLE_SIZE = 6; +const SHORT = 3; +const LONG = 4; + +const writeTiffPage = (view: DataView, bytes: Uint8Array, ifdOffset: number, page: TiffPage, nextIfdOffset: number) => { + const { width, height, photometric = TIFF_PHOTOMETRIC.rgb, compression = TIFF_COMPRESSION.none } = page; + const bitsOffset = ifdOffset + TIFF_IFD_SIZE; + const pixelsOffset = bitsOffset + TIFF_BITS_PER_SAMPLE_SIZE; + const entries: Array<[tag: number, type: number, count: number, value: number]> = [ + [256, LONG, 1, width], + [257, LONG, 1, height], + [258, SHORT, RGB_CHANNELS, bitsOffset], + [259, SHORT, 1, compression], + [262, SHORT, 1, photometric], + [273, LONG, 1, pixelsOffset], + [277, SHORT, 1, RGB_CHANNELS], + [278, LONG, 1, height], + [279, LONG, 1, width * height * RGB_CHANNELS], + ]; + + view.setUint16(ifdOffset, entries.length); + entries.forEach(([tag, type, count, value], index) => { + const entryOffset = ifdOffset + 2 + index * TIFF_ENTRY_SIZE; + const isInlineShort = type === SHORT && count === 1; + view.setUint16(entryOffset, tag); + view.setUint16(entryOffset + 2, type); + view.setUint32(entryOffset + 4, count); + if (isInlineShort) view.setUint16(entryOffset + 8, value); + else view.setUint32(entryOffset + 8, value); + }); + view.setUint32(ifdOffset + 2 + entries.length * TIFF_ENTRY_SIZE, nextIfdOffset); + [0, 2, 4].forEach((offset) => view.setUint16(bitsOffset + offset, 8)); + + for (let y = 0; y < height; y++) { + for (let x = 0; x < width; x++) { + const offset = pixelsOffset + (y * width + x) * RGB_CHANNELS; + bytes[offset] = Math.round((x * 255) / Math.max(width - 1, 1)); + bytes[offset + 1] = TIFF_PIXEL.green; + bytes[offset + 2] = TIFF_PIXEL.blue; + } + } +}; + +/** Builds a big-endian, uncompressed RGB TIFF with one single-strip page per entry. */ +export const buildTiff = (pages: TiffPage[]): ArrayBuffer => { + const pageSize = (page: TiffPage) => + TIFF_IFD_SIZE + TIFF_BITS_PER_SAMPLE_SIZE + page.width * page.height * RGB_CHANNELS; + const buffer = new ArrayBuffer(8 + pages.reduce((total, page) => total + pageSize(page), 0)); + const view = new DataView(buffer); + const bytes = new Uint8Array(buffer); + + bytes.set([0x4d, 0x4d, 0x00, 0x2a]); + view.setUint32(4, 8); + + let ifdOffset = 8; + pages.forEach((page, index) => { + const isLastPage = index === pages.length - 1; + const nextIfdOffset = isLastPage ? 0 : ifdOffset + pageSize(page); + writeTiffPage(view, bytes, ifdOffset, page, nextIfdOffset); + ifdOffset = nextIfdOffset; + }); + + return buffer; +}; diff --git a/yarn.lock b/yarn.lock index f21c8ce62..4b3c1c87c 100644 --- a/yarn.lock +++ b/yarn.lock @@ -9681,6 +9681,13 @@ use-sync-external-store@^1.6.0: resolved "https://registry.yarnpkg.com/use-sync-external-store/-/use-sync-external-store-1.6.0.tgz#b174bfa65cb2b526732d9f2ac0a408027876f32d" integrity sha512-Pp6GSwGP/NrPIrxVFAIkOQeyw8lFenOHijQWkUTrDvrF4ALqylP2C/KCkeS9dpUM3KvYRQhna5vt7IL95+ZQ9w== +utif2@4.1.0: + version "4.1.0" + resolved "https://registry.yarnpkg.com/utif2/-/utif2-4.1.0.tgz#e768d37bd619b995d56d9780b5d2b4611a3d932b" + integrity sha512-+oknB9FHrJ7oW7A2WZYajOcv4FcDR4CfoGB0dPNfxbi4GO05RRnFmt5oa23+9w32EanrYcSJWspUiJkLMs+37w== + dependencies: + pako "^1.0.11" + util-deprecate@^1.0.1, util-deprecate@^1.0.2, util-deprecate@~1.0.1: version "1.0.2" resolved "https://registry.yarnpkg.com/util-deprecate/-/util-deprecate-1.0.2.tgz#450d4dc9fa70de732762fbd2d4a28981419a0ccf"