diff --git a/grib/src/main/java/ucar/nc2/grib/grib2/Grib2DataReader.java b/grib/src/main/java/ucar/nc2/grib/grib2/Grib2DataReader.java index ac0b5b0008..0b09c7d99e 100644 --- a/grib/src/main/java/ucar/nc2/grib/grib2/Grib2DataReader.java +++ b/grib/src/main/java/ucar/nc2/grib/grib2/Grib2DataReader.java @@ -11,6 +11,7 @@ import edu.ucar.unidata.compression.jna.libaec.LibAec; import edu.ucar.unidata.compression.jna.libaec.LibAec.AecStream; import java.nio.ByteBuffer; +import java.nio.ByteOrder; import javax.annotation.Nullable; import ucar.nc2.grib.GribNumbers; import ucar.nc2.grib.GribUtils; @@ -1085,12 +1086,9 @@ private float[] getData50002(RandomAccessFile raf, Grib2Drs.Type50002 gdrs) thro } private float[] getData42(RandomAccessFile raf, Grib2Drs.Type42 gdrs) throws IOException { - byte[] decodedData; - // read CCSDS encoded stream from message int encodedLength = dataLength - 5; - byte[] inputData = new byte[encodedLength]; - raf.readFully(inputData); + byte[] inputData = new byte[Math.min(encodedLength, 64 * 1024)]; float[] data; if (encodedLength > 0) { @@ -1103,8 +1101,13 @@ private float[] getData42(RandomAccessFile raf, Grib2Drs.Type42 gdrs) throws IOE AecStream aecStreamDecode = AecStream.create(gdrs.numberOfBits, gdrs.blockSize, gdrs.referenceSampleInterval, gdrs.compressionOptionsMask); - // load data from grib message into memory - inputMemory.write(0, inputData, 0, inputData.length); + // Fill native input memory with bounded Java scratch storage. + for (int offset = 0; offset < encodedLength;) { + int count = Math.min(inputData.length, encodedLength - offset); + raf.readFully(inputData, 0, count); + inputMemory.write(offset, inputData, 0, count); + offset += count; + } aecStreamDecode.setInputMemory(inputMemory); aecStreamDecode.setOutputMemory(outputMemory); @@ -1115,32 +1118,27 @@ private float[] getData42(RandomAccessFile raf, Grib2Drs.Type42 gdrs) throws IOE System.out.printf("AEC Error: %s%n", ok); } - // read decoded data from native memory - decodedData = new byte[nbytesPerSample * totalNPoints]; - outputMemory.read(0, decodedData, 0, decodedData.length); - } - - // will use this to read out a long value using nbytesPerSample bytes - // see long getNextLong(ByteBuffer bb, int numberOfBytes) - ByteBuffer bb = ByteBuffer.wrap(decodedData); - - // decode following regulation 92.9.4, Note 4 - int D = gdrs.decimalScaleFactor; - float DD = (float) Math.pow((double) 10, (double) D); - float R = gdrs.referenceValue; - int E = gdrs.binaryScaleFactor; - float EE = (float) Math.pow(2.0, (double) E); - data = new float[decodedData.length]; - if (bitmap == null) { - for (int i = 0; i < totalNPoints; i++) { - data[i] = (R + getNextLong(bb, nbytesPerSample) * EE) / DD; - } - } else { - for (int i = 0; i < totalNPoints; i++) { - if (GribNumbers.testBitIsSet(bitmap[i / 8], i % 8)) { + // Keep the native output alive until conversion finishes; a byte-buffer view avoids a heap copy. + ByteBuffer bb = outputMemory.getByteBuffer(0, outputMemory.size()).order(ByteOrder.BIG_ENDIAN); + + // decode following regulation 92.9.4, Note 4 + int D = gdrs.decimalScaleFactor; + float DD = (float) Math.pow((double) 10, (double) D); + float R = gdrs.referenceValue; + int E = gdrs.binaryScaleFactor; + float EE = (float) Math.pow(2.0, (double) E); + data = new float[totalNPoints]; + if (bitmap == null) { + for (int i = 0; i < totalNPoints; i++) { data[i] = (R + getNextLong(bb, nbytesPerSample) * EE) / DD; - } else { - data[i] = staticMissingValue; + } + } else { + for (int i = 0; i < totalNPoints; i++) { + if (GribNumbers.testBitIsSet(bitmap[i / 8], i % 8)) { + data[i] = (R + getNextLong(bb, nbytesPerSample) * EE) / DD; + } else { + data[i] = staticMissingValue; + } } } } diff --git a/grib/src/test/java/ucar/nc2/grib/grib2/TestGrib2CcsdsInput.java b/grib/src/test/java/ucar/nc2/grib/grib2/TestGrib2CcsdsInput.java new file mode 100644 index 0000000000..1cfd4e47e5 --- /dev/null +++ b/grib/src/test/java/ucar/nc2/grib/grib2/TestGrib2CcsdsInput.java @@ -0,0 +1,119 @@ +/* + * Copyright (c) 2026 University Corporation for Atmospheric Research/Unidata + * See LICENSE for license information. + */ + +package ucar.nc2.grib.grib2; + +import static com.google.common.truth.Truth.assertThat; +import static org.junit.Assert.assertThrows; + +import com.google.common.io.ByteStreams; +import java.io.EOFException; +import java.io.IOException; +import java.io.InputStream; +import java.nio.ByteBuffer; +import java.util.Arrays; +import org.junit.Test; +import ucar.unidata.io.InMemoryRandomAccessFile; + +public class TestGrib2CcsdsInput { + private static final int GRID_POINTS = 32776; + private static final int HEADER_SIZE = 25; + + @Test + public void largeCompressedFieldUsesBoundedScratchStorage() throws IOException { + byte[] encoded = encodedField(); + assertThat(encoded.length).isGreaterThan(64 * 1024); + try (TrackingFile file = file(encoded, encoded.length, Integer.MAX_VALUE, true)) { + assertThat(read(file, GRID_POINTS, encoded.length)).isEqualTo(expectedValues()); + assertThat(file.maximumBufferSize).isAtMost(64 * 1024); + assertThat(file.readCalls).isGreaterThan(1); + assertThat(file.getFilePointer()).isEqualTo(HEADER_SIZE + encoded.length); + assertThat(file.read()).isEqualTo(0x42); + } + } + + @Test + public void partialReadsPreserveEverySample() throws IOException { + byte[] encoded = encodedField(); + try (TrackingFile file = file(encoded, encoded.length, 97, true)) { + assertThat(read(file, GRID_POINTS, encoded.length)).isEqualTo(expectedValues()); + assertThat(file.getFilePointer()).isEqualTo(HEADER_SIZE + encoded.length); + assertThat(file.read()).isEqualTo(0x42); + } + } + + @Test + public void truncatedCompressedFieldStillThrowsEof() throws IOException { + byte[] encoded = encodedField(); + try (TrackingFile file = file(Arrays.copyOf(encoded, encoded.length - 1), encoded.length, 97, false)) { + assertThrows(EOFException.class, () -> read(file, GRID_POINTS, encoded.length)); + } + } + + @Test + public void constantFieldNeedsNoInputTransfer() throws IOException { + try (TrackingFile file = file(new byte[0], 0, Integer.MAX_VALUE, true)) { + assertThat(read(file, 8, 0)).isEqualTo(new float[] {-2, -2, -2, -2, -2, -2, -2, -2}); + assertThat(file.readCalls).isEqualTo(0); + assertThat(file.getFilePointer()).isEqualTo(HEADER_SIZE); + assertThat(file.read()).isEqualTo(0x42); + } + } + + private static byte[] encodedField() throws IOException { + try (InputStream input = TestGrib2CcsdsInput.class.getResourceAsStream("ccsds-input-blocks.bin")) { + return ByteStreams.toByteArray(input); + } + } + + private static float[] expectedValues() { + float[] values = new float[GRID_POINTS]; + int state = 42; + for (int i = 0; i < values.length; i++) { + // The fixture encodes these unsigned 16-bit samples with libaec 1.1.7 (flags 12, block 8, RSI 16). + state ^= state << 13; + state ^= state >>> 17; + state ^= state << 5; + values[i] = (-2 + (state & 0xffff) * 2.0f) / 10.0f; + } + return values; + } + + private static TrackingFile file(byte[] payload, int encodedLength, int maximumRead, boolean trailer) { + ByteBuffer buffer = ByteBuffer.allocate(HEADER_SIZE + payload.length + (trailer ? 1 : 0)); + buffer.putFloat(-2).putShort((short) 1).putShort((short) 1).put((byte) 16).put((byte) 0); + buffer.put((byte) 12).put((byte) 8).putShort((short) 16); + buffer.putInt(6).put((byte) 6).put((byte) 255); + buffer.putInt(5 + encodedLength).put((byte) 7).put(payload); + if (trailer) { + buffer.put((byte) 0x42); + } + return new TrackingFile(buffer.array(), maximumRead); + } + + private static float[] read(TrackingFile file, int points, int encodedLength) throws IOException { + Grib2Drs.Type42 drs = new Grib2Drs.Type42(file); + Grib2SectionBitMap bitmap = new Grib2SectionBitMap(file); + return new Grib2DataReader(42, points, points, 0, 4, HEADER_SIZE - 5, encodedLength + 5).getData(file, bitmap, drs); + } + + private static final class TrackingFile extends InMemoryRandomAccessFile { + private final int maximumRead; + private int maximumBufferSize; + private int readCalls; + + TrackingFile(byte[] content, int maximumRead) { + super("ccsds-input", content); + this.maximumRead = maximumRead; + } + + @Override + public int read(byte[] bytes, int offset, int length) throws IOException { + maximumBufferSize = Math.max(maximumBufferSize, bytes.length); + readCalls++; + return super.read(bytes, offset, Math.min(length, maximumRead)); + } + } +} diff --git a/grib/src/test/java/ucar/nc2/grib/grib2/TestGrib2DataReaderCcsds.java b/grib/src/test/java/ucar/nc2/grib/grib2/TestGrib2DataReaderCcsds.java new file mode 100644 index 0000000000..2954161d34 --- /dev/null +++ b/grib/src/test/java/ucar/nc2/grib/grib2/TestGrib2DataReaderCcsds.java @@ -0,0 +1,119 @@ +/* + * Copyright (c) 2026 University Corporation for Atmospheric Research/Unidata + * See LICENSE for license information. + */ + +package ucar.nc2.grib.grib2; + +import static com.google.common.truth.Truth.assertThat; + +import com.google.common.io.BaseEncoding; +import java.io.IOException; +import java.nio.ByteBuffer; +import java.util.Arrays; +import java.util.Collection; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.junit.runners.Parameterized; +import ucar.unidata.io.InMemoryRandomAccessFile; + +@RunWith(Parameterized.class) +public class TestGrib2DataReaderCcsds { + // Unsigned, most-significant byte first, with preprocessing (GRIB2 code table 5.42). + private static final int COMPRESSION_OPTIONS = 4 | 8; + private static final int BLOCK_SIZE = 8; + private static final int REFERENCE_SAMPLE_INTERVAL = 16; + private static final int GRID_POINTS = 8; + private static final int NX = 4; + + // Small unsigned sample streams encoded with libaec, using the settings above. + @Parameterized.Parameters(name = "{0} bits per sample") + public static Collection parameters() { + return Arrays.asList(new Object[][] {{8, "C01F8084425529F0", "C01E03844AA7C000"}, + {16, "E0000FC040020020010028028014FFF8", "E0000F01C0020020050053FFE0000000"}, + {32, "F000000007E02000000100000010000000800000140000014000000A7FFFFFFC", + "F00000000780E00000010000001000000280000029FFFFFFF000000000000000"}}); + } + + private final int bits; + private final byte[] fullGrid; + private final byte[] withBitmap; + + public TestGrib2DataReaderCcsds(int bits, String fullGrid, String withBitmap) { + this.bits = bits; + this.fullGrid = BaseEncoding.base16().decode(fullGrid); + this.withBitmap = BaseEncoding.base16().decode(withBitmap); + } + + @Test + public void decodedArrayHasOneValuePerGridPoint() throws IOException { + long[] values = {0, 1, 2, 3, 10, 20, 30, (1L << bits) - 1}; + assertThat(read(values, null, 0)).isEqualTo(scaled(values)); + } + + @Test + public void bitmapRetainsMissingGridPoints() throws IOException { + long[] values = {0, 1, 2, 10, 20, (1L << bits) - 1}; + float[] scaled = scaled(values); + float[] expected = {scaled[0], Float.NaN, scaled[1], scaled[2], Float.NaN, scaled[3], scaled[4], scaled[5]}; + assertThat(read(values, new byte[] {(byte) 0b10110111}, 0)).isEqualTo(expected); + } + + @Test + public void negativeXScanReversesEachRow() throws IOException { + long[] values = {0, 1, 2, 3, 10, 20, 30, (1L << bits) - 1}; + float[] scaled = scaled(values); + float[] expected = {scaled[3], scaled[2], scaled[1], scaled[0], scaled[7], scaled[6], scaled[5], scaled[4]}; + assertThat(read(values, null, 128)).isEqualTo(expected); + } + + @Test + public void alternatingScanReversesOnlyAlternateRows() throws IOException { + long[] values = {0, 1, 2, 3, 10, 20, 30, (1L << bits) - 1}; + float[] scaled = scaled(values); + float[] expected = {scaled[0], scaled[1], scaled[2], scaled[3], scaled[7], scaled[6], scaled[5], scaled[4]}; + assertThat(read(values, null, 16)).isEqualTo(expected); + } + + @Test + public void decodedValuesRemainIndependentAfterOtherReads() throws IOException { + long[] values = {0, 1, 2, 3, 10, 20, 30, (1L << bits) - 1}; + float[] first = read(values, null, 0); + for (int i = 0; i < 16; i++) { + float[] next = read(new long[] {0, 1, 2, 10, 20, (1L << bits) - 1}, new byte[] {(byte) 0b10110111}, 0); + Arrays.fill(next, Float.NaN); + } + assertThat(first).isEqualTo(scaled(values)); + } + + private float[] read(long[] values, byte[] bitmap, int scanMode) throws IOException { + byte[] encoded = bitmap == null ? fullGrid : withBitmap; + int bitmapLength = 6 + (bitmap == null ? 0 : bitmap.length); + int dataLength = 5 + encoded.length; + ByteBuffer buffer = ByteBuffer.allocate(14 + bitmapLength + dataLength); + // Template 5.42, beginning at its reference value (octet 12). + buffer.putFloat(-2).putShort((short) 1).putShort((short) 1).put((byte) bits).put((byte) 0); + buffer.put((byte) COMPRESSION_OPTIONS).put((byte) BLOCK_SIZE).putShort((short) REFERENCE_SAMPLE_INTERVAL); + buffer.putInt(bitmapLength).put((byte) 6).put((byte) (bitmap == null ? 255 : 0)); + if (bitmap != null) { + buffer.put(bitmap); + } + long dataStart = buffer.position(); + buffer.putInt(dataLength).put((byte) 7).put(encoded); + + try (InMemoryRandomAccessFile raf = new InMemoryRandomAccessFile("ccsds", buffer.array())) { + Grib2Drs.Type42 drs = new Grib2Drs.Type42(raf); + Grib2SectionBitMap bitmapSection = new Grib2SectionBitMap(raf); + Grib2DataReader reader = new Grib2DataReader(42, GRID_POINTS, values.length, scanMode, NX, dataStart, dataLength); + return reader.getData(raf, bitmapSection, drs); + } + } + + private static float[] scaled(long[] values) { + float[] result = new float[values.length]; + for (int i = 0; i < values.length; i++) { + result[i] = (-2 + values[i] * 2.0f) / 10.0f; + } + return result; + } +} diff --git a/grib/src/test/resources/ucar/nc2/grib/grib2/ccsds-input-blocks.bin b/grib/src/test/resources/ucar/nc2/grib/grib2/ccsds-input-blocks.bin new file mode 100644 index 0000000000..b7c5d97ba8 Binary files /dev/null and b/grib/src/test/resources/ucar/nc2/grib/grib2/ccsds-input-blocks.bin differ