Reduce transient allocations when decoding CCSDS GRIB2 fields

grib/src/main/java/ucar/nc2/grib/grib2/Grib2DataReader.java

@@ -11,6 +11,7 @@
 import edu.ucar.unidata.compression.jna.libaec.LibAec;
 import edu.ucar.unidata.compression.jna.libaec.LibAec.AecStream;
 import java.nio.ByteBuffer;
+import java.nio.ByteOrder;
 import javax.annotation.Nullable;
 import ucar.nc2.grib.GribNumbers;
 import ucar.nc2.grib.GribUtils;
@@ -1085,12 +1086,9 @@ private float[] getData50002(RandomAccessFile raf, Grib2Drs.Type50002 gdrs) thro
   }
 
   private float[] getData42(RandomAccessFile raf, Grib2Drs.Type42 gdrs) throws IOException {
-    byte[] decodedData;
-
     // read CCSDS encoded stream from message
     int encodedLength = dataLength - 5;
-    byte[] inputData = new byte[encodedLength];
-    raf.readFully(inputData);
+    byte[] inputData = new byte[Math.min(encodedLength, 64 * 1024)];
 
     float[] data;
     if (encodedLength > 0) {
@@ -1103,8 +1101,13 @@ private float[] getData42(RandomAccessFile raf, Grib2Drs.Type42 gdrs) throws IOE
         AecStream aecStreamDecode = AecStream.create(gdrs.numberOfBits, gdrs.blockSize, gdrs.referenceSampleInterval,
             gdrs.compressionOptionsMask);
 
-        // load data from grib message into memory
-        inputMemory.write(0, inputData, 0, inputData.length);
+        // Fill native input memory with bounded Java scratch storage.
+        for (int offset = 0; offset < encodedLength;) {
+          int count = Math.min(inputData.length, encodedLength - offset);
+          raf.readFully(inputData, 0, count);
+          inputMemory.write(offset, inputData, 0, count);
+          offset += count;
+        }
 
         aecStreamDecode.setInputMemory(inputMemory);
         aecStreamDecode.setOutputMemory(outputMemory);
@@ -1115,32 +1118,27 @@ private float[] getData42(RandomAccessFile raf, Grib2Drs.Type42 gdrs) throws IOE
           System.out.printf("AEC Error: %s%n", ok);
         }
 
-        // read decoded data from native memory
-        decodedData = new byte[nbytesPerSample * totalNPoints];
-        outputMemory.read(0, decodedData, 0, decodedData.length);
-      }
-
-      // will use this to read out a long value using nbytesPerSample bytes
-      // see long getNextLong(ByteBuffer bb, int numberOfBytes)
-      ByteBuffer bb = ByteBuffer.wrap(decodedData);
-
-      // decode following regulation 92.9.4, Note 4
-      int D = gdrs.decimalScaleFactor;
-      float DD = (float) Math.pow((double) 10, (double) D);
-      float R = gdrs.referenceValue;
-      int E = gdrs.binaryScaleFactor;
-      float EE = (float) Math.pow(2.0, (double) E);
-      data = new float[decodedData.length];
-      if (bitmap == null) {
-        for (int i = 0; i < totalNPoints; i++) {
-          data[i] = (R + getNextLong(bb, nbytesPerSample) * EE) / DD;
-        }
-      } else {
-        for (int i = 0; i < totalNPoints; i++) {
-          if (GribNumbers.testBitIsSet(bitmap[i / 8], i % 8)) {
+        // Keep the native output alive until conversion finishes; a byte-buffer view avoids a heap copy.
+        ByteBuffer bb = outputMemory.getByteBuffer(0, outputMemory.size()).order(ByteOrder.BIG_ENDIAN);
+
+        // decode following regulation 92.9.4, Note 4
+        int D = gdrs.decimalScaleFactor;
+        float DD = (float) Math.pow((double) 10, (double) D);
+        float R = gdrs.referenceValue;
+        int E = gdrs.binaryScaleFactor;
+        float EE = (float) Math.pow(2.0, (double) E);
+        data = new float[totalNPoints];
+        if (bitmap == null) {
+          for (int i = 0; i < totalNPoints; i++) {
             data[i] = (R + getNextLong(bb, nbytesPerSample) * EE) / DD;
-          } else {
-            data[i] = staticMissingValue;
+          }
+        } else {
+          for (int i = 0; i < totalNPoints; i++) {
+            if (GribNumbers.testBitIsSet(bitmap[i / 8], i % 8)) {
+              data[i] = (R + getNextLong(bb, nbytesPerSample) * EE) / DD;
+            } else {
+              data[i] = staticMissingValue;
+            }
           }
         }
       }

grib/src/test/java/ucar/nc2/grib/grib2/TestGrib2CcsdsInput.java

@@ -0,0 +1,119 @@
+/*
+ * Copyright (c) 2026 University Corporation for Atmospheric Research/Unidata
+ * See LICENSE for license information.
+ */
+
+package ucar.nc2.grib.grib2;
+
+import static com.google.common.truth.Truth.assertThat;
+import static org.junit.Assert.assertThrows;
+
+import com.google.common.io.ByteStreams;
+import java.io.EOFException;
+import java.io.IOException;
+import java.io.InputStream;
+import java.nio.ByteBuffer;
+import java.util.Arrays;
+import org.junit.Test;
+import ucar.unidata.io.InMemoryRandomAccessFile;
+
+public class TestGrib2CcsdsInput {
+  private static final int GRID_POINTS = 32776;
+  private static final int HEADER_SIZE = 25;
+
+  @Test
+  public void largeCompressedFieldUsesBoundedScratchStorage() throws IOException {
+    byte[] encoded = encodedField();
+    assertThat(encoded.length).isGreaterThan(64 * 1024);
+    try (TrackingFile file = file(encoded, encoded.length, Integer.MAX_VALUE, true)) {
+      assertThat(read(file, GRID_POINTS, encoded.length)).isEqualTo(expectedValues());
+      assertThat(file.maximumBufferSize).isAtMost(64 * 1024);
+      assertThat(file.readCalls).isGreaterThan(1);
+      assertThat(file.getFilePointer()).isEqualTo(HEADER_SIZE + encoded.length);
+      assertThat(file.read()).isEqualTo(0x42);
+    }
+  }
+
+  @Test
+  public void partialReadsPreserveEverySample() throws IOException {
+    byte[] encoded = encodedField();
+    try (TrackingFile file = file(encoded, encoded.length, 97, true)) {
+      assertThat(read(file, GRID_POINTS, encoded.length)).isEqualTo(expectedValues());
+      assertThat(file.getFilePointer()).isEqualTo(HEADER_SIZE + encoded.length);
+      assertThat(file.read()).isEqualTo(0x42);
+    }
+  }
+
+  @Test
+  public void truncatedCompressedFieldStillThrowsEof() throws IOException {
+    byte[] encoded = encodedField();
+    try (TrackingFile file = file(Arrays.copyOf(encoded, encoded.length - 1), encoded.length, 97, false)) {
+      assertThrows(EOFException.class, () -> read(file, GRID_POINTS, encoded.length));
+    }
+  }
+
+  @Test
+  public void constantFieldNeedsNoInputTransfer() throws IOException {
+    try (TrackingFile file = file(new byte[0], 0, Integer.MAX_VALUE, true)) {
+      assertThat(read(file, 8, 0)).isEqualTo(new float[] {-2, -2, -2, -2, -2, -2, -2, -2});
+      assertThat(file.readCalls).isEqualTo(0);
+      assertThat(file.getFilePointer()).isEqualTo(HEADER_SIZE);
+      assertThat(file.read()).isEqualTo(0x42);
+    }
+  }
+
+  private static byte[] encodedField() throws IOException {
+    try (InputStream input = TestGrib2CcsdsInput.class.getResourceAsStream("ccsds-input-blocks.bin")) {
+      return ByteStreams.toByteArray(input);
+    }
+  }
+
+  private static float[] expectedValues() {
+    float[] values = new float[GRID_POINTS];
+    int state = 42;
+    for (int i = 0; i < values.length; i++) {
+      // The fixture encodes these unsigned 16-bit samples with libaec 1.1.7 (flags 12, block 8, RSI 16).
+      state ^= state << 13;
+      state ^= state >>> 17;
+      state ^= state << 5;
+      values[i] = (-2 + (state & 0xffff) * 2.0f) / 10.0f;
+    }
+    return values;
+  }
+
+  private static TrackingFile file(byte[] payload, int encodedLength, int maximumRead, boolean trailer) {
+    ByteBuffer buffer = ByteBuffer.allocate(HEADER_SIZE + payload.length + (trailer ? 1 : 0));
+    buffer.putFloat(-2).putShort((short) 1).putShort((short) 1).put((byte) 16).put((byte) 0);
+    buffer.put((byte) 12).put((byte) 8).putShort((short) 16);
+    buffer.putInt(6).put((byte) 6).put((byte) 255);
+    buffer.putInt(5 + encodedLength).put((byte) 7).put(payload);
+    if (trailer) {
+      buffer.put((byte) 0x42);
+    }
+    return new TrackingFile(buffer.array(), maximumRead);
+  }
+
+  private static float[] read(TrackingFile file, int points, int encodedLength) throws IOException {
+    Grib2Drs.Type42 drs = new Grib2Drs.Type42(file);
+    Grib2SectionBitMap bitmap = new Grib2SectionBitMap(file);
+    return new Grib2DataReader(42, points, points, 0, 4, HEADER_SIZE - 5, encodedLength + 5).getData(file, bitmap, drs);
+  }
+
+  private static final class TrackingFile extends InMemoryRandomAccessFile {
+    private final int maximumRead;
+    private int maximumBufferSize;
+    private int readCalls;
+
+    TrackingFile(byte[] content, int maximumRead) {
+      super("ccsds-input", content);
+      this.maximumRead = maximumRead;
+    }
+
+    @Override
+    public int read(byte[] bytes, int offset, int length) throws IOException {
+      maximumBufferSize = Math.max(maximumBufferSize, bytes.length);
+      readCalls++;
+      return super.read(bytes, offset, Math.min(length, maximumRead));
+    }
+  }
+}

grib/src/test/java/ucar/nc2/grib/grib2/TestGrib2DataReaderCcsds.java

@@ -0,0 +1,119 @@
+/*
+ * Copyright (c) 2026 University Corporation for Atmospheric Research/Unidata
+ * See LICENSE for license information.
+ */
+
+package ucar.nc2.grib.grib2;
+
+import static com.google.common.truth.Truth.assertThat;
+
+import com.google.common.io.BaseEncoding;
+import java.io.IOException;
+import java.nio.ByteBuffer;
+import java.util.Arrays;
+import java.util.Collection;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.junit.runners.Parameterized;
+import ucar.unidata.io.InMemoryRandomAccessFile;
+
+@RunWith(Parameterized.class)
+public class TestGrib2DataReaderCcsds {
+  // Unsigned, most-significant byte first, with preprocessing (GRIB2 code table 5.42).
+  private static final int COMPRESSION_OPTIONS = 4 | 8;
+  private static final int BLOCK_SIZE = 8;
+  private static final int REFERENCE_SAMPLE_INTERVAL = 16;
+  private static final int GRID_POINTS = 8;
+  private static final int NX = 4;
+
+  // Small unsigned sample streams encoded with libaec, using the settings above.
+  @Parameterized.Parameters(name = "{0} bits per sample")
+  public static Collection<Object[]> parameters() {
+    return Arrays.asList(new Object[][] {{8, "C01F8084425529F0", "C01E03844AA7C000"},
+        {16, "E0000FC040020020010028028014FFF8", "E0000F01C0020020050053FFE0000000"},
+        {32, "F000000007E02000000100000010000000800000140000014000000A7FFFFFFC",
+            "F00000000780E00000010000001000000280000029FFFFFFF000000000000000"}});
+  }
+
+  private final int bits;
+  private final byte[] fullGrid;
+  private final byte[] withBitmap;
+
+  public TestGrib2DataReaderCcsds(int bits, String fullGrid, String withBitmap) {
+    this.bits = bits;
+    this.fullGrid = BaseEncoding.base16().decode(fullGrid);
+    this.withBitmap = BaseEncoding.base16().decode(withBitmap);
+  }
+
+  @Test
+  public void decodedArrayHasOneValuePerGridPoint() throws IOException {
+    long[] values = {0, 1, 2, 3, 10, 20, 30, (1L << bits) - 1};
+    assertThat(read(values, null, 0)).isEqualTo(scaled(values));
+  }
+
+  @Test
+  public void bitmapRetainsMissingGridPoints() throws IOException {
+    long[] values = {0, 1, 2, 10, 20, (1L << bits) - 1};
+    float[] scaled = scaled(values);
+    float[] expected = {scaled[0], Float.NaN, scaled[1], scaled[2], Float.NaN, scaled[3], scaled[4], scaled[5]};
+    assertThat(read(values, new byte[] {(byte) 0b10110111}, 0)).isEqualTo(expected);
+  }
+
+  @Test
+  public void negativeXScanReversesEachRow() throws IOException {
+    long[] values = {0, 1, 2, 3, 10, 20, 30, (1L << bits) - 1};
+    float[] scaled = scaled(values);
+    float[] expected = {scaled[3], scaled[2], scaled[1], scaled[0], scaled[7], scaled[6], scaled[5], scaled[4]};
+    assertThat(read(values, null, 128)).isEqualTo(expected);
+  }
+
+  @Test
+  public void alternatingScanReversesOnlyAlternateRows() throws IOException {
+    long[] values = {0, 1, 2, 3, 10, 20, 30, (1L << bits) - 1};
+    float[] scaled = scaled(values);
+    float[] expected = {scaled[0], scaled[1], scaled[2], scaled[3], scaled[7], scaled[6], scaled[5], scaled[4]};
+    assertThat(read(values, null, 16)).isEqualTo(expected);
+  }
+
+  @Test
+  public void decodedValuesRemainIndependentAfterOtherReads() throws IOException {
+    long[] values = {0, 1, 2, 3, 10, 20, 30, (1L << bits) - 1};
+    float[] first = read(values, null, 0);
+    for (int i = 0; i < 16; i++) {
+      float[] next = read(new long[] {0, 1, 2, 10, 20, (1L << bits) - 1}, new byte[] {(byte) 0b10110111}, 0);
+      Arrays.fill(next, Float.NaN);
+    }
+    assertThat(first).isEqualTo(scaled(values));
+  }
+
+  private float[] read(long[] values, byte[] bitmap, int scanMode) throws IOException {
+    byte[] encoded = bitmap == null ? fullGrid : withBitmap;
+    int bitmapLength = 6 + (bitmap == null ? 0 : bitmap.length);
+    int dataLength = 5 + encoded.length;
+    ByteBuffer buffer = ByteBuffer.allocate(14 + bitmapLength + dataLength);
+    // Template 5.42, beginning at its reference value (octet 12).
+    buffer.putFloat(-2).putShort((short) 1).putShort((short) 1).put((byte) bits).put((byte) 0);
+    buffer.put((byte) COMPRESSION_OPTIONS).put((byte) BLOCK_SIZE).putShort((short) REFERENCE_SAMPLE_INTERVAL);
+    buffer.putInt(bitmapLength).put((byte) 6).put((byte) (bitmap == null ? 255 : 0));
+    if (bitmap != null) {
+      buffer.put(bitmap);
+    }
+    long dataStart = buffer.position();
+    buffer.putInt(dataLength).put((byte) 7).put(encoded);
+
+    try (InMemoryRandomAccessFile raf = new InMemoryRandomAccessFile("ccsds", buffer.array())) {
+      Grib2Drs.Type42 drs = new Grib2Drs.Type42(raf);
+      Grib2SectionBitMap bitmapSection = new Grib2SectionBitMap(raf);
+      Grib2DataReader reader = new Grib2DataReader(42, GRID_POINTS, values.length, scanMode, NX, dataStart, dataLength);
+      return reader.getData(raf, bitmapSection, drs);
+    }
+  }
+
+  private static float[] scaled(long[] values) {
+    float[] result = new float[values.length];
+    for (int i = 0; i < values.length; i++) {
+      result[i] = (-2 + values[i] * 2.0f) / 10.0f;
+    }
+    return result;
+  }
+}