+ * If {@code strictMode} is {@code true}, then throws an exception if fields + * have been defined on the encoded object that are not supported by the parser. This + * breaks forwards compatibility (an older parser cannot parse a newer encoded object), + * which is sometimes requires to avoid parsing an object that is newer than the code + * parsing it is prepared to handle. + *
+ * The {@code maxDepth} specifies the maximum allowed depth of nested messages. The parsing + * will fail with a ParseException if the maximum depth is reached. + *
+ * The {@code maxSize} specifies a custom value for the default `Codec.DEFAULT_MAX_SIZE` limit. IMPORTANT: + * specifying a value larger than the default one can put the application at risk because a maliciously-crafted + * payload can cause the parser to allocate too much memory which can result in OutOfMemory and/or crashes. + * It's important to carefully estimate the maximum size limit that a particular protobuf model type should support, + * and then pass that value as a parameter. Note that the estimated limit should apply to the **type** as a whole, + * rather than to individual instances of the model. In other words, this value should be a constant, or a config + * value that is controlled by the application, rather than come from the input that the application reads. + * When in doubt, use the other overloaded versions of this method that use the default `Codec.DEFAULT_MAX_SIZE`. + * + * @param input The {@link Bytes} from which to read the data to construct an object + * @param strictMode when {@code true}, the parser errors out on unknown fields; otherwise they'll be simply skipped. + * @param parseUnknownFields when {@code true} and strictMode is {@code false}, the parser will collect unknown + * fields in the unknownFields list in the model; otherwise they'll be simply skipped. + * @param maxDepth a ParseException will be thrown if the depth of nested messages exceeds the maxDepth value. + * @param maxSize a ParseException will be thrown if the size of a delimited field exceeds the limit + * @return The parsed object. It must not return null. + * @throws ParseException If parsing fails + */ + @NonNull + public final T parse( + @NonNull Bytes input, boolean strictMode, boolean parseUnknownFields, int maxDepth, int maxSize) + throws ParseException { + final PbjReader reader = new PbjReader(input); + return parse(reader, strictMode, parseUnknownFields, maxDepth, maxSize); } /** @@ -277,7 +321,7 @@ public final T parse(@NonNull ReadableSequentialData input) throws ParseExceptio */ @NonNull public final T parse(@NonNull Bytes bytes) throws ParseException { - return parse(bytes.toReadableSequentialData()); + return parse(bytes, false, false, DEFAULT_MAX_DEPTH, DEFAULT_MAX_SIZE); } /** @@ -311,7 +355,7 @@ public final T parseStrict(@NonNull ReadableSequentialData input) throws ParseEx */ @NonNull public final T parseStrict(@NonNull Bytes bytes) throws ParseException { - return parseStrict(bytes.toReadableSequentialData()); + return parse(bytes, true, DEFAULT_MAX_DEPTH); } /** @@ -489,7 +533,7 @@ public final boolean fastEquals(@NonNull T item, @NonNull ReadableSequentialData * to write to the {@link WritableStreamingData} */ public Bytes toBytes(@NonNull T item) { - // TODO: Confirm if this next line is accurate with PbjWriter + // TODO: Confirm if this next line is still true with PbjWriter // it is cheaper performance wise to measure the size of the object first than grow a buffer as needed final byte[] bytes = new byte[measureRecord(item)]; final PbjWriter writer = new PbjWriter(bytes, 0); diff --git a/pbj-core/pbj-runtime/src/main/java/com/hedera/pbj/runtime/JsonCodec.java b/pbj-core/pbj-runtime/src/main/java/com/hedera/pbj/runtime/JsonCodec.java index cab9a1bf..d79e73c2 100644 --- a/pbj-core/pbj-runtime/src/main/java/com/hedera/pbj/runtime/JsonCodec.java +++ b/pbj-core/pbj-runtime/src/main/java/com/hedera/pbj/runtime/JsonCodec.java @@ -90,6 +90,7 @@ protected final void writeImpl(@NonNull T item, @NonNull PbjWriter output) { public void write(@NonNull T item, @NonNull WritableSequentialData output) throws IOException { output.writeUTF8(toJSON(item)); } + /** * Returns JSON string representing an item. * diff --git a/pbj-core/pbj-runtime/src/main/java/com/hedera/pbj/runtime/io/buffer/Bytes.java b/pbj-core/pbj-runtime/src/main/java/com/hedera/pbj/runtime/io/buffer/Bytes.java index 843b9ebf..20859a9f 100644 --- a/pbj-core/pbj-runtime/src/main/java/com/hedera/pbj/runtime/io/buffer/Bytes.java +++ b/pbj-core/pbj-runtime/src/main/java/com/hedera/pbj/runtime/io/buffer/Bytes.java @@ -494,6 +494,16 @@ public ReadableSequentialData toReadableSequentialData() { return new RandomAccessSequenceAdapter(this); } + /** + * Create and return a new {@link PbjReader} that is backed by this {@link Bytes}. + * + * @return A {@link PbjReader} backed by this {@link Bytes}. + */ + @NonNull + public PbjReader toPbjReader() { + return new PbjReader(this); + } + @NonNull public void resetPbjReader(@NonNull PbjReader reader) { reader.resetWith(buffer, start, start + length); diff --git a/pbj-core/pbj-runtime/src/test/java/com/hedera/pbj/runtime/ProtoParserToolsTest.java b/pbj-core/pbj-runtime/src/test/java/com/hedera/pbj/runtime/ProtoParserToolsTest.java index 8d25fc83..f9e63087 100644 --- a/pbj-core/pbj-runtime/src/test/java/com/hedera/pbj/runtime/ProtoParserToolsTest.java +++ b/pbj-core/pbj-runtime/src/test/java/com/hedera/pbj/runtime/ProtoParserToolsTest.java @@ -29,17 +29,14 @@ import static org.junit.jupiter.api.Assertions.assertNull; import static org.junit.jupiter.api.Assertions.assertThrows; -import com.hedera.pbj.runtime.io.ReadableSequentialData; import com.hedera.pbj.runtime.io.buffer.BufferedData; import com.hedera.pbj.runtime.io.buffer.Bytes; import com.hedera.pbj.runtime.io.buffer.PbjReader; import com.hedera.pbj.runtime.io.buffer.PbjWriter; import com.hedera.pbj.runtime.io.stream.ReadableStreamingData; -import com.hedera.pbj.runtime.io.stream.WritableStreamingData; import com.hedera.pbj.runtime.test.UncheckedThrowingFunction; import edu.umd.cs.findbugs.annotations.NonNull; import java.io.ByteArrayInputStream; -import java.io.ByteArrayOutputStream; import java.io.IOException; import java.nio.BufferUnderflowException; import java.nio.ByteOrder; @@ -359,20 +356,18 @@ void testSkipUnsupported(ProtoConstants unsupportedType) { @Test void testExtractBytesNullInput() { final FieldDefinition field = createFieldDefinition(BYTES); - assertThrows( - NullPointerException.class, - () -> ProtoParserTools.extractFieldBytes((ReadableSequentialData) null, field)); + assertThrows(NullPointerException.class, () -> ProtoParserTools.extractFieldBytes((PbjReader) null, field)); } @Test void testExtractBytesNullField() { - final ReadableSequentialData input = Bytes.EMPTY.toReadableSequentialData(); + final PbjReader input = Bytes.EMPTY.toPbjReader(); assertThrows(NullPointerException.class, () -> ProtoParserTools.extractFieldBytes(input, null)); } @Test void testExtractBytesRepeatedField() { - final ReadableSequentialData input = Bytes.EMPTY.toReadableSequentialData(); + final PbjReader input = Bytes.EMPTY.toPbjReader(); final FieldDefinition field = new FieldDefinition("field", FieldType.BYTES, true, true, false, 1); assertThrows(IllegalArgumentException.class, () -> ProtoParserTools.extractFieldBytes(input, field)); } @@ -407,22 +402,20 @@ void testExtractBytesRepeatedField() { private static final FieldDefinition BOOL_F = new FieldDefinition("boolfield", BOOL, false, true, false, 11); private static final boolean BOOL_V = true; - private static Bytes prepareExtractBytesTestInput() throws IOException { - try (final ByteArrayOutputStream bout = new ByteArrayOutputStream(); - final WritableStreamingData out = new WritableStreamingData(bout)) { - ProtoWriterTools.writeInteger(out, INT32_F, INT32_V); - ProtoWriterTools.writeInteger(out, FIXED_F, FIXED32_V); - ProtoWriterTools.writeString(out, STRING_F, STRING_V); - ProtoWriterTools.writeBytes(out, BYTES_F, BYTES_V); - ProtoWriterTools.writeMessage(out, MESSAGE_F, MESSAGE_V, TestMessageCodec.INSTANCE); - ProtoWriterTools.writeDouble(out, DOUBLE_F, DOUBLE32_V); - return Bytes.wrap(bout.toByteArray()); - } + private static PbjReader prepareExtractBytesTestInput() throws IOException { + PbjWriter out = new PbjWriter(); + ProtoWriterTools.writeInteger(out, INT32_F, INT32_V); + ProtoWriterTools.writeInteger(out, FIXED_F, FIXED32_V); + ProtoWriterTools.writeString(out, STRING_F, STRING_V); + ProtoWriterTools.writeBytes(out, BYTES_F, BYTES_V); + ProtoWriterTools.writeMessage(out, MESSAGE_F, MESSAGE_V, TestMessageCodec.INSTANCE); + ProtoWriterTools.writeDouble(out, DOUBLE_F, DOUBLE32_V); + return out.toPbjReader(); } @Test void testExtractBytesStringField() throws IOException, ParseException { - final ReadableSequentialData input = prepareExtractBytesTestInput().toReadableSequentialData(); + final PbjReader input = prepareExtractBytesTestInput(); final Bytes bytes = ProtoParserTools.extractFieldBytes(input, STRING_F); assertNotNull(bytes); assertEquals(STRING_V, new String(bytes.toByteArray(), StandardCharsets.UTF_8)); @@ -430,14 +423,14 @@ void testExtractBytesStringField() throws IOException, ParseException { @Test void testExtractFieldBytesInvalidType() throws IOException, ParseException { - final ReadableSequentialData input = prepareExtractBytesTestInput().toReadableSequentialData(); + final PbjReader input = prepareExtractBytesTestInput(); // should throw because INT32 is not a delimited type assertThrows(IllegalArgumentException.class, () -> ProtoParserTools.extractFieldBytes(input, INT32_F)); } @Test void testExtractBytesBytesField() throws IOException, ParseException { - final ReadableSequentialData input = prepareExtractBytesTestInput().toReadableSequentialData(); + final PbjReader input = prepareExtractBytesTestInput(); final Bytes bytes = ProtoParserTools.extractFieldBytes(input, BYTES_F); assertNotNull(bytes); assertEquals(BYTES_V, bytes); @@ -445,52 +438,58 @@ void testExtractBytesBytesField() throws IOException, ParseException { @Test void testExtractBytesMessageField() throws IOException, ParseException { - final ReadableSequentialData input = prepareExtractBytesTestInput().toReadableSequentialData(); + final PbjReader input = prepareExtractBytesTestInput(); final Bytes bytes = ProtoParserTools.extractFieldBytes(input, MESSAGE_F); assertNotNull(bytes); - final TestMessage value = TestMessageCodec.INSTANCE.parse(bytes.toReadableSequentialData()); + final TestMessage value = TestMessageCodec.INSTANCE.parse(bytes); assertNotNull(value); assertEquals(MESSAGE_V, value); } @Test void testExtractBytesUnknownField() throws IOException, ParseException { - final ReadableSequentialData input = prepareExtractBytesTestInput().toReadableSequentialData(); + final PbjReader input = prepareExtractBytesTestInput(); final Bytes bytes = ProtoParserTools.extractFieldBytes(input, UNKNOWN_F); assertNull(bytes); } @Test void testExtractField32Bit() throws IOException, ParseException { - final ReadableSequentialData input = prepareExtractBytesTestInput().toReadableSequentialData(); + final PbjReader input = prepareExtractBytesTestInput(); final var res = ProtoParserTools.extractField(input, WIRE_TYPE_FIXED_32_BIT, 32); assertNotNull(res); } @Test void testExtractField64Bit() throws IOException, ParseException { - final ReadableSequentialData input = prepareExtractBytesTestInput().toReadableSequentialData(); + final PbjReader input = prepareExtractBytesTestInput(); final var res = ProtoParserTools.extractField(input, WIRE_TYPE_FIXED_64_BIT, 32); assertNotNull(res); } @Test void testExtractFieldVarInt() throws IOException, ParseException { - final ReadableSequentialData input = prepareExtractBytesTestInput().toReadableSequentialData(); + final PbjReader input = prepareExtractBytesTestInput(); final var res = ProtoParserTools.extractField(input, WIRE_TYPE_VARINT_OR_ZIGZAG, 32); assertNotNull(res); } @Test void testExtractFieldGroupStartUnsupported() throws IOException { - final ReadableSequentialData input = prepareExtractBytesTestInput().toReadableSequentialData(); - assertThrows(IOException.class, () -> ProtoParserTools.extractField(input, WIRE_TYPE_GROUP_START, 32)); + final PbjReader input = prepareExtractBytesTestInput(); + assertThrows(IOException.class, () -> { + ProtoParserTools.extractField(input, WIRE_TYPE_GROUP_START, 32); + input.throwOnError2(); + }); } @Test void testExtractFieldGroupEndUnsupported() throws IOException { - final ReadableSequentialData input = prepareExtractBytesTestInput().toReadableSequentialData(); - assertThrows(IOException.class, () -> ProtoParserTools.extractField(input, WIRE_TYPE_GROUP_END, 32)); + final PbjReader input = prepareExtractBytesTestInput(); + assertThrows(IOException.class, () -> { + ProtoParserTools.extractField(input, WIRE_TYPE_GROUP_END, 32); + input.throwOnError2(); + }); } private static void skipTag(BufferedData data) { diff --git a/pbj-core/pbj-runtime/src/test/java/com/hedera/pbj/runtime/io/buffer/PbjReaderWriterTest.java b/pbj-core/pbj-runtime/src/test/java/com/hedera/pbj/runtime/io/buffer/PbjReaderWriterTest.java index 8d36ec5d..877e17b0 100644 --- a/pbj-core/pbj-runtime/src/test/java/com/hedera/pbj/runtime/io/buffer/PbjReaderWriterTest.java +++ b/pbj-core/pbj-runtime/src/test/java/com/hedera/pbj/runtime/io/buffer/PbjReaderWriterTest.java @@ -929,17 +929,6 @@ void writeBytesArrayZeroAndNegativeLengthWritesNothing() { assertEquals(0, writer.position()); } - @Test - void quickTest() { - byte[] a = {2, 3}; - var by = Bytes.wrap(a); - var adap = by.toReadableSequentialData(); - var r = new PbjReader(adap); - var w = new PbjWriter(); - w.setError(4, ""); - int q = 0; - } - @Test void largeWriteBypassCorrectness() { ByteArrayOutputStream out = new ByteArrayOutputStream(); diff --git a/pbj-integration-tests/src/test/java/com/hedera/pbj/integration/test/MaxSizeTest.java b/pbj-integration-tests/src/test/java/com/hedera/pbj/integration/test/MaxSizeTest.java index 5ca4f5ed..90b302c3 100644 --- a/pbj-integration-tests/src/test/java/com/hedera/pbj/integration/test/MaxSizeTest.java +++ b/pbj-integration-tests/src/test/java/com/hedera/pbj/integration/test/MaxSizeTest.java @@ -88,32 +88,22 @@ void testNestedMaxSize() throws Exception { final Bytes bytes = Everything.PROTOBUF.toBytes(everything); // Try negative cases first: - assertThrows(ParseException.class, () -> Everything.PROTOBUF.parse(bytes.toReadableSequentialData())); + assertThrows(ParseException.class, () -> Everything.PROTOBUF.parse(bytes)); assertThrows( ParseException.class, - () -> Everything.PROTOBUF.parse( - bytes.toReadableSequentialData(), false, false, Codec.DEFAULT_MAX_DEPTH, 256)); + () -> Everything.PROTOBUF.parse(bytes, false, false, Codec.DEFAULT_MAX_DEPTH, 256)); assertThrows( ParseException.class, - () -> Everything.PROTOBUF.parse( - bytes.toReadableSequentialData(), - false, - false, - Codec.DEFAULT_MAX_DEPTH, - Codec.DEFAULT_MAX_SIZE)); + () -> Everything.PROTOBUF.parse(bytes, false, false, Codec.DEFAULT_MAX_DEPTH, Codec.DEFAULT_MAX_SIZE)); // +1 still shouldn't work because the outer and the inner objects are still larger: assertThrows( ParseException.class, () -> Everything.PROTOBUF.parse( - bytes.toReadableSequentialData(), - false, - false, - Codec.DEFAULT_MAX_DEPTH, - Codec.DEFAULT_MAX_SIZE + 1)); + bytes, false, false, Codec.DEFAULT_MAX_DEPTH, Codec.DEFAULT_MAX_SIZE + 1)); // Now try supplying a large enough maxSize to parse it: - final Everything parsedEverything = Everything.PROTOBUF.parse( - bytes.toReadableSequentialData(), false, false, Codec.DEFAULT_MAX_DEPTH, Codec.DEFAULT_MAX_SIZE * 2); + final Everything parsedEverything = + Everything.PROTOBUF.parse(bytes, false, false, Codec.DEFAULT_MAX_DEPTH, Codec.DEFAULT_MAX_SIZE * 2); assertEquals(everything, parsedEverything); } diff --git a/pbj-integration-tests/src/test/java/com/hedera/pbj/integration/test/UnknownFieldsTest.java b/pbj-integration-tests/src/test/java/com/hedera/pbj/integration/test/UnknownFieldsTest.java index c631cabc..0ccf9121 100644 --- a/pbj-integration-tests/src/test/java/com/hedera/pbj/integration/test/UnknownFieldsTest.java +++ b/pbj-integration-tests/src/test/java/com/hedera/pbj/integration/test/UnknownFieldsTest.java @@ -6,6 +6,7 @@ import static org.junit.jupiter.api.Assertions.assertTrue; import com.hedera.pbj.integration.EverythingTestData; +import com.hedera.pbj.runtime.Codec; import com.hedera.pbj.runtime.OneOf; import com.hedera.pbj.runtime.ProtoConstants; import com.hedera.pbj.runtime.UnknownField; @@ -58,8 +59,7 @@ void testUnknownFieldsRoundTrip() throws Exception { final Bytes bytes2 = MessageWithBytes.PROTOBUF.toBytes(msg2); // now read it as MessageWithBytesAndString, w/o even enabling unknown fields - final MessageWithBytesAndString msg3 = - MessageWithBytesAndString.PROTOBUF.parse(bytes2.toReadableSequentialData()); + final MessageWithBytesAndString msg3 = MessageWithBytesAndString.PROTOBUF.parse(bytes2.toPbjReader()); assertEquals(msg1, msg3); } @@ -77,8 +77,8 @@ void testEverythingRoundTrip() throws Exception { assertEquals(0, msgWithUnknownFieldsDisabled.getUnknownFields().size()); // Now let's enable parsing unknown fields: - final MessageWithEverythingUnknownFields msg = MessageWithEverythingUnknownFields.PROTOBUF.parse( - everythingBytes.toReadableSequentialData(), false, true, 16); + final MessageWithEverythingUnknownFields msg = + MessageWithEverythingUnknownFields.PROTOBUF.parse(everythingBytes.toPbjReader(), false, true, 16); assertEquals(0, msg.knownField()); assertEquals(65, msg.getUnknownFields().size()); @@ -108,7 +108,7 @@ public void testUnknownFieldsInInnerMessage() throws Exception { // then read it as MessageWithBytes with unknown fields final MessageWithBytes messageWithBytes = MessageWithBytes.PROTOBUF.parse( - messageWithBytesAndStringBytes.toReadableSequentialData(), false, true, 16); + messageWithBytesAndStringBytes, false, true, 16, Codec.DEFAULT_MAX_SIZE); final MessageWithBytesWrapper messageWithBytesWrapper = new MessageWithBytesWrapper( new OneOf<>(MessageWithBytesWrapper.MessageValidOneOfType.MESSAGE_WITH_BYTES, messageWithBytes)); @@ -122,7 +122,7 @@ public void testUnknownFieldsInInnerMessage() throws Exception { // parse bytes back as a receiving user would and confirm unknown fields exist in inner message final MessageWithBytesWrapper parsedWrapper = MessageWithBytesWrapper.PROTOBUF.parse( - messageWithBytesWrapperBytes.toReadableSequentialData(), false, true, 16); + messageWithBytesWrapperBytes, false, true, 16, Codec.DEFAULT_MAX_SIZE); MessageWithBytes parsedBytes = parsedWrapper.messageWithBytes(); assertFalse(parsedBytes.getUnknownFields().isEmpty()); assertEquals(1, parsedBytes.getUnknownFields().size()); @@ -130,7 +130,7 @@ public void testUnknownFieldsInInnerMessage() throws Exception { // now confirm that user can retrieve unknown fields when using expanded message MessageWithBytesAndString final Bytes messageWithBytesBytes = MessageWithBytes.PROTOBUF.toBytes(parsedBytes); final MessageWithBytesAndString messageWithBytesAndStringParsed = - MessageWithBytesAndString.PROTOBUF.parse(messageWithBytesBytes.toReadableSequentialData()); + MessageWithBytesAndString.PROTOBUF.parse(messageWithBytesBytes); assertTrue(messageWithBytesAndStringParsed.getUnknownFields().isEmpty()); assertEquals(messageWithBytesAndString, messageWithBytesAndStringParsed); }