From fb3a5f6bef69b4c9518e36fcc612ab5b81f0dfcd Mon Sep 17 00:00:00 2001 From: jihuayu Date: Tue, 8 Sep 2026 16:03:08 +0800 Subject: [PATCH 01/20] feat(java): expose input stream reader execution options --- bindings/java/src/lib.rs | 63 +++++++++++++++++-- .../java/org/apache/opendal/Operator.java | 19 +++++- .../apache/opendal/OperatorInputStream.java | 18 +++++- .../org/apache/opendal/ReaderOptions.java | 62 ++++++++++++++++++ bindings/java/src/operator_input_stream.rs | 14 +++-- 5 files changed, 162 insertions(+), 14 deletions(-) create mode 100644 bindings/java/src/main/java/org/apache/opendal/ReaderOptions.java diff --git a/bindings/java/src/lib.rs b/bindings/java/src/lib.rs index e95057856796..5e0a22f2dc53 100644 --- a/bindings/java/src/lib.rs +++ b/bindings/java/src/lib.rs @@ -292,9 +292,64 @@ fn make_read_options<'a>( }) } -fn make_reader_options<'a>( - _: &mut Env<'a>, - _: &JObject, +fn make_reader_options( + env: &mut Env, + options: &JObject, ) -> Result { - Ok(opendal::options::ReaderOptions::default()) + Ok(build_reader_options( + convert::read_int_field(env, options, "concurrent")?, + convert::read_int64_field(env, options, "chunk")?, + convert::read_int_field(env, options, "prefetch")?, + convert::read_int64_field(env, options, "contentLengthHint")?, + )?) +} + +fn build_reader_options( + concurrent: i32, + chunk: i64, + prefetch: i32, + content_length_hint: i64, +) -> opendal::Result { + use opendal::{Error, ErrorKind}; + + if concurrent <= 0 { + return Err(Error::new( + ErrorKind::ConfigInvalid, + "concurrent must be positive", + )); + } + let concurrent = usize::try_from(concurrent) + .map_err(|_| Error::new(ErrorKind::ConfigInvalid, "concurrent is too large"))?; + let chunk = match chunk { + -1 => None, + value if value > 0 => Some( + usize::try_from(value) + .map_err(|_| Error::new(ErrorKind::ConfigInvalid, "chunk is too large"))?, + ), + _ => { + return Err(Error::new( + ErrorKind::ConfigInvalid, + "chunk must be -1 or positive", + )); + } + }; + let prefetch = usize::try_from(prefetch) + .map_err(|_| Error::new(ErrorKind::ConfigInvalid, "prefetch must be non-negative"))?; + let content_length_hint = match content_length_hint { + -1 => None, + value => Some(u64::try_from(value).map_err(|_| { + Error::new( + ErrorKind::ConfigInvalid, + "contentLengthHint must be -1 or non-negative", + ) + })?), + }; + + Ok(opendal::options::ReaderOptions { + concurrent, + chunk, + prefetch, + content_length_hint, + ..Default::default() + }) } diff --git a/bindings/java/src/main/java/org/apache/opendal/Operator.java b/bindings/java/src/main/java/org/apache/opendal/Operator.java index 8e48d1831949..123edd516254 100644 --- a/bindings/java/src/main/java/org/apache/opendal/Operator.java +++ b/bindings/java/src/main/java/org/apache/opendal/Operator.java @@ -115,11 +115,26 @@ public byte[] read(String path, ReadOptions options) { } public OperatorInputStream createInputStream(String path) { - return new OperatorInputStream(this, path, ReadOptions.builder().build()); + return createInputStream( + path, ReadOptions.builder().build(), ReaderOptions.builder().build()); } public OperatorInputStream createInputStream(String path, ReadOptions options) { - return new OperatorInputStream(this, path, options); + return createInputStream(path, options, ReaderOptions.builder().build()); + } + + /** + * Creates a stream over the requested range using the supplied reader execution options. + * The stream ends at the range boundary. Closing it releases its native reader. + * + * @param path object path + * @param readOptions logical offset and length + * @param readerOptions internal chunk request and buffering controls + * @return a stream that the caller must close + * @throws OpenDALException if reader options are invalid (ConfigInvalid) or creation fails + */ + public OperatorInputStream createInputStream(String path, ReadOptions readOptions, ReaderOptions readerOptions) { + return new OperatorInputStream(this, path, readOptions, readerOptions); } public void delete(String path) { diff --git a/bindings/java/src/main/java/org/apache/opendal/OperatorInputStream.java b/bindings/java/src/main/java/org/apache/opendal/OperatorInputStream.java index a236db7dfae7..89077d66608e 100644 --- a/bindings/java/src/main/java/org/apache/opendal/OperatorInputStream.java +++ b/bindings/java/src/main/java/org/apache/opendal/OperatorInputStream.java @@ -40,8 +40,21 @@ protected void disposeInternal(long handle) { private byte[] bytes = new byte[0]; public OperatorInputStream(Operator operator, String path, ReadOptions options) { + this(operator, path, options, ReaderOptions.builder().build()); + } + + /** + * Creates a stream with a logical range and independent execution controls. + * + * @param operator operator that reads the object + * @param path object path + * @param readOptions logical offset and length + * @param readerOptions internal chunk request and buffering controls + * @throws OpenDALException if reader options are invalid (ConfigInvalid) or creation fails + */ + public OperatorInputStream(Operator operator, String path, ReadOptions readOptions, ReaderOptions readerOptions) { final long op = operator.nativeHandle; - this.reader = new Reader(constructReader(op, path, options)); + this.reader = new Reader(constructReader(op, path, readOptions, readerOptions)); } @Override @@ -99,7 +112,8 @@ public void close() { reader.close(); } - private static native long constructReader(long op, String path, ReadOptions options); + private static native long constructReader( + long op, String path, ReadOptions readOptions, ReaderOptions readerOptions); private static native void disposeReader(long reader); diff --git a/bindings/java/src/main/java/org/apache/opendal/ReaderOptions.java b/bindings/java/src/main/java/org/apache/opendal/ReaderOptions.java new file mode 100644 index 000000000000..120bda530603 --- /dev/null +++ b/bindings/java/src/main/java/org/apache/opendal/ReaderOptions.java @@ -0,0 +1,62 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.opendal; + +import lombok.Builder; + +/** + * Controls how an input stream executes reads, independently of its logical range. + * Setting concurrent without setting chunk does not enable concurrent range reads. + * Invalid values cause an OpenDALException with code ConfigInvalid when the stream is created. + */ +@Builder +public final class ReaderOptions { + /** + * Maximum number of internal chunk requests executed concurrently. Must be positive. + * This is not the number of application transfers or Java threads and only affects chunked reads. + */ + @Builder.Default + public final int concurrent = 1; + + /** + * Target size of each internal range request, in bytes. A positive value enables chunked reads. + * The default of -1 keeps unchunked streaming. Zero and other negative values are invalid. + * The value must fit the native platform's unsigned pointer-sized integer. + */ + @Builder.Default + public final long chunk = -1L; + + /** + * Maximum number of completed chunks buffered ahead of consumption, not a byte count. + * Must be non-negative. The default of zero applies strict backpressure. + * This option only affects chunked reads. + */ + @Builder.Default + public final int prefetch = 0; + + /** + * Known full object content length in bytes, independent of the requested range. + * The default of -1 means unknown; zero is valid for an empty object. + * This hint can avoid a metadata request and is not a consistency condition. + * An incorrect hint can cause incomplete reads or errors. Values below -1 are invalid. + */ + @Builder.Default + public final long contentLengthHint = -1L; +} diff --git a/bindings/java/src/operator_input_stream.rs b/bindings/java/src/operator_input_stream.rs index 57a7299791db..d8e712af7cba 100644 --- a/bindings/java/src/operator_input_stream.rs +++ b/bindings/java/src/operator_input_stream.rs @@ -39,11 +39,12 @@ pub unsafe extern "system" fn Java_org_apache_opendal_OperatorInputStream_constr _: JClass<'local>, op: *mut blocking::Operator, path: JString<'local>, - options: JObject<'local>, + read_options: JObject<'local>, + reader_options: JObject<'local>, ) -> jlong { env.with_env(|env| { let op_ref = unsafe { &mut *op }; - intern_construct_reader(env, op_ref, path, options) + intern_construct_reader(env, op_ref, path, read_options, reader_options) }) .resolve::() } @@ -52,16 +53,17 @@ fn intern_construct_reader( env: &mut Env, op: &mut blocking::Operator, path: JString, - options: JObject, + read_options: JObject, + reader_options: JObject, ) -> crate::Result { use crate::convert; use crate::make_reader_options; let path = jstring_to_string(env, &path)?; - let reader_options = make_reader_options(env, &options)?; + let reader_options = make_reader_options(env, &reader_options)?; - let offset = convert::read_int64_field(env, &options, "offset")?; - let length = convert::read_int64_field(env, &options, "length")?; + let offset = convert::read_int64_field(env, &read_options, "offset")?; + let length = convert::read_int64_field(env, &read_options, "length")?; let range = convert::offset_length_to_range(offset, length)?; let reader = op From 0ec482aa4c05fc4367869ec5e53dbbfa1896dce8 Mon Sep 17 00:00:00 2001 From: jihuayu Date: Tue, 8 Sep 2026 16:05:38 +0800 Subject: [PATCH 02/20] test(java): verify reader options mapping and stream behavior --- bindings/java/README.md | 45 ++++++++ bindings/java/src/lib.rs | 56 ++++++++++ .../test/OperatorInputOutputStreamTest.java | 104 ++++++++++++++++++ .../test/behavior/BlockingWriteTest.java | 27 +++++ 4 files changed, 232 insertions(+) diff --git a/bindings/java/README.md b/bindings/java/README.md index 22437ba5fc60..2a8c3a9b19e4 100644 --- a/bindings/java/README.md +++ b/bindings/java/README.md @@ -92,6 +92,51 @@ public class Main { Use the synchronous `Operator` for blocking calls, or `AsyncOperator` for `CompletableFuture`-based calls. +## Tune input stream reads + +`ReadOptions` selects the logical byte range. `ReaderOptions` controls how the +stream executes reads. Existing `createInputStream(path)` and +`createInputStream(path, readOptions)` calls keep unchunked streaming defaults. + +```java +ReaderOptions readerOptions = ReaderOptions.builder() + .concurrent(4) + .chunk(8 * 1024 * 1024L) + .prefetch(2) + .build(); + +try (OperatorInputStream in = op.createInputStream( + "large.bin", ReadOptions.builder().build(), readerOptions)) { + byte[] buffer = new byte[8192]; + int count; + while ((count = in.read(buffer)) != -1) { + // Process buffer[0..count). + } +} +``` + +- `concurrent` limits internal chunk requests, not application transfers or Java + threads. Its default is `1`, and it must be positive. +- `chunk` sets the target range request size in bytes. Its default is `-1` + (unchunked streaming); a positive value enables chunked reads and must fit the + native platform's unsigned pointer-sized integer. Setting `concurrent` without + setting `chunk` does not enable concurrent range reads. +- `prefetch` limits completed chunks buffered ahead of consumption, not bytes. + Its default is `0` for strict backpressure, and it must be non-negative. +- `contentLengthHint` supplies the full object length in bytes, even when reading + a subrange. Its default is `-1` (unknown); `0` is valid for an empty object. + This hint can avoid a metadata request. It does not enforce consistency, and an + incorrect hint can cause incomplete reads or errors. + +Invalid values produce an `OpenDALException` with code `ConfigInvalid` when the +stream is created. Payload memory usage generally grows with chunk size, +concurrency, and prefetching; SDK, JNI, and Java buffers add further overhead. +These options do not imply a fixed memory formula or a throughput guarantee. + +The core `gap` option only affects multi-range `Reader::fetch` and is not +applicable to this continuous stream API. Version and conditional read options +are outside this API's current scope. + ## Documentation The full user guide — getting started, connecting to services, common tasks, and diff --git a/bindings/java/src/lib.rs b/bindings/java/src/lib.rs index 5e0a22f2dc53..acff3cf7a01d 100644 --- a/bindings/java/src/lib.rs +++ b/bindings/java/src/lib.rs @@ -353,3 +353,59 @@ fn build_reader_options( ..Default::default() }) } + +#[cfg(test)] +mod reader_options_tests { + use super::build_reader_options; + use opendal::ErrorKind; + + #[test] + fn default_reader_options() { + let options = build_reader_options(1, -1, 0, -1).unwrap(); + assert_eq!(options.concurrent, 1); + assert_eq!(options.chunk, None); + assert_eq!(options.prefetch, 0); + assert_eq!(options.content_length_hint, None); + } + + #[test] + fn tuned_reader_options() { + let options = build_reader_options(4, 8 * 1024 * 1024, 2, 128 * 1024 * 1024).unwrap(); + assert_eq!(options.concurrent, 4); + assert_eq!(options.chunk, Some(8 * 1024 * 1024)); + assert_eq!(options.prefetch, 2); + assert_eq!(options.content_length_hint, Some(128 * 1024 * 1024)); + } + + #[test] + fn empty_content_length_hint() { + let options = build_reader_options(1, 1, 0, 0).unwrap(); + assert_eq!(options.content_length_hint, Some(0)); + } + + #[test] + fn invalid_reader_options() { + for (concurrent, chunk, prefetch, hint, field) in [ + (0, -1, 0, -1, "concurrent"), + (-1, -1, 0, -1, "concurrent"), + (1, 0, 0, -1, "chunk"), + (1, -2, 0, -1, "chunk"), + (1, -1, -1, -1, "prefetch"), + (1, -1, 0, -2, "contentLengthHint"), + ] { + let err = build_reader_options(concurrent, chunk, prefetch, hint).unwrap_err(); + assert_eq!(err.kind(), ErrorKind::ConfigInvalid); + assert!(err.message().contains(field)); + } + } + + #[test] + fn chunk_conversion_respects_native_width() { + let options = build_reader_options(1, i64::MAX, 0, -1); + if usize::BITS < 64 { + assert_eq!(options.unwrap_err().kind(), ErrorKind::ConfigInvalid); + } else { + assert_eq!(options.unwrap().chunk, usize::try_from(i64::MAX).ok()); + } + } +} diff --git a/bindings/java/src/test/java/org/apache/opendal/test/OperatorInputOutputStreamTest.java b/bindings/java/src/test/java/org/apache/opendal/test/OperatorInputOutputStreamTest.java index 23717996323a..adffafed8ef3 100644 --- a/bindings/java/src/test/java/org/apache/opendal/test/OperatorInputOutputStreamTest.java +++ b/bindings/java/src/test/java/org/apache/opendal/test/OperatorInputOutputStreamTest.java @@ -20,19 +20,30 @@ package org.apache.opendal.test; import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; import java.io.BufferedReader; import java.io.InputStreamReader; +import java.nio.charset.StandardCharsets; import java.nio.file.Path; +import java.util.Random; import java.util.concurrent.atomic.AtomicLong; import java.util.stream.Stream; +import org.apache.commons.io.IOUtils; +import org.apache.opendal.OpenDALException; import org.apache.opendal.Operator; import org.apache.opendal.OperatorInputStream; import org.apache.opendal.OperatorOutputStream; import org.apache.opendal.ReadOptions; +import org.apache.opendal.ReaderOptions; import org.apache.opendal.ServiceConfig; import org.apache.opendal.WriteOptions; +import org.apache.opendal.test.condition.OpenDALExceptionCondition; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.io.TempDir; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.Arguments; +import org.junit.jupiter.params.provider.MethodSource; +import org.junit.jupiter.params.provider.ValueSource; public class OperatorInputOutputStreamTest { @TempDir @@ -86,6 +97,99 @@ void testCreateInputStreamWithOptions() { } } + @ParameterizedTest + @ValueSource(booleans = {false, true}) + void testChunkedInputStream(boolean knownLength) throws Exception { + final byte[] content = new byte[4 * 1024 * 1024 + 13]; + new Random(8252).nextBytes(content); + final ServiceConfig.Fs fs = + ServiceConfig.Fs.builder().root(tempDir.toString()).build(); + try (final Operator op = Operator.of(fs)) { + final String path = "chunked.bin"; + op.write(path, content); + final ReaderOptions options = ReaderOptions.builder() + .concurrent(4) + .chunk(64 * 1024L) + .prefetch(2) + .contentLengthHint(knownLength ? content.length : -1L) + .build(); + try (final OperatorInputStream in = + op.createInputStream(path, ReadOptions.builder().build(), options)) { + assertThat(IOUtils.toByteArray(in)).isEqualTo(content); + assertThat(in.read()).isEqualTo(-1); + } + } + } + + @Test + void testRangeWithReaderOptions() throws Exception { + final ServiceConfig.Fs fs = + ServiceConfig.Fs.builder().root(tempDir.toString()).build(); + try (final Operator op = Operator.of(fs)) { + final String path = "chunked-range.txt"; + op.write(path, "0123456789"); + final ReadOptions range = ReadOptions.builder().offset(4).length(5).build(); + final ReaderOptions options = ReaderOptions.builder() + .concurrent(2) + .chunk(2) + .prefetch(1) + .contentLengthHint(10) + .build(); + try (final OperatorInputStream in = op.createInputStream(path, range, options)) { + assertThat(IOUtils.toByteArray(in)).isEqualTo("45678".getBytes(StandardCharsets.UTF_8)); + assertThat(in.read()).isEqualTo(-1); + } + try (final OperatorInputStream in = new OperatorInputStream(op, path, range)) { + assertThat(IOUtils.toByteArray(in)).isEqualTo("45678".getBytes(StandardCharsets.UTF_8)); + assertThat(in.read()).isEqualTo(-1); + } + } + } + + @ParameterizedTest + @ValueSource(longs = {-1L, 0L}) + void testEmptyInputStreamWithReaderOptions(long hint) throws Exception { + final ServiceConfig.Fs fs = + ServiceConfig.Fs.builder().root(tempDir.toString()).build(); + try (final Operator op = Operator.of(fs)) { + final String path = "empty.bin"; + op.write(path, new byte[0]); + final ReaderOptions options = + ReaderOptions.builder().chunk(2).contentLengthHint(hint).build(); + try (final OperatorInputStream in = + op.createInputStream(path, ReadOptions.builder().build(), options)) { + assertThat(in.read()).isEqualTo(-1); + } + } + } + + static Stream invalidReaderOptions() { + return Stream.of( + Arguments.of(ReaderOptions.builder().concurrent(0).build(), "concurrent"), + Arguments.of(ReaderOptions.builder().concurrent(-1).build(), "concurrent"), + Arguments.of(ReaderOptions.builder().chunk(0).build(), "chunk"), + Arguments.of(ReaderOptions.builder().chunk(-2).build(), "chunk"), + Arguments.of(ReaderOptions.builder().prefetch(-1).build(), "prefetch"), + Arguments.of(ReaderOptions.builder().contentLengthHint(-2).build(), "contentLengthHint")); + } + + @ParameterizedTest + @MethodSource("invalidReaderOptions") + void testInvalidReaderOptions(ReaderOptions options, String field) { + final ServiceConfig.Fs fs = + ServiceConfig.Fs.builder().root(tempDir.toString()).build(); + try (final Operator op = Operator.of(fs)) { + assertThatThrownBy(() -> { + try (final OperatorInputStream in = op.createInputStream( + "invalid-options", ReadOptions.builder().build(), options)) { + in.read(); + } + }) + .is(OpenDALExceptionCondition.ofSync(OpenDALException.Code.ConfigInvalid)) + .hasMessageContaining(field); + } + } + @Test void testCreateOutputStreamWithOptions() { final ServiceConfig.Fs fs = diff --git a/bindings/java/src/test/java/org/apache/opendal/test/behavior/BlockingWriteTest.java b/bindings/java/src/test/java/org/apache/opendal/test/behavior/BlockingWriteTest.java index ea18d288ac05..7b2296e0c84c 100644 --- a/bindings/java/src/test/java/org/apache/opendal/test/behavior/BlockingWriteTest.java +++ b/bindings/java/src/test/java/org/apache/opendal/test/behavior/BlockingWriteTest.java @@ -22,10 +22,15 @@ import static org.assertj.core.api.Assertions.assertThat; import static org.assertj.core.api.Assertions.assertThatThrownBy; import static org.junit.jupiter.api.Assumptions.assumeTrue; +import java.util.Random; import java.util.UUID; +import org.apache.commons.io.IOUtils; import org.apache.opendal.Capability; import org.apache.opendal.Metadata; import org.apache.opendal.OpenDALException; +import org.apache.opendal.OperatorInputStream; +import org.apache.opendal.ReadOptions; +import org.apache.opendal.ReaderOptions; import org.apache.opendal.test.condition.OpenDALExceptionCondition; import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.Test; @@ -62,6 +67,28 @@ public void testBlockingReadFull() { op().delete(path); } + @Test + public void testBlockingInputStreamWithReaderOptions() throws Exception { + final String path = UUID.randomUUID().toString(); + final byte[] content = new byte[1024 * 1024 + 13]; + new Random(8252).nextBytes(content); + op().write(path, content); + try { + final ReaderOptions options = ReaderOptions.builder() + .concurrent(4) + .chunk(256 * 1024L) + .prefetch(2) + .build(); + try (final OperatorInputStream in = + op().createInputStream(path, ReadOptions.builder().build(), options)) { + assertThat(IOUtils.toByteArray(in)).isEqualTo(content); + assertThat(in.read()).isEqualTo(-1); + } + } finally { + op().delete(path); + } + } + /** * Stat existing file should return metadata. */ From 04218e4782e11119c6a376fa31eaa7fe95e2152d Mon Sep 17 00:00:00 2001 From: jihuayu Date: Tue, 8 Sep 2026 16:25:31 +0800 Subject: [PATCH 03/20] test(java): trim redundant reader options coverage and docs --- bindings/java/README.md | 22 ++++++------------- bindings/java/src/lib.rs | 16 -------------- .../test/OperatorInputOutputStreamTest.java | 18 +++++---------- 3 files changed, 12 insertions(+), 44 deletions(-) diff --git a/bindings/java/README.md b/bindings/java/README.md index 2a8c3a9b19e4..5a727e8c955c 100644 --- a/bindings/java/README.md +++ b/bindings/java/README.md @@ -115,21 +115,13 @@ try (OperatorInputStream in = op.createInputStream( } ``` -- `concurrent` limits internal chunk requests, not application transfers or Java - threads. Its default is `1`, and it must be positive. -- `chunk` sets the target range request size in bytes. Its default is `-1` - (unchunked streaming); a positive value enables chunked reads and must fit the - native platform's unsigned pointer-sized integer. Setting `concurrent` without - setting `chunk` does not enable concurrent range reads. -- `prefetch` limits completed chunks buffered ahead of consumption, not bytes. - Its default is `0` for strict backpressure, and it must be non-negative. -- `contentLengthHint` supplies the full object length in bytes, even when reading - a subrange. Its default is `-1` (unknown); `0` is valid for an empty object. - This hint can avoid a metadata request. It does not enforce consistency, and an - incorrect hint can cause incomplete reads or errors. - -Invalid values produce an `OpenDALException` with code `ConfigInvalid` when the -stream is created. Payload memory usage generally grows with chunk size, +A positive `chunk` enables internal range requests. `concurrent` limits these +requests, not application transfers or Java threads; `prefetch` counts completed +chunks, not bytes. Setting `concurrent` alone keeps unchunked streaming. +See [ReaderOptions](src/main/java/org/apache/opendal/ReaderOptions.java) for +all defaults, valid values, and content length hint semantics. + +Payload memory usage generally grows with chunk size, concurrency, and prefetching; SDK, JNI, and Java buffers add further overhead. These options do not imply a fixed memory formula or a throughput guarantee. diff --git a/bindings/java/src/lib.rs b/bindings/java/src/lib.rs index acff3cf7a01d..178a9f3f794a 100644 --- a/bindings/java/src/lib.rs +++ b/bindings/java/src/lib.rs @@ -383,22 +383,6 @@ mod reader_options_tests { assert_eq!(options.content_length_hint, Some(0)); } - #[test] - fn invalid_reader_options() { - for (concurrent, chunk, prefetch, hint, field) in [ - (0, -1, 0, -1, "concurrent"), - (-1, -1, 0, -1, "concurrent"), - (1, 0, 0, -1, "chunk"), - (1, -2, 0, -1, "chunk"), - (1, -1, -1, -1, "prefetch"), - (1, -1, 0, -2, "contentLengthHint"), - ] { - let err = build_reader_options(concurrent, chunk, prefetch, hint).unwrap_err(); - assert_eq!(err.kind(), ErrorKind::ConfigInvalid); - assert!(err.message().contains(field)); - } - } - #[test] fn chunk_conversion_respects_native_width() { let options = build_reader_options(1, i64::MAX, 0, -1); diff --git a/bindings/java/src/test/java/org/apache/opendal/test/OperatorInputOutputStreamTest.java b/bindings/java/src/test/java/org/apache/opendal/test/OperatorInputOutputStreamTest.java index adffafed8ef3..c70479f20184 100644 --- a/bindings/java/src/test/java/org/apache/opendal/test/OperatorInputOutputStreamTest.java +++ b/bindings/java/src/test/java/org/apache/opendal/test/OperatorInputOutputStreamTest.java @@ -43,7 +43,6 @@ import org.junit.jupiter.params.ParameterizedTest; import org.junit.jupiter.params.provider.Arguments; import org.junit.jupiter.params.provider.MethodSource; -import org.junit.jupiter.params.provider.ValueSource; public class OperatorInputOutputStreamTest { @TempDir @@ -97,9 +96,8 @@ void testCreateInputStreamWithOptions() { } } - @ParameterizedTest - @ValueSource(booleans = {false, true}) - void testChunkedInputStream(boolean knownLength) throws Exception { + @Test + void testChunkedInputStream() throws Exception { final byte[] content = new byte[4 * 1024 * 1024 + 13]; new Random(8252).nextBytes(content); final ServiceConfig.Fs fs = @@ -111,7 +109,6 @@ void testChunkedInputStream(boolean knownLength) throws Exception { .concurrent(4) .chunk(64 * 1024L) .prefetch(2) - .contentLengthHint(knownLength ? content.length : -1L) .build(); try (final OperatorInputStream in = op.createInputStream(path, ReadOptions.builder().build(), options)) { @@ -139,23 +136,18 @@ void testRangeWithReaderOptions() throws Exception { assertThat(IOUtils.toByteArray(in)).isEqualTo("45678".getBytes(StandardCharsets.UTF_8)); assertThat(in.read()).isEqualTo(-1); } - try (final OperatorInputStream in = new OperatorInputStream(op, path, range)) { - assertThat(IOUtils.toByteArray(in)).isEqualTo("45678".getBytes(StandardCharsets.UTF_8)); - assertThat(in.read()).isEqualTo(-1); - } } } - @ParameterizedTest - @ValueSource(longs = {-1L, 0L}) - void testEmptyInputStreamWithReaderOptions(long hint) throws Exception { + @Test + void testEmptyInputStreamWithReaderOptions() throws Exception { final ServiceConfig.Fs fs = ServiceConfig.Fs.builder().root(tempDir.toString()).build(); try (final Operator op = Operator.of(fs)) { final String path = "empty.bin"; op.write(path, new byte[0]); final ReaderOptions options = - ReaderOptions.builder().chunk(2).contentLengthHint(hint).build(); + ReaderOptions.builder().chunk(2).contentLengthHint(0).build(); try (final OperatorInputStream in = op.createInputStream(path, ReadOptions.builder().build(), options)) { assertThat(in.read()).isEqualTo(-1); From 0026df1cfec26d55846bf6f38217a584d9aa04b2 Mon Sep 17 00:00:00 2001 From: jihuayu Date: Mon, 14 Sep 2026 16:26:47 +0800 Subject: [PATCH 04/20] feat(java): expose reusable OperatorReader for range reads --- bindings/java/src/lib.rs | 1 + .../java/org/apache/opendal/Operator.java | 35 +++++++ .../org/apache/opendal/OperatorReader.java | 95 ++++++++++++++++++ bindings/java/src/operator_reader.rs | 78 +++++++++++++++ .../opendal/test/OperatorReaderTest.java | 98 +++++++++++++++++++ 5 files changed, 307 insertions(+) create mode 100644 bindings/java/src/main/java/org/apache/opendal/OperatorReader.java create mode 100644 bindings/java/src/operator_reader.rs create mode 100644 bindings/java/src/test/java/org/apache/opendal/test/OperatorReaderTest.java diff --git a/bindings/java/src/lib.rs b/bindings/java/src/lib.rs index 178a9f3f794a..8b170c8fce03 100644 --- a/bindings/java/src/lib.rs +++ b/bindings/java/src/lib.rs @@ -39,6 +39,7 @@ mod layer; mod operator; mod operator_input_stream; mod operator_output_stream; +mod operator_reader; mod utility; pub(crate) type Result = std::result::Result; diff --git a/bindings/java/src/main/java/org/apache/opendal/Operator.java b/bindings/java/src/main/java/org/apache/opendal/Operator.java index 123edd516254..749c8f80734b 100644 --- a/bindings/java/src/main/java/org/apache/opendal/Operator.java +++ b/bindings/java/src/main/java/org/apache/opendal/Operator.java @@ -23,6 +23,7 @@ import java.util.Arrays; import java.util.List; import java.util.Map; +import java.util.Objects; /** * Operator represents an underneath OpenDAL operator that accesses data @@ -114,6 +115,40 @@ public byte[] read(String path, ReadOptions options) { return read(nativeHandle, path, options); } + /** + * Creates a reusable reader for a file with default execution options. + * + * @param path file path + * @return a reader that the caller must close + * @see #reader(String, ReaderOptions) + */ + public OperatorReader reader(String path) { + return reader(path, ReaderOptions.builder().build()); + } + + /** + * Creates a reusable reader for a file. Options apply to every read through the reader; + * each read selects its own byte range. The reader can outlive this operator. + * Creation does not read file contents or guarantee that the file exists. + * + * @param path file path + * @param options reader execution options + * @return a reader that the caller must close + * @throws OpenDALException if options are invalid (ConfigInvalid), the path is a directory + * (IsADirectory), or the service does not support reads (Unsupported) + * @throws IllegalStateException if this operator is closed + */ + public OperatorReader reader(String path, ReaderOptions options) { + if (isDisposed()) { + throw new IllegalStateException("Operator is closed"); + } + Objects.requireNonNull(path, "path"); + Objects.requireNonNull(options, "options"); + return new OperatorReader(reader(nativeHandle, path, options)); + } + + private static native long reader(long operator, String path, ReaderOptions options); + public OperatorInputStream createInputStream(String path) { return createInputStream( path, ReadOptions.builder().build(), ReaderOptions.builder().build()); diff --git a/bindings/java/src/main/java/org/apache/opendal/OperatorReader.java b/bindings/java/src/main/java/org/apache/opendal/OperatorReader.java new file mode 100644 index 000000000000..e7f26d425659 --- /dev/null +++ b/bindings/java/src/main/java/org/apache/opendal/OperatorReader.java @@ -0,0 +1,95 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.opendal; + +import java.util.Objects; + +/** + * Reads a file synchronously through a reusable Rust core reader. + * Each read selects its own range and does not advance a shared cursor. + * Reader options apply to every read. A reader does not snapshot the file; + * changes to the file may be visible to subsequent reads. + * + *

Close the reader when it is no longer needed, preferably with try-with-resources. + * Closing the operator that created it does not close the reader. + * Calls on one reader are serialized, including close. + * + * @see Operator#reader(String, ReaderOptions) + */ +public final class OperatorReader extends NativeObject { + OperatorReader(long nativeHandle) { + super(nativeHandle); + } + + /** + * Reads the whole file into a Java byte array. + * + * @return file contents + * @throws OpenDALException if the file does not exist (NotFound) or reading fails + * @throws IllegalStateException if this reader is closed + */ + public byte[] read() { + return read(0, -1); + } + + /** + * Reads a byte range into a Java byte array without advancing a shared cursor. + * + * @param offset non-negative starting byte offset + * @param length number of bytes to read, or -1 to read to the end; zero returns an empty array + * @return contents of the requested range + * @throws OpenDALException if the range is invalid (RangeNotSatisfied), the file does not + * exist (NotFound), or reading fails + * @throws IllegalStateException if this reader is closed + */ + public synchronized byte[] read(long offset, long length) { + if (isDisposed()) { + throw new IllegalStateException("OperatorReader is closed"); + } + return readBytes(nativeHandle, offset, length); + } + + /** + * Reads the range selected by the supplied options. + * + * @param options logical offset and length + * @return contents of the requested range + * @see #read(long, long) + */ + public byte[] read(ReadOptions options) { + Objects.requireNonNull(options, "options"); + return read(options.offset, options.length); + } + + /** Releases this reader's native resources. Repeated calls have no effect. */ + @Override + public synchronized void close() { + super.close(); + } + + @Override + protected void disposeInternal(long handle) { + disposeReader(handle); + } + + private static native byte[] readBytes(long reader, long offset, long length); + + private static native void disposeReader(long reader); +} diff --git a/bindings/java/src/operator_reader.rs b/bindings/java/src/operator_reader.rs new file mode 100644 index 000000000000..9ac2a60b8e6e --- /dev/null +++ b/bindings/java/src/operator_reader.rs @@ -0,0 +1,78 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +use jni::EnvUnowned; +use jni::objects::{JByteArray, JClass, JObject, JString}; +use jni::sys::jlong; +use opendal::blocking; + +use crate::convert; +use crate::error::ThrowException; + +/// # Safety +/// +/// `op` must point to a live blocking operator for the duration of this call. +#[unsafe(no_mangle)] +pub unsafe extern "system" fn Java_org_apache_opendal_Operator_reader<'local>( + mut env: EnvUnowned<'local>, + _: JClass<'local>, + op: *const blocking::Operator, + path: JString<'local>, + options: JObject<'local>, +) -> jlong { + env.with_env(|env| -> crate::Result<_> { + let op = unsafe { &*op }; + let path = convert::jstring_to_string(env, &path)?; + let options = crate::make_reader_options(env, &options)?; + let reader = op.reader_options(&path, options)?; + Ok(Box::into_raw(Box::new(reader)) as jlong) + }) + .resolve::() +} + +/// # Safety +/// +/// `reader` must point to a live blocking reader for the duration of this call. +#[unsafe(no_mangle)] +pub unsafe extern "system" fn Java_org_apache_opendal_OperatorReader_readBytes<'local>( + mut env: EnvUnowned<'local>, + _: JClass<'local>, + reader: *const blocking::Reader, + offset: jlong, + length: jlong, +) -> JByteArray<'local> { + env.with_env(|env| -> crate::Result<_> { + let reader = unsafe { &*reader }; + let range = convert::offset_length_to_range(offset, length)?; + let content = reader.read(range)?; + convert::bytes_to_jbytearray(env, content.to_vec()) + }) + .resolve::() +} + +/// # Safety +/// +/// `reader` must be a live handle allocated by `Operator.reader`, with no calls in progress. +/// It must not be used after this call. +#[unsafe(no_mangle)] +pub unsafe extern "system" fn Java_org_apache_opendal_OperatorReader_disposeReader<'local>( + _: EnvUnowned<'local>, + _: JClass<'local>, + reader: *mut blocking::Reader, +) { + unsafe { drop(Box::from_raw(reader)) }; +} diff --git a/bindings/java/src/test/java/org/apache/opendal/test/OperatorReaderTest.java b/bindings/java/src/test/java/org/apache/opendal/test/OperatorReaderTest.java new file mode 100644 index 000000000000..ea144d43d90e --- /dev/null +++ b/bindings/java/src/test/java/org/apache/opendal/test/OperatorReaderTest.java @@ -0,0 +1,98 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.opendal.test; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; +import java.nio.charset.StandardCharsets; +import java.nio.file.Path; +import org.apache.opendal.OpenDALException; +import org.apache.opendal.Operator; +import org.apache.opendal.OperatorReader; +import org.apache.opendal.ReadOptions; +import org.apache.opendal.ReaderOptions; +import org.apache.opendal.ServiceConfig; +import org.apache.opendal.test.condition.OpenDALExceptionCondition; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.CsvSource; + +public class OperatorReaderTest { + @TempDir + private Path tempDir; + + @Test + void testReusableReaderOutlivesOperator() { + final OperatorReader reader; + try (Operator op = + Operator.of(ServiceConfig.Fs.builder().root(tempDir.toString()).build())) { + op.write("file", "0123456789"); + reader = op.reader( + "file", + ReaderOptions.builder() + .concurrent(2) + .chunk(2) + .prefetch(1) + .contentLengthHint(10) + .build()); + } + try (OperatorReader r = reader) { + assertThat(r.read(4, 5)).isEqualTo("45678".getBytes(StandardCharsets.UTF_8)); + assertThat(r.read(ReadOptions.builder().offset(1).length(2).build())) + .isEqualTo("12".getBytes(StandardCharsets.UTF_8)); + assertThat(r.read(8, -1)).isEqualTo("89".getBytes(StandardCharsets.UTF_8)); + assertThat(r.read(0, 0)).isEmpty(); + assertThat(r.read()).isEqualTo("0123456789".getBytes(StandardCharsets.UTF_8)); + } + reader.close(); + assertThatThrownBy(reader::read).isInstanceOf(IllegalStateException.class); + } + + @ParameterizedTest + @CsvSource({"-1, 1", "0, -2"}) + void testInvalidRange(long offset, long length) { + try (Operator op = Operator.of( + ServiceConfig.Fs.builder().root(tempDir.toString()).build()); + OperatorReader reader = op.reader("missing")) { + assertThatThrownBy(() -> reader.read(offset, length)) + .is(OpenDALExceptionCondition.ofSync(OpenDALException.Code.RangeNotSatisfied)); + } + } + + @Test + void testMissingFileFailsOnRead() { + try (Operator op = Operator.of( + ServiceConfig.Fs.builder().root(tempDir.toString()).build()); + OperatorReader reader = op.reader("missing")) { + assertThatThrownBy(reader::read).is(OpenDALExceptionCondition.ofSync(OpenDALException.Code.NotFound)); + } + } + + @Test + void testInvalidReaderOptions() { + try (Operator op = + Operator.of(ServiceConfig.Fs.builder().root(tempDir.toString()).build())) { + assertThatThrownBy(() -> op.reader( + "missing", ReaderOptions.builder().chunk(0).build())) + .is(OpenDALExceptionCondition.ofSync(OpenDALException.Code.ConfigInvalid)); + } + } +} From a4415dcd33353f7b9240b0113ada29f8dce72737 Mon Sep 17 00:00:00 2001 From: jihuayu Date: Mon, 14 Sep 2026 16:31:07 +0800 Subject: [PATCH 05/20] refactor(java): create input streams through OperatorReader --- bindings/java/README.md | 21 +++++--- .../java/org/apache/opendal/Operator.java | 2 + .../apache/opendal/OperatorInputStream.java | 51 +++++++++++------- .../org/apache/opendal/OperatorReader.java | 47 ++++++++++++++++ .../org/apache/opendal/ReaderOptions.java | 4 +- bindings/java/src/operator_input_stream.rs | 53 +------------------ bindings/java/src/operator_reader.rs | 20 +++++++ .../opendal/test/OperatorReaderTest.java | 47 ++++++++++++++++ website/docs/20-bindings/java/04-tasks.md | 29 ++++++++++ 9 files changed, 196 insertions(+), 78 deletions(-) diff --git a/bindings/java/README.md b/bindings/java/README.md index 5a727e8c955c..5f2aaff8f982 100644 --- a/bindings/java/README.md +++ b/bindings/java/README.md @@ -92,11 +92,13 @@ public class Main { Use the synchronous `Operator` for blocking calls, or `AsyncOperator` for `CompletableFuture`-based calls. -## Tune input stream reads +## Reuse a reader and tune reads -`ReadOptions` selects the logical byte range. `ReaderOptions` controls how the -stream executes reads. Existing `createInputStream(path)` and -`createInputStream(path, readOptions)` calls keep unchunked streaming defaults. +`Operator.reader(path, readerOptions)` creates an `OperatorReader` backed by a +Rust core reader. `ReaderOptions` controls how it executes reads; each call to +`read` or `createInputStream` selects its own logical byte range. `ReadOptions` +can also supply that range. Existing `Operator.createInputStream` overloads use +the same abstraction and preserve their defaults. ```java ReaderOptions readerOptions = ReaderOptions.builder() @@ -105,8 +107,8 @@ ReaderOptions readerOptions = ReaderOptions.builder() .prefetch(2) .build(); -try (OperatorInputStream in = op.createInputStream( - "large.bin", ReadOptions.builder().build(), readerOptions)) { +try (OperatorReader reader = op.reader("large.bin", readerOptions); + OperatorInputStream in = reader.createInputStream()) { byte[] buffer = new byte[8192]; int count; while ((count = in.read(buffer)) != -1) { @@ -115,6 +117,13 @@ try (OperatorInputStream in = op.createInputStream( } ``` +Use `reader.read(offset, length)` to collect a range into a byte array, or +`reader.createInputStream(offset, length)` to stream it. A length of `-1` reads +to the end; zero selects an empty range. Repeated reads do not share a cursor. +Close each reader and stream separately. Streams remain usable after their +reader closes, and readers remain usable after their operator closes. A reader +does not snapshot the file, so later reads may observe changes. + A positive `chunk` enables internal range requests. `concurrent` limits these requests, not application transfers or Java threads; `prefetch` counts completed chunks, not bytes. Setting `concurrent` alone keeps unchunked streaming. diff --git a/bindings/java/src/main/java/org/apache/opendal/Operator.java b/bindings/java/src/main/java/org/apache/opendal/Operator.java index 749c8f80734b..e8e65353cace 100644 --- a/bindings/java/src/main/java/org/apache/opendal/Operator.java +++ b/bindings/java/src/main/java/org/apache/opendal/Operator.java @@ -167,6 +167,8 @@ public OperatorInputStream createInputStream(String path, ReadOptions options) { * @param readerOptions internal chunk request and buffering controls * @return a stream that the caller must close * @throws OpenDALException if reader options are invalid (ConfigInvalid) or creation fails + * @see #reader(String, ReaderOptions) + * @see OperatorReader#createInputStream(ReadOptions) */ public OperatorInputStream createInputStream(String path, ReadOptions readOptions, ReaderOptions readerOptions) { return new OperatorInputStream(this, path, readOptions, readerOptions); diff --git a/bindings/java/src/main/java/org/apache/opendal/OperatorInputStream.java b/bindings/java/src/main/java/org/apache/opendal/OperatorInputStream.java index 89077d66608e..8fad0d2e2c52 100644 --- a/bindings/java/src/main/java/org/apache/opendal/OperatorInputStream.java +++ b/bindings/java/src/main/java/org/apache/opendal/OperatorInputStream.java @@ -22,9 +22,14 @@ import java.io.InputStream; import java.util.Objects; +/** + * Reads a byte range sequentially through an {@link OperatorReader}. + * Each stream owns its native iterator and must be closed independently of its source reader. + * Reading a closed stream throws {@link IllegalStateException}. + */ public class OperatorInputStream extends InputStream { - private static class Reader extends NativeObject { - private Reader(long nativeHandle) { + private static class BytesIterator extends NativeObject { + private BytesIterator(long nativeHandle) { super(nativeHandle); } @@ -34,11 +39,15 @@ protected void disposeInternal(long handle) { } } - private final Reader reader; + private final BytesIterator reader; private int offset = 0; private byte[] bytes = new byte[0]; + OperatorInputStream(long nativeHandle) { + this.reader = new BytesIterator(nativeHandle); + } + public OperatorInputStream(Operator operator, String path, ReadOptions options) { this(operator, path, options, ReaderOptions.builder().build()); } @@ -53,13 +62,18 @@ public OperatorInputStream(Operator operator, String path, ReadOptions options) * @throws OpenDALException if reader options are invalid (ConfigInvalid) or creation fails */ public OperatorInputStream(Operator operator, String path, ReadOptions readOptions, ReaderOptions readerOptions) { - final long op = operator.nativeHandle; - this.reader = new Reader(constructReader(op, path, readOptions, readerOptions)); + Objects.requireNonNull(readOptions, "readOptions"); + try (OperatorReader source = operator.reader(path, readerOptions)) { + this.reader = new BytesIterator(source.createBytesIterator(readOptions.offset, readOptions.length)); + } } @Override - public int read() { - if (bytes != null && offset >= bytes.length) { + public synchronized int read() { + if (reader.isDisposed()) { + throw new IllegalStateException("OperatorInputStream is closed"); + } + while (bytes != null && offset >= bytes.length) { bytes = readNextBytes(reader.nativeHandle); offset = 0; } @@ -72,17 +86,23 @@ public int read() { } @Override - public int read(byte[] b, int off, int len) { + public synchronized int read(byte[] b, int off, int len) { Objects.requireNonNull(b); if ((b.length | off | len) < 0 || len > b.length - off) { // Objects.checkFromIndexSize has only been available since Java 9 throw new IndexOutOfBoundsException( String.format("Range [%s, % 0) { - if (bytes != null && offset >= bytes.length) { + while (bytes != null && offset >= bytes.length) { bytes = readNextBytes(reader.nativeHandle); offset = 0; } @@ -99,22 +119,15 @@ public int read(byte[] b, int off, int len) { len -= n; } - if (bytes != null && offset >= bytes.length) { - bytes = readNextBytes(reader.nativeHandle); - offset = 0; - } - - return bytes != null ? read : (read != 0 ? read : -1); + return read; } @Override - public void close() { + public synchronized void close() { reader.close(); + bytes = null; } - private static native long constructReader( - long op, String path, ReadOptions readOptions, ReaderOptions readerOptions); - private static native void disposeReader(long reader); private static native byte[] readNextBytes(long reader); diff --git a/bindings/java/src/main/java/org/apache/opendal/OperatorReader.java b/bindings/java/src/main/java/org/apache/opendal/OperatorReader.java index e7f26d425659..f5b12f663d1a 100644 --- a/bindings/java/src/main/java/org/apache/opendal/OperatorReader.java +++ b/bindings/java/src/main/java/org/apache/opendal/OperatorReader.java @@ -78,6 +78,51 @@ public byte[] read(ReadOptions options) { return read(options.offset, options.length); } + /** + * Creates an independent stream over the whole file. + * + * @return a stream that the caller must close + * @see #createInputStream(long, long) + */ + public OperatorInputStream createInputStream() { + return createInputStream(0, -1); + } + + /** + * Creates a stream over a byte range using this reader's options. + * Each stream has its own cursor and native resources. Closing this reader does not + * close its streams, and closing a stream does not close this reader or its other streams. + * + * @param offset non-negative starting byte offset + * @param length number of bytes to read, or -1 to read to the end; zero creates an empty stream + * @return a stream that the caller must close + * @throws OpenDALException if the range is invalid (RangeNotSatisfied) or opening the stream fails + * @throws IllegalStateException if this reader is closed + */ + public OperatorInputStream createInputStream(long offset, long length) { + return new OperatorInputStream(createBytesIterator(offset, length)); + } + + /** + * Creates a stream over the range selected by the supplied options. + * + * @param options logical offset and length + * @return a stream that the caller must close + * @see #createInputStream(long, long) + */ + public OperatorInputStream createInputStream(ReadOptions options) { + Objects.requireNonNull(options, "options"); + return createInputStream(options.offset, options.length); + } + + // The caller owns the returned iterator independently of this reader. + synchronized long createBytesIterator(long offset, long length) { + if (isDisposed()) { + throw new IllegalStateException("OperatorReader is closed"); + } + return createBytesIterator(nativeHandle, offset, length); + } + /** Releases this reader's native resources. Repeated calls have no effect. */ @Override public synchronized void close() { @@ -91,5 +136,7 @@ protected void disposeInternal(long handle) { private static native byte[] readBytes(long reader, long offset, long length); + private static native long createBytesIterator(long reader, long offset, long length); + private static native void disposeReader(long reader); } diff --git a/bindings/java/src/main/java/org/apache/opendal/ReaderOptions.java b/bindings/java/src/main/java/org/apache/opendal/ReaderOptions.java index 120bda530603..416abc26e04b 100644 --- a/bindings/java/src/main/java/org/apache/opendal/ReaderOptions.java +++ b/bindings/java/src/main/java/org/apache/opendal/ReaderOptions.java @@ -22,9 +22,9 @@ import lombok.Builder; /** - * Controls how an input stream executes reads, independently of its logical range. + * Controls how an {@link OperatorReader} and its streams execute reads, independently of each logical range. * Setting concurrent without setting chunk does not enable concurrent range reads. - * Invalid values cause an OpenDALException with code ConfigInvalid when the stream is created. + * Invalid values cause an OpenDALException with code ConfigInvalid when the reader is created. */ @Builder public final class ReaderOptions { diff --git a/bindings/java/src/operator_input_stream.rs b/bindings/java/src/operator_input_stream.rs index d8e712af7cba..cf22253d5834 100644 --- a/bindings/java/src/operator_input_stream.rs +++ b/bindings/java/src/operator_input_stream.rs @@ -19,62 +19,13 @@ use jni::Env; use jni::EnvUnowned; use jni::objects::JByteArray; use jni::objects::JClass; -use jni::objects::JObject; -use jni::objects::JString; -use jni::sys::jlong; -use opendal::blocking; use opendal::blocking::StdBytesIterator; -use crate::convert::jstring_to_string; use crate::error::ThrowException; /// # Safety /// -/// This function should not be called before the Operator is ready. -#[unsafe(no_mangle)] -pub unsafe extern "system" fn Java_org_apache_opendal_OperatorInputStream_constructReader< - 'local, ->( - mut env: EnvUnowned<'local>, - _: JClass<'local>, - op: *mut blocking::Operator, - path: JString<'local>, - read_options: JObject<'local>, - reader_options: JObject<'local>, -) -> jlong { - env.with_env(|env| { - let op_ref = unsafe { &mut *op }; - intern_construct_reader(env, op_ref, path, read_options, reader_options) - }) - .resolve::() -} - -fn intern_construct_reader( - env: &mut Env, - op: &mut blocking::Operator, - path: JString, - read_options: JObject, - reader_options: JObject, -) -> crate::Result { - use crate::convert; - use crate::make_reader_options; - - let path = jstring_to_string(env, &path)?; - let reader_options = make_reader_options(env, &reader_options)?; - - let offset = convert::read_int64_field(env, &read_options, "offset")?; - let length = convert::read_int64_field(env, &read_options, "length")?; - let range = convert::offset_length_to_range(offset, length)?; - - let reader = op - .reader_options(&path, reader_options)? - .into_bytes_iterator(range)?; - Ok(Box::into_raw(Box::new(reader)) as jlong) -} - -/// # Safety -/// -/// This function should not be called before the Operator is ready. +/// `reader` must point to a live iterator, with no other calls in progress. #[unsafe(no_mangle)] pub unsafe extern "system" fn Java_org_apache_opendal_OperatorInputStream_disposeReader<'local>( _: EnvUnowned<'local>, @@ -88,7 +39,7 @@ pub unsafe extern "system" fn Java_org_apache_opendal_OperatorInputStream_dispos /// # Safety /// -/// This function should not be called before the Operator is ready. +/// `reader` must point to a live iterator, with no other calls in progress. #[unsafe(no_mangle)] pub unsafe extern "system" fn Java_org_apache_opendal_OperatorInputStream_readNextBytes<'local>( mut env: EnvUnowned<'local>, diff --git a/bindings/java/src/operator_reader.rs b/bindings/java/src/operator_reader.rs index 9ac2a60b8e6e..5d9b1a59b1b7 100644 --- a/bindings/java/src/operator_reader.rs +++ b/bindings/java/src/operator_reader.rs @@ -64,6 +64,26 @@ pub unsafe extern "system" fn Java_org_apache_opendal_OperatorReader_readBytes<' .resolve::() } +/// # Safety +/// +/// `reader` must point to a live blocking reader for the duration of this call. +#[unsafe(no_mangle)] +pub unsafe extern "system" fn Java_org_apache_opendal_OperatorReader_createBytesIterator<'local>( + mut env: EnvUnowned<'local>, + _: JClass<'local>, + reader: *const blocking::Reader, + offset: jlong, + length: jlong, +) -> jlong { + env.with_env(|_| -> crate::Result<_> { + let reader = unsafe { &*reader }; + let range = convert::offset_length_to_range(offset, length)?; + let iter = reader.clone().into_bytes_iterator(range)?; + Ok(Box::into_raw(Box::new(iter)) as jlong) + }) + .resolve::() +} + /// # Safety /// /// `reader` must be a live handle allocated by `Operator.reader`, with no calls in progress. diff --git a/bindings/java/src/test/java/org/apache/opendal/test/OperatorReaderTest.java b/bindings/java/src/test/java/org/apache/opendal/test/OperatorReaderTest.java index ea144d43d90e..a01eba071491 100644 --- a/bindings/java/src/test/java/org/apache/opendal/test/OperatorReaderTest.java +++ b/bindings/java/src/test/java/org/apache/opendal/test/OperatorReaderTest.java @@ -23,8 +23,10 @@ import static org.assertj.core.api.Assertions.assertThatThrownBy; import java.nio.charset.StandardCharsets; import java.nio.file.Path; +import org.apache.commons.io.IOUtils; import org.apache.opendal.OpenDALException; import org.apache.opendal.Operator; +import org.apache.opendal.OperatorInputStream; import org.apache.opendal.OperatorReader; import org.apache.opendal.ReadOptions; import org.apache.opendal.ReaderOptions; @@ -74,6 +76,8 @@ void testInvalidRange(long offset, long length) { OperatorReader reader = op.reader("missing")) { assertThatThrownBy(() -> reader.read(offset, length)) .is(OpenDALExceptionCondition.ofSync(OpenDALException.Code.RangeNotSatisfied)); + assertThatThrownBy(() -> reader.createInputStream(offset, length)) + .is(OpenDALExceptionCondition.ofSync(OpenDALException.Code.RangeNotSatisfied)); } } @@ -86,6 +90,49 @@ void testMissingFileFailsOnRead() { } } + @Test + void testIndependentStreams() throws Exception { + try (Operator op = + Operator.of(ServiceConfig.Fs.builder().root(tempDir.toString()).build())) { + op.write("file", "0123456789"); + try (OperatorReader reader = + op.reader("file", ReaderOptions.builder().chunk(2).build())) { + try (OperatorInputStream in = reader.createInputStream( + ReadOptions.builder().offset(4).length(3).build())) { + assertThat(IOUtils.toByteArray(in)).isEqualTo("456".getBytes(StandardCharsets.UTF_8)); + } + assertThat(reader.read(0, 2)).isEqualTo("01".getBytes(StandardCharsets.UTF_8)); + try (OperatorInputStream first = reader.createInputStream(); + OperatorInputStream second = reader.createInputStream(4, 5)) { + reader.close(); + op.close(); + assertThatThrownBy(reader::createInputStream).isInstanceOf(IllegalStateException.class); + assertThat(first.read()).isEqualTo('0'); + assertThat(second.read()).isEqualTo('4'); + first.close(); + first.close(); + assertThatThrownBy(first::read).isInstanceOf(IllegalStateException.class); + assertThatThrownBy(() -> first.read(new byte[2], 0, 2)).isInstanceOf(IllegalStateException.class); + assertThat(IOUtils.toByteArray(second)).isEqualTo("5678".getBytes(StandardCharsets.UTF_8)); + assertThat(second.read()).isEqualTo(-1); + } + } + } + } + + @Test + void testZeroLengthStreamReads() { + try (Operator op = Operator.of( + ServiceConfig.Fs.builder().root(tempDir.toString()).build()); + OperatorReader reader = op.reader("missing"); + OperatorInputStream in = reader.createInputStream(0, 0)) { + byte[] bytes = new byte[1]; + assertThat(in.read(bytes, 0, 0)).isZero(); + assertThat(in.read()).isEqualTo(-1); + assertThat(in.read(bytes, 0, 0)).isZero(); + } + } + @Test void testInvalidReaderOptions() { try (Operator op = diff --git a/website/docs/20-bindings/java/04-tasks.md b/website/docs/20-bindings/java/04-tasks.md index 9a31802e0e1c..bfc3bfe263f5 100644 --- a/website/docs/20-bindings/java/04-tasks.md +++ b/website/docs/20-bindings/java/04-tasks.md @@ -34,6 +34,29 @@ String text = new String(data); byte[] data = op.read("path/to/file", 0, 1024); ``` +For repeated reads from the same file, create an `OperatorReader`. It uses a +Rust core reader and lets each call select an independent byte range: + +```java +import org.apache.opendal.OperatorReader; +import org.apache.opendal.ReaderOptions; + +ReaderOptions options = ReaderOptions.builder() + .chunk(8 * 1024 * 1024L) + .concurrent(4) + .prefetch(2) + .build(); +try (OperatorReader reader = op.reader("path/to/file", options)) { + byte[] first = reader.read(0, 1024); + byte[] next = reader.read(1024, 1024); +} +``` + +A positive `chunk` enables internal range requests; `concurrent` limits those +requests and `prefetch` counts completed chunks buffered ahead. Setting +`concurrent` alone keeps unchunked streaming. A reader does not snapshot the +file, so subsequent reads may observe changes. + ## Stream a large file Don't load gigabytes into memory — read through an `InputStream` in chunks: @@ -50,6 +73,12 @@ try (InputStream in = op.createInputStream("big.bin")) { } ``` +For a configured reader, use `reader.createInputStream()` for the whole file +or `reader.createInputStream(offset, length)` for a range. A length of `-1` +reads to the end; zero selects an empty range. Each stream has its own cursor. +Close each reader and stream separately: closing a reader does not close its +streams, and closing the operator does not close its readers. + ## Write a whole file ```java From 7e502777437de4219f81e2a9123231f008d0bfbc Mon Sep 17 00:00:00 2001 From: jihuayu Date: Mon, 14 Sep 2026 16:44:43 +0800 Subject: [PATCH 06/20] feat(core): expose multi-range fetch on blocking readers --- core/core/src/blocking/read/reader.rs | 51 +++++++++++++++++++++++++++ 1 file changed, 51 insertions(+) diff --git a/core/core/src/blocking/read/reader.rs b/core/core/src/blocking/read/reader.rs index 38d2240edc2d..97b983865c91 100644 --- a/core/core/src/blocking/read/reader.rs +++ b/core/core/src/blocking/read/reader.rs @@ -15,6 +15,8 @@ // specific language governing permissions and limitations // under the License. +use std::ops::Range; + use bytes::BufMut; use super::BufferIterator; @@ -59,6 +61,55 @@ impl Reader { self.handle.block_on(inner.read(range)) } + /// Fetch multiple byte ranges into buffers in the same order as the input. + /// + /// Overlapping and adjacent ranges are merged. The reader's + /// [`options::ReaderOptions::gap`] also allows merging ranges separated by + /// at most that many bytes; `0` disables merging across gaps. The reader + /// splits merged requests using `chunk` and executes them with `concurrent` + /// and `prefetch`. Returned buffers contain only the requested bytes and + /// may share their underlying storage. + /// + /// Empty or reversed ranges return empty buffers. An empty input returns + /// an empty vector without reading. A failed request fails the whole call. + /// A request that returns fewer bytes than planned fails with + /// [`ErrorKind::Unexpected`]. The reader's version and conditions apply to + /// every request: a missing file returns [`ErrorKind::NotFound`], a failed + /// condition returns [`ErrorKind::ConditionNotMatch`], and an unsupported + /// condition returns [`ErrorKind::Unsupported`]. + /// + /// # Examples + /// + /// ``` + /// use opendal_core::{Operator, Result, blocking, options, services}; + /// + /// # fn main() -> Result<()> { + /// let runtime = tokio::runtime::Runtime::new().unwrap(); + /// let op = { + /// let _guard = runtime.enter(); + /// blocking::Operator::new(Operator::new(services::Memory::default())?)? + /// }; + /// op.write("file", "0123456789")?; + /// let reader = op.reader_options("file", options::ReaderOptions { + /// gap: Some(2), + /// ..Default::default() + /// })?; + /// let buffers = reader.fetch(vec![6..8, 0..2, 3..3, 0..2])?; + /// assert_eq!(buffers[0].to_vec(), b"67"); + /// assert_eq!(buffers[1].to_vec(), b"01"); + /// assert!(buffers[2].is_empty()); + /// assert_eq!(buffers[3].to_vec(), b"01"); + /// # Ok(()) + /// # } + /// ``` + pub fn fetch(&self, ranges: Vec>) -> Result> { + let inner = self + .inner + .as_ref() + .ok_or_else(|| Error::new(ErrorKind::Unexpected, "reader has been dropped"))?; + self.handle.block_on(inner.fetch(ranges)) + } + /// /// This operation will copy and write bytes into given [`BufMut`]. Allocation happens while /// [`BufMut`] doesn't have enough space. From 8ccdd821363c72a8bde2e96e4fca00288045c5ff Mon Sep 17 00:00:00 2001 From: jihuayu Date: Mon, 14 Sep 2026 16:53:12 +0800 Subject: [PATCH 07/20] feat(java): support all reader options and multi-range fetch --- bindings/java/README.md | 38 +++- bindings/java/src/lib.rs | 66 ++++-- .../java/org/apache/opendal/Operator.java | 6 +- .../org/apache/opendal/OperatorReader.java | 39 +++- .../org/apache/opendal/ReaderOptions.java | 66 +++++- bindings/java/src/operator_input_stream.rs | 10 +- bindings/java/src/operator_reader.rs | 58 ++++- .../opendal/test/OperatorReaderTest.java | 56 +++++ .../opendal/test/ReaderOptionsTest.java | 214 ++++++++++++++++++ website/docs/20-bindings/java/04-tasks.md | 29 +++ 10 files changed, 551 insertions(+), 31 deletions(-) create mode 100644 bindings/java/src/test/java/org/apache/opendal/test/ReaderOptionsTest.java diff --git a/bindings/java/README.md b/bindings/java/README.md index 5f2aaff8f982..1cc2a7122f04 100644 --- a/bindings/java/README.md +++ b/bindings/java/README.md @@ -95,8 +95,9 @@ Use the synchronous `Operator` for blocking calls, or `AsyncOperator` for ## Reuse a reader and tune reads `Operator.reader(path, readerOptions)` creates an `OperatorReader` backed by a -Rust core reader. `ReaderOptions` controls how it executes reads; each call to -`read` or `createInputStream` selects its own logical byte range. `ReadOptions` +Rust core reader. `ReaderOptions` selects versions, conditions, and execution +controls; each call to `read`, `fetch`, or `createInputStream` selects its own +logical byte ranges. `ReadOptions` can also supply that range. Existing `Operator.createInputStream` overloads use the same abstraction and preserve their defaults. @@ -134,9 +135,36 @@ Payload memory usage generally grows with chunk size, concurrency, and prefetching; SDK, JNI, and Java buffers add further overhead. These options do not imply a fixed memory formula or a throughput guarantee. -The core `gap` option only affects multi-range `Reader::fetch` and is not -applicable to this continuous stream API. Version and conditional read options -are outside this API's current scope. +`ReaderOptions` exposes every Rust core reader option: + +- `version` selects a stored version instead of the current file. +- `ifMatch` and `ifNoneMatch` check the file's ETag. +- `ifVersionMatch` and `ifVersionNotMatch` check the file's version. +- `ifModifiedSince` and `ifUnmodifiedSince` accept `java.time.Instant` values. +- `concurrent`, `chunk`, `prefetch`, and `contentLengthHint` control execution. +- `gap` controls merging nearby ranges during `fetch`. + +Versions and conditions require service support. Unsupported options fail with +`Unsupported`; a failed condition on an existing file fails with +`ConditionNotMatch`, and a missing file fails with `NotFound`. Errors may +surface at reader creation or during a read. All conditions must hold for each +request, including requests from streams and `fetch`. + +Use `fetch` to read multiple bounded ranges in one call: + +```java +ReaderOptions options = ReaderOptions.builder().gap(4096).build(); +try (OperatorReader reader = op.reader("large.bin", options)) { + byte[][] parts = reader.fetch( + ReadOptions.builder().offset(0).length(1024).build(), + ReadOptions.builder().offset(2048).length(1024).build()); +} +``` + +Results preserve input order, including duplicates and empty ranges. Fetch +requires non-negative lengths, so `-1` is not valid here. A `gap` of `0` disables +merging across gaps; `-1` uses the core default of 1 MiB. Gap bytes are excluded +from results. This option does not affect `read` or `createInputStream`. ## Documentation diff --git a/bindings/java/src/lib.rs b/bindings/java/src/lib.rs index 8b170c8fce03..a9610ef2c432 100644 --- a/bindings/java/src/lib.rs +++ b/bindings/java/src/lib.rs @@ -297,12 +297,30 @@ fn make_reader_options( env: &mut Env, options: &JObject, ) -> Result { - Ok(build_reader_options( - convert::read_int_field(env, options, "concurrent")?, - convert::read_int64_field(env, options, "chunk")?, - convert::read_int_field(env, options, "prefetch")?, - convert::read_int64_field(env, options, "contentLengthHint")?, - )?) + Ok(opendal::options::ReaderOptions { + version: convert::read_string_field(env, options, "version")?, + if_match: convert::read_string_field(env, options, "ifMatch")?, + if_none_match: convert::read_string_field(env, options, "ifNoneMatch")?, + if_version_match: convert::read_string_field(env, options, "ifVersionMatch")?, + if_version_not_match: convert::read_string_field(env, options, "ifVersionNotMatch")?, + if_modified_since: convert::read_instant_field_to_timestamp( + env, + options, + "ifModifiedSince", + )?, + if_unmodified_since: convert::read_instant_field_to_timestamp( + env, + options, + "ifUnmodifiedSince", + )?, + ..build_reader_options( + convert::read_int_field(env, options, "concurrent")?, + convert::read_int64_field(env, options, "chunk")?, + convert::read_int_field(env, options, "prefetch")?, + convert::read_int64_field(env, options, "contentLengthHint")?, + convert::read_int64_field(env, options, "gap")?, + )? + }) } fn build_reader_options( @@ -310,6 +328,7 @@ fn build_reader_options( chunk: i64, prefetch: i32, content_length_hint: i64, + gap: i64, ) -> opendal::Result { use opendal::{Error, ErrorKind}; @@ -345,12 +364,22 @@ fn build_reader_options( ) })?), }; + let gap = match gap { + -1 => None, + value => Some(usize::try_from(value).map_err(|_| { + Error::new( + ErrorKind::ConfigInvalid, + "gap must be -1 or a non-negative native-sized integer", + ) + })?), + }; Ok(opendal::options::ReaderOptions { concurrent, chunk, prefetch, content_length_hint, + gap, ..Default::default() }) } @@ -362,35 +391,42 @@ mod reader_options_tests { #[test] fn default_reader_options() { - let options = build_reader_options(1, -1, 0, -1).unwrap(); + let options = build_reader_options(1, -1, 0, -1, -1).unwrap(); assert_eq!(options.concurrent, 1); assert_eq!(options.chunk, None); assert_eq!(options.prefetch, 0); assert_eq!(options.content_length_hint, None); + assert_eq!(options.gap, None); } #[test] fn tuned_reader_options() { - let options = build_reader_options(4, 8 * 1024 * 1024, 2, 128 * 1024 * 1024).unwrap(); + let options = build_reader_options(4, 8 * 1024 * 1024, 2, 128 * 1024 * 1024, 16).unwrap(); assert_eq!(options.concurrent, 4); assert_eq!(options.chunk, Some(8 * 1024 * 1024)); assert_eq!(options.prefetch, 2); assert_eq!(options.content_length_hint, Some(128 * 1024 * 1024)); + assert_eq!(options.gap, Some(16)); } #[test] fn empty_content_length_hint() { - let options = build_reader_options(1, 1, 0, 0).unwrap(); + let options = build_reader_options(1, 1, 0, 0, 0).unwrap(); assert_eq!(options.content_length_hint, Some(0)); + assert_eq!(options.gap, Some(0)); } #[test] - fn chunk_conversion_respects_native_width() { - let options = build_reader_options(1, i64::MAX, 0, -1); - if usize::BITS < 64 { - assert_eq!(options.unwrap_err().kind(), ErrorKind::ConfigInvalid); - } else { - assert_eq!(options.unwrap().chunk, usize::try_from(i64::MAX).ok()); + fn reader_sizes_respect_native_width() { + for (chunk, gap) in [(i64::MAX, -1), (-1, i64::MAX)] { + let options = build_reader_options(1, chunk, 0, -1, gap); + if usize::BITS < 64 { + assert_eq!(options.unwrap_err().kind(), ErrorKind::ConfigInvalid); + } else { + let options = options.unwrap(); + assert_eq!(options.chunk, usize::try_from(chunk).ok()); + assert_eq!(options.gap, usize::try_from(gap).ok()); + } } } } diff --git a/bindings/java/src/main/java/org/apache/opendal/Operator.java b/bindings/java/src/main/java/org/apache/opendal/Operator.java index e8e65353cace..9c0a0489b9f5 100644 --- a/bindings/java/src/main/java/org/apache/opendal/Operator.java +++ b/bindings/java/src/main/java/org/apache/opendal/Operator.java @@ -130,12 +130,16 @@ public OperatorReader reader(String path) { * Creates a reusable reader for a file. Options apply to every read through the reader; * each read selects its own byte range. The reader can outlive this operator. * Creation does not read file contents or guarantee that the file exists. + * Version selection and conditions apply to all requests through the reader, including + * streams and fetches. All conditions must hold; a failed condition on an existing file + * fails with ConditionNotMatch and a missing file fails with NotFound. Depending on the + * service, errors may surface at creation or during reading. * * @param path file path * @param options reader execution options * @return a reader that the caller must close * @throws OpenDALException if options are invalid (ConfigInvalid), the path is a directory - * (IsADirectory), or the service does not support reads (Unsupported) + * (IsADirectory), or the service does not support reads or a requested option (Unsupported) * @throws IllegalStateException if this operator is closed */ public OperatorReader reader(String path, ReaderOptions options) { diff --git a/bindings/java/src/main/java/org/apache/opendal/OperatorReader.java b/bindings/java/src/main/java/org/apache/opendal/OperatorReader.java index f5b12f663d1a..0bd2ef728386 100644 --- a/bindings/java/src/main/java/org/apache/opendal/OperatorReader.java +++ b/bindings/java/src/main/java/org/apache/opendal/OperatorReader.java @@ -26,6 +26,10 @@ * Each read selects its own range and does not advance a shared cursor. * Reader options apply to every read. A reader does not snapshot the file; * changes to the file may be visible to subsequent reads. + * Version selection and conditions require service support. Unsupported options fail with + * Unsupported, failed conditions on existing files fail with ConditionNotMatch, and missing + * files fail with NotFound. These errors may surface at creation or during reads, streams, + * and fetches. All configured conditions must hold for each request. * *

Close the reader when it is no longer needed, preferably with try-with-resources. * Closing the operator that created it does not close the reader. @@ -42,7 +46,8 @@ public final class OperatorReader extends NativeObject { * Reads the whole file into a Java byte array. * * @return file contents - * @throws OpenDALException if the file does not exist (NotFound) or reading fails + * @throws OpenDALException if the file does not exist (NotFound), a condition fails + * (ConditionNotMatch), or reading fails * @throws IllegalStateException if this reader is closed */ public byte[] read() { @@ -56,7 +61,7 @@ public byte[] read() { * @param length number of bytes to read, or -1 to read to the end; zero returns an empty array * @return contents of the requested range * @throws OpenDALException if the range is invalid (RangeNotSatisfied), the file does not - * exist (NotFound), or reading fails + * exist (NotFound), a condition fails (ConditionNotMatch), or reading fails * @throws IllegalStateException if this reader is closed */ public synchronized byte[] read(long offset, long length) { @@ -78,6 +83,34 @@ public byte[] read(ReadOptions options) { return read(options.offset, options.length); } + /** + * Fetches multiple bounded ranges into byte arrays in the same order as the input. + * Overlapping and adjacent ranges are merged; {@link ReaderOptions#gap} also allows + * merging nearby ranges. Chunking, concurrency, prefetch, version, and conditions apply + * to all requests. Empty ranges produce empty arrays, and empty input performs no reads. + * A failed request fails the whole call. This method does not advance a shared cursor. + * + * @param ranges ranges with non-negative offsets and lengths; -1 lengths are not supported + * @return one byte array per input range, including duplicates and empty ranges + * @throws OpenDALException if a range is invalid (RangeNotSatisfied), a request returns + * fewer bytes than planned (Unexpected), a condition fails (ConditionNotMatch), or reading fails + * @throws IllegalStateException if this reader is closed + */ + public synchronized byte[][] fetch(ReadOptions... ranges) { + if (isDisposed()) { + throw new IllegalStateException("OperatorReader is closed"); + } + Objects.requireNonNull(ranges, "ranges"); + long[] offsets = new long[ranges.length]; + long[] lengths = new long[ranges.length]; + for (int i = 0; i < ranges.length; i++) { + ReadOptions range = Objects.requireNonNull(ranges[i], "range"); + offsets[i] = range.offset; + lengths[i] = range.length; + } + return fetchRanges(nativeHandle, offsets, lengths); + } + /** * Creates an independent stream over the whole file. * @@ -136,6 +169,8 @@ protected void disposeInternal(long handle) { private static native byte[] readBytes(long reader, long offset, long length); + private static native byte[][] fetchRanges(long reader, long[] offsets, long[] lengths); + private static native long createBytesIterator(long reader, long offset, long length); private static native void disposeReader(long reader); diff --git a/bindings/java/src/main/java/org/apache/opendal/ReaderOptions.java b/bindings/java/src/main/java/org/apache/opendal/ReaderOptions.java index 416abc26e04b..1ebfe7bba18a 100644 --- a/bindings/java/src/main/java/org/apache/opendal/ReaderOptions.java +++ b/bindings/java/src/main/java/org/apache/opendal/ReaderOptions.java @@ -19,15 +19,66 @@ package org.apache.opendal; +import java.time.Instant; import lombok.Builder; /** - * Controls how an {@link OperatorReader} and its streams execute reads, independently of each logical range. + * Selects a file version, read conditions, and execution options for an {@link OperatorReader}. + * All conditions must hold for data to be returned. Missing files fail with NotFound; + * failed conditions on existing files fail with ConditionNotMatch. Unsupported conditions + * fail with Unsupported. Depending on the service, errors can surface when creating the + * reader or while reading. Conditions apply to every request, including streams and fetches. + * Null version and condition fields leave those options unset. + * + *

Execution options apply independently of each logical range. * Setting concurrent without setting chunk does not enable concurrent range reads. - * Invalid values cause an OpenDALException with code ConfigInvalid when the reader is created. + * Invalid execution values fail with ConfigInvalid when the reader is created. + * An Instant outside the Rust core timestamp range fails with Unexpected. */ @Builder public final class ReaderOptions { + /** + * Selects a stored file version instead of the current one. This is not a condition. + * Requires service support for reading versions; a missing version fails with NotFound. + */ + public final String version; + + /** + * Reads only when the file has this exact ETag. Requires service support for if-match reads. + * Only concrete ETags are portable; a wildcard such as "*" has no portable meaning. + */ + public final String ifMatch; + + /** + * Reads only when the file exists with a different ETag. Requires service support for + * if-none-match reads. Only concrete ETags are portable; "*" has no portable meaning. + */ + public final String ifNoneMatch; + + /** + * Reads only when the file has this exact version. Requires service support for version-match reads. + * This checks the selected file's identity rather than selecting a stored version. + */ + public final String ifVersionMatch; + + /** + * Reads only when the file exists with a different version. + * Requires service support for version-not-match reads. + */ + public final String ifVersionNotMatch; + + /** + * Reads only when the file was modified after this time. + * Requires service support for if-modified-since reads. + */ + public final Instant ifModifiedSince; + + /** + * Reads only when the file was not modified after this time. + * Requires service support for if-unmodified-since reads. + */ + public final Instant ifUnmodifiedSince; + /** * Maximum number of internal chunk requests executed concurrently. Must be positive. * This is not the number of application transfers or Java threads and only affects chunked reads. @@ -43,6 +94,17 @@ public final class ReaderOptions { @Builder.Default public final long chunk = -1L; + /** + * Maximum gap in bytes between ranges that {@link OperatorReader#fetch(ReadOptions...)} + * may merge into one request. The unrequested bytes are discarded from the returned arrays. + * Zero disables merging across gaps; overlapping and adjacent ranges still merge. + * The default of -1 uses the core default (1 MiB). Values below -1 are invalid, + * and non-negative values must fit the native platform's unsigned pointer-sized integer. + * This option does not affect read or createInputStream calls. + */ + @Builder.Default + public final long gap = -1L; + /** * Maximum number of completed chunks buffered ahead of consumption, not a byte count. * Must be non-negative. The default of zero applies strict backpressure. diff --git a/bindings/java/src/operator_input_stream.rs b/bindings/java/src/operator_input_stream.rs index cf22253d5834..57ee29d30423 100644 --- a/bindings/java/src/operator_input_stream.rs +++ b/bindings/java/src/operator_input_stream.rs @@ -57,11 +57,11 @@ fn intern_read_next_bytes<'local>( env: &mut Env<'local>, reader: &mut StdBytesIterator, ) -> crate::Result> { - match reader - .next() - .transpose() - .map_err(|err| opendal::Error::new(opendal::ErrorKind::Unexpected, err.to_string()))? - { + match reader.next().transpose().map_err(|err| { + err.downcast::().unwrap_or_else(|err| { + opendal::Error::new(opendal::ErrorKind::Unexpected, err.to_string()) + }) + })? { None => Ok(JByteArray::default()), Some(content) => Ok(env.byte_array_from_slice(&content)?), } diff --git a/bindings/java/src/operator_reader.rs b/bindings/java/src/operator_reader.rs index 5d9b1a59b1b7..0b352b54bb77 100644 --- a/bindings/java/src/operator_reader.rs +++ b/bindings/java/src/operator_reader.rs @@ -16,9 +16,12 @@ // under the License. use jni::EnvUnowned; -use jni::objects::{JByteArray, JClass, JObject, JString}; +use jni::jni_str; +use jni::objects::{JByteArray, JClass, JLongArray, JObject, JObjectArray, JString}; use jni::sys::jlong; +use jni::sys::jsize; use opendal::blocking; +use std::ops::Bound; use crate::convert; use crate::error::ThrowException; @@ -84,6 +87,59 @@ pub unsafe extern "system" fn Java_org_apache_opendal_OperatorReader_createBytes .resolve::() } +/// # Safety +/// +/// `reader` must point to a live blocking reader for the duration of this call. +#[unsafe(no_mangle)] +pub unsafe extern "system" fn Java_org_apache_opendal_OperatorReader_fetchRanges<'local>( + mut env: EnvUnowned<'local>, + _: JClass<'local>, + reader: *const blocking::Reader, + offsets: JLongArray<'local>, + lengths: JLongArray<'local>, +) -> JObjectArray<'local> { + env.with_env(|env| -> crate::Result<_> { + let count = offsets.len(env)?; + if lengths.len(env)? != count { + return Err(opendal::Error::new( + opendal::ErrorKind::RangeNotSatisfied, + "offsets and lengths must have the same size", + ) + .into()); + } + let mut starts = vec![0; count]; + let mut sizes = vec![0; count]; + offsets.get_region(env, 0, &mut starts)?; + lengths.get_region(env, 0, &mut sizes)?; + let ranges = starts + .into_iter() + .zip(sizes) + .map( + |(offset, length)| match convert::offset_length_to_range(offset, length)? { + (Bound::Included(start), Bound::Excluded(end)) => Ok(start..end), + _ => Err(opendal::Error::new( + opendal::ErrorKind::RangeNotSatisfied, + "fetch requires non-negative lengths", + ) + .into()), + }, + ) + .collect::>>()?; + let reader = unsafe { &*reader }; + let buffers = reader.fetch(ranges)?; + let output = env.new_object_array(count as jsize, jni_str!("[B"), JObject::null())?; + for (index, buffer) in buffers.into_iter().enumerate() { + env.with_local_frame(2, |env| -> crate::Result<()> { + let bytes = convert::bytes_to_jbytearray(env, buffer.to_vec())?; + output.set_element(env, index, &bytes)?; + Ok(()) + })?; + } + Ok(output) + }) + .resolve::() +} + /// # Safety /// /// `reader` must be a live handle allocated by `Operator.reader`, with no calls in progress. diff --git a/bindings/java/src/test/java/org/apache/opendal/test/OperatorReaderTest.java b/bindings/java/src/test/java/org/apache/opendal/test/OperatorReaderTest.java index a01eba071491..390f0a68bf03 100644 --- a/bindings/java/src/test/java/org/apache/opendal/test/OperatorReaderTest.java +++ b/bindings/java/src/test/java/org/apache/opendal/test/OperatorReaderTest.java @@ -23,6 +23,8 @@ import static org.assertj.core.api.Assertions.assertThatThrownBy; import java.nio.charset.StandardCharsets; import java.nio.file.Path; +import java.time.Instant; +import java.util.stream.Stream; import org.apache.commons.io.IOUtils; import org.apache.opendal.OpenDALException; import org.apache.opendal.Operator; @@ -35,7 +37,9 @@ import org.junit.jupiter.api.Test; import org.junit.jupiter.api.io.TempDir; import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.Arguments; import org.junit.jupiter.params.provider.CsvSource; +import org.junit.jupiter.params.provider.MethodSource; public class OperatorReaderTest { @TempDir @@ -78,6 +82,9 @@ void testInvalidRange(long offset, long length) { .is(OpenDALExceptionCondition.ofSync(OpenDALException.Code.RangeNotSatisfied)); assertThatThrownBy(() -> reader.createInputStream(offset, length)) .is(OpenDALExceptionCondition.ofSync(OpenDALException.Code.RangeNotSatisfied)); + assertThatThrownBy(() -> reader.fetch( + ReadOptions.builder().offset(offset).length(length).build())) + .is(OpenDALExceptionCondition.ofSync(OpenDALException.Code.RangeNotSatisfied)); } } @@ -107,6 +114,7 @@ void testIndependentStreams() throws Exception { reader.close(); op.close(); assertThatThrownBy(reader::createInputStream).isInstanceOf(IllegalStateException.class); + assertThatThrownBy(reader::fetch).isInstanceOf(IllegalStateException.class); assertThat(first.read()).isEqualTo('0'); assertThat(second.read()).isEqualTo('4'); first.close(); @@ -133,6 +141,54 @@ void testZeroLengthStreamReads() { } } + @Test + void testFetchRequiresBoundedRanges() { + try (Operator op = Operator.of( + ServiceConfig.Fs.builder().root(tempDir.toString()).build()); + OperatorReader reader = op.reader("missing")) { + assertThatThrownBy(() -> reader.fetch(ReadOptions.builder().build())) + .is(OpenDALExceptionCondition.ofSync(OpenDALException.Code.RangeNotSatisfied)); + } + } + + static Stream unsupportedReaderConditions() { + return Stream.of( + Arguments.of(ReaderOptions.builder().version("v").build(), "version"), + Arguments.of(ReaderOptions.builder().ifMatch("etag").build(), "if_match"), + Arguments.of(ReaderOptions.builder().ifNoneMatch("etag").build(), "if_none_match"), + Arguments.of(ReaderOptions.builder().ifVersionMatch("v").build(), "if_version_match"), + Arguments.of(ReaderOptions.builder().ifVersionNotMatch("v").build(), "if_version_not_match"), + Arguments.of( + ReaderOptions.builder().ifModifiedSince(Instant.EPOCH).build(), "if_modified_since"), + Arguments.of( + ReaderOptions.builder().ifUnmodifiedSince(Instant.EPOCH).build(), "if_unmodified_since")); + } + + @ParameterizedTest + @MethodSource("unsupportedReaderConditions") + void testUnsupportedConditionsAreNotDropped(ReaderOptions options, String field) { + try (Operator op = + Operator.of(ServiceConfig.Fs.builder().root(tempDir.toString()).build())) { + assertThatThrownBy(() -> op.reader("missing", options)) + .is(OpenDALExceptionCondition.ofSync(OpenDALException.Code.Unsupported)) + .hasMessageContaining(field); + } + } + + @Test + void testInvalidGapAndTimestamp() { + try (Operator op = + Operator.of(ServiceConfig.Fs.builder().root(tempDir.toString()).build())) { + assertThatThrownBy(() -> + op.reader("missing", ReaderOptions.builder().gap(-2).build())) + .is(OpenDALExceptionCondition.ofSync(OpenDALException.Code.ConfigInvalid)); + assertThatThrownBy(() -> op.reader( + "missing", + ReaderOptions.builder().ifModifiedSince(Instant.MIN).build())) + .is(OpenDALExceptionCondition.ofSync(OpenDALException.Code.Unexpected)); + } + } + @Test void testInvalidReaderOptions() { try (Operator op = diff --git a/bindings/java/src/test/java/org/apache/opendal/test/ReaderOptionsTest.java b/bindings/java/src/test/java/org/apache/opendal/test/ReaderOptionsTest.java new file mode 100644 index 000000000000..52add73d0a4e --- /dev/null +++ b/bindings/java/src/test/java/org/apache/opendal/test/ReaderOptionsTest.java @@ -0,0 +1,214 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.opendal.test; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; +import com.sun.net.httpserver.Headers; +import com.sun.net.httpserver.HttpServer; +import java.net.InetSocketAddress; +import java.net.URI; +import java.nio.charset.StandardCharsets; +import java.time.Instant; +import java.util.Arrays; +import java.util.List; +import java.util.concurrent.CopyOnWriteArrayList; +import org.apache.commons.io.IOUtils; +import org.apache.opendal.OpenDALException; +import org.apache.opendal.Operator; +import org.apache.opendal.OperatorInputStream; +import org.apache.opendal.OperatorReader; +import org.apache.opendal.ReadOptions; +import org.apache.opendal.ReaderOptions; +import org.apache.opendal.ServiceConfig; +import org.apache.opendal.test.condition.OpenDALExceptionCondition; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.Timeout; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.CsvSource; + +/** Verifies Java-to-core option forwarding against recorded local HTTP requests. */ +@Timeout(10) +public class ReaderOptionsTest { + private static final byte[] CONTENT = "0123456789".getBytes(StandardCharsets.UTF_8); + private final List requests = new CopyOnWriteArrayList<>(); + private final List urls = new CopyOnWriteArrayList<>(); + private HttpServer server; + private String endpoint; + + @BeforeEach + void startServer() throws Exception { + server = HttpServer.create(new InetSocketAddress("127.0.0.1", 0), 0); + server.createContext("/", exchange -> { + try { + requests.add(exchange.getRequestHeaders()); + urls.add(exchange.getRequestURI()); + if ("changed".equals(exchange.getRequestHeaders().getFirst("If-Match"))) { + exchange.sendResponseHeaders(412, -1); + return; + } + final String range = exchange.getRequestHeaders().getFirst("Range"); + final byte[] body; + if (range != null) { + String[] bounds = range.substring("bytes=".length()).split("-", -1); + int start = Integer.parseInt(bounds[0]); + int end = bounds[1].isEmpty() ? CONTENT.length - 1 : Integer.parseInt(bounds[1]); + body = Arrays.copyOfRange(CONTENT, start, end + 1); + exchange.getResponseHeaders() + .set("Content-Range", "bytes " + start + "-" + end + "/" + CONTENT.length); + } else { + body = CONTENT; + } + exchange.getResponseHeaders().set("ETag", "\"etag\""); + exchange.sendResponseHeaders(range == null ? 200 : 206, body.length); + exchange.getResponseBody().write(body); + } finally { + exchange.close(); + } + }); + server.start(); + endpoint = "http://127.0.0.1:" + server.getAddress().getPort(); + } + + @AfterEach + void stopServer() { + server.stop(0); + } + + @Test + void testConditionsReachEveryChunkAndStream() throws Exception { + ReaderOptions options = ReaderOptions.builder() + .ifMatch("\"etag\"") + .ifNoneMatch("\"other\"") + .ifModifiedSince(Instant.parse("2024-01-01T00:00:00Z")) + .ifUnmodifiedSince(Instant.parse("2024-01-02T00:00:00Z")) + .chunk(2) + .concurrent(2) + .prefetch(1) + .contentLengthHint(CONTENT.length) + .build(); + try (Operator op = Operator.of( + ServiceConfig.Http.builder().endpoint(endpoint).build()); + OperatorReader reader = op.reader("file", options)) { + assertThat(reader.read(0, 4)).isEqualTo("0123".getBytes(StandardCharsets.UTF_8)); + try (OperatorInputStream in = reader.createInputStream(4, 4)) { + assertThat(IOUtils.toByteArray(in)).isEqualTo("4567".getBytes(StandardCharsets.UTF_8)); + } + } + assertThat(requests).hasSize(4); + for (Headers headers : requests) { + assertThat(headers.getFirst("If-Match")).isEqualTo("\"etag\""); + assertThat(headers.getFirst("If-None-Match")).isEqualTo("\"other\""); + assertThat(headers.getFirst("If-Modified-Since")).isEqualTo("Mon, 01 Jan 2024 00:00:00 GMT"); + assertThat(headers.getFirst("If-Unmodified-Since")).isEqualTo("Tue, 02 Jan 2024 00:00:00 GMT"); + } + } + + @Test + void testVersionReachesReadsAndStreams() throws Exception { + try (Operator op = Operator.of(ServiceConfig.S3 + .builder() + .bucket("bucket") + .region("us-east-1") + .endpoint(endpoint) + .skipSignature(true) + .build()); + OperatorReader reader = op.reader( + "file", ReaderOptions.builder().version("version-one").build())) { + assertThat(reader.read(0, 2)).isEqualTo("01".getBytes(StandardCharsets.UTF_8)); + try (OperatorInputStream in = reader.createInputStream(4, 2)) { + assertThat(IOUtils.toByteArray(in)).isEqualTo("45".getBytes(StandardCharsets.UTF_8)); + } + } + assertThat(urls).hasSize(2); + for (URI uri : urls) { + assertThat(uri.getQuery()).contains("versionId=version-one"); + } + } + + @Test + void testVersionConditionsReachFetch() { + try (Operator op = Operator.of(ServiceConfig.Gcs.builder() + .bucket("bucket") + .endpoint(endpoint) + .skipSignature(true) + .build()); + OperatorReader reader = op.reader( + "file", + ReaderOptions.builder() + .ifVersionMatch("17") + .ifVersionNotMatch("18") + .build())) { + byte[][] data = reader.fetch( + ReadOptions.builder().offset(0).length(2).build(), + ReadOptions.builder().offset(4).length(2).build()); + assertThat(data).isDeepEqualTo(new byte[][] { + "01".getBytes(StandardCharsets.UTF_8), "45".getBytes(StandardCharsets.UTF_8) + }); + } + assertThat(urls).hasSize(1); + assertThat(urls.get(0).getQuery()).contains("ifGenerationMatch=17", "ifGenerationNotMatch=18"); + } + + @ParameterizedTest + @CsvSource({"-1, 1", "0, 2", "2, 1"}) + void testFetchGapAndRangeOrdering(long gap, int requestCount) { + try (Operator op = Operator.of( + ServiceConfig.Http.builder().endpoint(endpoint).build()); + OperatorReader reader = + op.reader("file", ReaderOptions.builder().gap(gap).build())) { + byte[][] data = reader.fetch( + ReadOptions.builder().offset(6).length(2).build(), + ReadOptions.builder().offset(0).length(2).build(), + ReadOptions.builder().offset(2).length(2).build(), + ReadOptions.builder().offset(6).length(2).build(), + ReadOptions.builder().offset(4).length(0).build()); + assertThat(data).isDeepEqualTo(new byte[][] { + "67".getBytes(StandardCharsets.UTF_8), + "01".getBytes(StandardCharsets.UTF_8), + "23".getBytes(StandardCharsets.UTF_8), + "67".getBytes(StandardCharsets.UTF_8), + new byte[0] + }); + assertThat(reader.fetch()).isEmpty(); + } + assertThat(requests).hasSize(requestCount); + } + + @Test + void testConditionalErrorsKeepTheirCode() { + try (Operator op = Operator.of( + ServiceConfig.Http.builder().endpoint(endpoint).build()); + OperatorReader reader = op.reader( + "file", ReaderOptions.builder().ifMatch("changed").build())) { + assertThatThrownBy(() -> reader.read(0, 2)) + .is(OpenDALExceptionCondition.ofSync(OpenDALException.Code.ConditionNotMatch)); + assertThatThrownBy( + () -> reader.fetch(ReadOptions.builder().length(2).build())) + .is(OpenDALExceptionCondition.ofSync(OpenDALException.Code.ConditionNotMatch)); + try (OperatorInputStream in = reader.createInputStream(0, 2)) { + assertThatThrownBy(in::read) + .is(OpenDALExceptionCondition.ofSync(OpenDALException.Code.ConditionNotMatch)); + } + } + } +} diff --git a/website/docs/20-bindings/java/04-tasks.md b/website/docs/20-bindings/java/04-tasks.md index bfc3bfe263f5..869a8263321c 100644 --- a/website/docs/20-bindings/java/04-tasks.md +++ b/website/docs/20-bindings/java/04-tasks.md @@ -57,6 +57,35 @@ requests and `prefetch` counts completed chunks buffered ahead. Setting `concurrent` alone keeps unchunked streaming. A reader does not snapshot the file, so subsequent reads may observe changes. +`ReaderOptions` also supports `version`, `ifMatch`, `ifNoneMatch`, +`ifVersionMatch`, `ifVersionNotMatch`, `ifModifiedSince`, and +`ifUnmodifiedSince`. The time conditions accept `java.time.Instant`. +Version selection and conditions require service support. Unsupported options +fail with `Unsupported`; a failed condition on an existing file fails with +`ConditionNotMatch`. Conditions apply to all requests through the reader, +including its streams and fetches. Errors may surface when creating the reader +or while reading from it. + +For multiple bounded ranges, use `fetch`: + +```java +import org.apache.opendal.ReadOptions; + +ReaderOptions options = ReaderOptions.builder().gap(4096).build(); +try (OperatorReader reader = op.reader("path/to/file", options)) { + byte[][] parts = reader.fetch( + ReadOptions.builder().offset(0).length(1024).build(), + ReadOptions.builder().offset(2048).length(1024).build()); +} +``` + +The returned arrays follow input order. Fetch requires non-negative offsets +and lengths; it does not accept `-1` lengths. The `gap` option merges nearby +ranges to reduce requests while excluding gap bytes from the results. Set it +to `0` to disable merging across gaps, or leave it at `-1` for the core default +of 1 MiB. Overlapping and adjacent ranges still merge. `gap` does not affect +single-range reads or streams. + ## Stream a large file Don't load gigabytes into memory — read through an `InputStream` in chunks: From 9a584f4ba9b91fa9bab1e30e180f6f8a60a87b84 Mon Sep 17 00:00:00 2001 From: jihuayu Date: Mon, 14 Sep 2026 17:15:31 +0800 Subject: [PATCH 08/20] test(java): remove redundant reader option cases --- .../opendal/test/OperatorReaderTest.java | 37 +++---------------- 1 file changed, 5 insertions(+), 32 deletions(-) diff --git a/bindings/java/src/test/java/org/apache/opendal/test/OperatorReaderTest.java b/bindings/java/src/test/java/org/apache/opendal/test/OperatorReaderTest.java index 390f0a68bf03..b2294ce141e8 100644 --- a/bindings/java/src/test/java/org/apache/opendal/test/OperatorReaderTest.java +++ b/bindings/java/src/test/java/org/apache/opendal/test/OperatorReaderTest.java @@ -24,7 +24,6 @@ import java.nio.charset.StandardCharsets; import java.nio.file.Path; import java.time.Instant; -import java.util.stream.Stream; import org.apache.commons.io.IOUtils; import org.apache.opendal.OpenDALException; import org.apache.opendal.Operator; @@ -37,9 +36,7 @@ import org.junit.jupiter.api.Test; import org.junit.jupiter.api.io.TempDir; import org.junit.jupiter.params.ParameterizedTest; -import org.junit.jupiter.params.provider.Arguments; import org.junit.jupiter.params.provider.CsvSource; -import org.junit.jupiter.params.provider.MethodSource; public class OperatorReaderTest { @TempDir @@ -151,27 +148,13 @@ void testFetchRequiresBoundedRanges() { } } - static Stream unsupportedReaderConditions() { - return Stream.of( - Arguments.of(ReaderOptions.builder().version("v").build(), "version"), - Arguments.of(ReaderOptions.builder().ifMatch("etag").build(), "if_match"), - Arguments.of(ReaderOptions.builder().ifNoneMatch("etag").build(), "if_none_match"), - Arguments.of(ReaderOptions.builder().ifVersionMatch("v").build(), "if_version_match"), - Arguments.of(ReaderOptions.builder().ifVersionNotMatch("v").build(), "if_version_not_match"), - Arguments.of( - ReaderOptions.builder().ifModifiedSince(Instant.EPOCH).build(), "if_modified_since"), - Arguments.of( - ReaderOptions.builder().ifUnmodifiedSince(Instant.EPOCH).build(), "if_unmodified_since")); - } - - @ParameterizedTest - @MethodSource("unsupportedReaderConditions") - void testUnsupportedConditionsAreNotDropped(ReaderOptions options, String field) { + @Test + void testUnsupportedReaderCondition() { try (Operator op = Operator.of(ServiceConfig.Fs.builder().root(tempDir.toString()).build())) { - assertThatThrownBy(() -> op.reader("missing", options)) - .is(OpenDALExceptionCondition.ofSync(OpenDALException.Code.Unsupported)) - .hasMessageContaining(field); + assertThatThrownBy(() -> op.reader( + "missing", ReaderOptions.builder().ifMatch("etag").build())) + .is(OpenDALExceptionCondition.ofSync(OpenDALException.Code.Unsupported)); } } @@ -188,14 +171,4 @@ void testInvalidGapAndTimestamp() { .is(OpenDALExceptionCondition.ofSync(OpenDALException.Code.Unexpected)); } } - - @Test - void testInvalidReaderOptions() { - try (Operator op = - Operator.of(ServiceConfig.Fs.builder().root(tempDir.toString()).build())) { - assertThatThrownBy(() -> op.reader( - "missing", ReaderOptions.builder().chunk(0).build())) - .is(OpenDALExceptionCondition.ofSync(OpenDALException.Code.ConfigInvalid)); - } - } } From 09c92b03274f0c99857205bdd1acbec13f1d385d Mon Sep 17 00:00:00 2001 From: jihuayu Date: Mon, 14 Sep 2026 17:16:29 +0800 Subject: [PATCH 09/20] refactor(java): simplify input stream read dispatch --- .../apache/opendal/OperatorInputStream.java | 4 --- bindings/java/src/operator_input_stream.rs | 28 +++++++------------ 2 files changed, 10 insertions(+), 22 deletions(-) diff --git a/bindings/java/src/main/java/org/apache/opendal/OperatorInputStream.java b/bindings/java/src/main/java/org/apache/opendal/OperatorInputStream.java index 8fad0d2e2c52..dfb97c11e890 100644 --- a/bindings/java/src/main/java/org/apache/opendal/OperatorInputStream.java +++ b/bindings/java/src/main/java/org/apache/opendal/OperatorInputStream.java @@ -96,10 +96,6 @@ public synchronized int read(byte[] b, int off, int len) { if (reader.isDisposed()) { throw new IllegalStateException("OperatorInputStream is closed"); } - if (len == 0) { - return 0; - } - int read = 0; while (len > 0) { while (bytes != null && offset >= bytes.length) { diff --git a/bindings/java/src/operator_input_stream.rs b/bindings/java/src/operator_input_stream.rs index 57ee29d30423..ebe10245f67e 100644 --- a/bindings/java/src/operator_input_stream.rs +++ b/bindings/java/src/operator_input_stream.rs @@ -15,7 +15,6 @@ // specific language governing permissions and limitations // under the License. -use jni::Env; use jni::EnvUnowned; use jni::objects::JByteArray; use jni::objects::JClass; @@ -46,23 +45,16 @@ pub unsafe extern "system" fn Java_org_apache_opendal_OperatorInputStream_readNe _: JClass<'local>, reader: *mut StdBytesIterator, ) -> JByteArray<'local> { - env.with_env(|env| { - let reader_ref = unsafe { &mut *reader }; - intern_read_next_bytes(env, reader_ref) + env.with_env(|env| -> crate::Result<_> { + let reader = unsafe { &mut *reader }; + match reader.next().transpose().map_err(|err| { + err.downcast::().unwrap_or_else(|err| { + opendal::Error::new(opendal::ErrorKind::Unexpected, err.to_string()) + }) + })? { + None => Ok(JByteArray::default()), + Some(content) => Ok(env.byte_array_from_slice(&content)?), + } }) .resolve::() } - -fn intern_read_next_bytes<'local>( - env: &mut Env<'local>, - reader: &mut StdBytesIterator, -) -> crate::Result> { - match reader.next().transpose().map_err(|err| { - err.downcast::().unwrap_or_else(|err| { - opendal::Error::new(opendal::ErrorKind::Unexpected, err.to_string()) - }) - })? { - None => Ok(JByteArray::default()), - Some(content) => Ok(env.byte_array_from_slice(&content)?), - } -} From c601baaf4e94f94db5148effb8cf4a494455ddba Mon Sep 17 00:00:00 2001 From: jihuayu Date: Mon, 14 Sep 2026 17:18:42 +0800 Subject: [PATCH 10/20] docs(java): trim duplicate reader guidance --- bindings/java/README.md | 77 +++---------------- .../org/apache/opendal/OperatorReader.java | 8 +- core/core/src/blocking/read/reader.rs | 11 +-- 3 files changed, 17 insertions(+), 79 deletions(-) diff --git a/bindings/java/README.md b/bindings/java/README.md index 1cc2a7122f04..b9179e150411 100644 --- a/bindings/java/README.md +++ b/bindings/java/README.md @@ -92,79 +92,24 @@ public class Main { Use the synchronous `Operator` for blocking calls, or `AsyncOperator` for `CompletableFuture`-based calls. -## Reuse a reader and tune reads +## Reuse a reader -`Operator.reader(path, readerOptions)` creates an `OperatorReader` backed by a -Rust core reader. `ReaderOptions` selects versions, conditions, and execution -controls; each call to `read`, `fetch`, or `createInputStream` selects its own -logical byte ranges. `ReadOptions` -can also supply that range. Existing `Operator.createInputStream` overloads use -the same abstraction and preserve their defaults. +`Operator.reader(path, readerOptions)` creates an `OperatorReader` for repeated +reads. `ReaderOptions` selects versions, conditions, and execution controls; +each call selects its own byte range. ```java -ReaderOptions readerOptions = ReaderOptions.builder() - .concurrent(4) - .chunk(8 * 1024 * 1024L) - .prefetch(2) - .build(); - -try (OperatorReader reader = op.reader("large.bin", readerOptions); - OperatorInputStream in = reader.createInputStream()) { - byte[] buffer = new byte[8192]; - int count; - while ((count = in.read(buffer)) != -1) { - // Process buffer[0..count). - } -} -``` - -Use `reader.read(offset, length)` to collect a range into a byte array, or -`reader.createInputStream(offset, length)` to stream it. A length of `-1` reads -to the end; zero selects an empty range. Repeated reads do not share a cursor. -Close each reader and stream separately. Streams remain usable after their -reader closes, and readers remain usable after their operator closes. A reader -does not snapshot the file, so later reads may observe changes. - -A positive `chunk` enables internal range requests. `concurrent` limits these -requests, not application transfers or Java threads; `prefetch` counts completed -chunks, not bytes. Setting `concurrent` alone keeps unchunked streaming. -See [ReaderOptions](src/main/java/org/apache/opendal/ReaderOptions.java) for -all defaults, valid values, and content length hint semantics. - -Payload memory usage generally grows with chunk size, -concurrency, and prefetching; SDK, JNI, and Java buffers add further overhead. -These options do not imply a fixed memory formula or a throughput guarantee. - -`ReaderOptions` exposes every Rust core reader option: - -- `version` selects a stored version instead of the current file. -- `ifMatch` and `ifNoneMatch` check the file's ETag. -- `ifVersionMatch` and `ifVersionNotMatch` check the file's version. -- `ifModifiedSince` and `ifUnmodifiedSince` accept `java.time.Instant` values. -- `concurrent`, `chunk`, `prefetch`, and `contentLengthHint` control execution. -- `gap` controls merging nearby ranges during `fetch`. - -Versions and conditions require service support. Unsupported options fail with -`Unsupported`; a failed condition on an existing file fails with -`ConditionNotMatch`, and a missing file fails with `NotFound`. Errors may -surface at reader creation or during a read. All conditions must hold for each -request, including requests from streams and `fetch`. - -Use `fetch` to read multiple bounded ranges in one call: - -```java -ReaderOptions options = ReaderOptions.builder().gap(4096).build(); +ReaderOptions options = ReaderOptions.builder().chunk(8 * 1024 * 1024L).build(); try (OperatorReader reader = op.reader("large.bin", options)) { - byte[][] parts = reader.fetch( - ReadOptions.builder().offset(0).length(1024).build(), - ReadOptions.builder().offset(2048).length(1024).build()); + byte[] first = reader.read(0, 1024); + byte[] next = reader.read(1024, 1024); } ``` -Results preserve input order, including duplicates and empty ranges. Fetch -requires non-negative lengths, so `-1` is not valid here. A `gap` of `0` disables -merging across gaps; `-1` uses the core default of 1 MiB. Gap bytes are excluded -from results. This option does not affect `read` or `createInputStream`. +The [Java task guide](../../website/docs/20-bindings/java/04-tasks.md#read-part-of-a-file) +covers streams, multi-range `fetch`, and resource lifetimes. +See [ReaderOptions](src/main/java/org/apache/opendal/ReaderOptions.java) for all +supported options, defaults, and constraints. ## Documentation diff --git a/bindings/java/src/main/java/org/apache/opendal/OperatorReader.java b/bindings/java/src/main/java/org/apache/opendal/OperatorReader.java index 0bd2ef728386..bdee5f706f9d 100644 --- a/bindings/java/src/main/java/org/apache/opendal/OperatorReader.java +++ b/bindings/java/src/main/java/org/apache/opendal/OperatorReader.java @@ -24,18 +24,16 @@ /** * Reads a file synchronously through a reusable Rust core reader. * Each read selects its own range and does not advance a shared cursor. - * Reader options apply to every read. A reader does not snapshot the file; + * Reader options apply to every request, including streams and fetches. + * A reader does not snapshot the file; * changes to the file may be visible to subsequent reads. - * Version selection and conditions require service support. Unsupported options fail with - * Unsupported, failed conditions on existing files fail with ConditionNotMatch, and missing - * files fail with NotFound. These errors may surface at creation or during reads, streams, - * and fetches. All configured conditions must hold for each request. * *

Close the reader when it is no longer needed, preferably with try-with-resources. * Closing the operator that created it does not close the reader. * Calls on one reader are serialized, including close. * * @see Operator#reader(String, ReaderOptions) + * @see ReaderOptions */ public final class OperatorReader extends NativeObject { OperatorReader(long nativeHandle) { diff --git a/core/core/src/blocking/read/reader.rs b/core/core/src/blocking/read/reader.rs index 97b983865c91..31d21832e921 100644 --- a/core/core/src/blocking/read/reader.rs +++ b/core/core/src/blocking/read/reader.rs @@ -81,7 +81,7 @@ impl Reader { /// # Examples /// /// ``` - /// use opendal_core::{Operator, Result, blocking, options, services}; + /// use opendal_core::{Operator, Result, blocking, services}; /// /// # fn main() -> Result<()> { /// let runtime = tokio::runtime::Runtime::new().unwrap(); @@ -90,15 +90,10 @@ impl Reader { /// blocking::Operator::new(Operator::new(services::Memory::default())?)? /// }; /// op.write("file", "0123456789")?; - /// let reader = op.reader_options("file", options::ReaderOptions { - /// gap: Some(2), - /// ..Default::default() - /// })?; - /// let buffers = reader.fetch(vec![6..8, 0..2, 3..3, 0..2])?; + /// let reader = op.reader("file")?; + /// let buffers = reader.fetch(vec![6..8, 0..2])?; /// assert_eq!(buffers[0].to_vec(), b"67"); /// assert_eq!(buffers[1].to_vec(), b"01"); - /// assert!(buffers[2].is_empty()); - /// assert_eq!(buffers[3].to_vec(), b"01"); /// # Ok(()) /// # } /// ``` From 2d58355a7a5206985cd9bfe9f99eb51dcac9bdcb Mon Sep 17 00:00:00 2001 From: jihuayu Date: Mon, 14 Sep 2026 17:32:26 +0800 Subject: [PATCH 11/20] test(java): verify reader version conditions through read --- .../java/org/apache/opendal/test/ReaderOptionsTest.java | 9 ++------- 1 file changed, 2 insertions(+), 7 deletions(-) diff --git a/bindings/java/src/test/java/org/apache/opendal/test/ReaderOptionsTest.java b/bindings/java/src/test/java/org/apache/opendal/test/ReaderOptionsTest.java index 52add73d0a4e..ac253f657fb4 100644 --- a/bindings/java/src/test/java/org/apache/opendal/test/ReaderOptionsTest.java +++ b/bindings/java/src/test/java/org/apache/opendal/test/ReaderOptionsTest.java @@ -146,7 +146,7 @@ void testVersionReachesReadsAndStreams() throws Exception { } @Test - void testVersionConditionsReachFetch() { + void testVersionConditionsReachReads() { try (Operator op = Operator.of(ServiceConfig.Gcs.builder() .bucket("bucket") .endpoint(endpoint) @@ -158,12 +158,7 @@ void testVersionConditionsReachFetch() { .ifVersionMatch("17") .ifVersionNotMatch("18") .build())) { - byte[][] data = reader.fetch( - ReadOptions.builder().offset(0).length(2).build(), - ReadOptions.builder().offset(4).length(2).build()); - assertThat(data).isDeepEqualTo(new byte[][] { - "01".getBytes(StandardCharsets.UTF_8), "45".getBytes(StandardCharsets.UTF_8) - }); + assertThat(reader.read(0, 2)).isEqualTo("01".getBytes(StandardCharsets.UTF_8)); } assertThat(urls).hasSize(1); assertThat(urls.get(0).getQuery()).contains("ifGenerationMatch=17", "ifGenerationNotMatch=18"); From 0473924bb7e26dc83449836e81d7d4710e9b3224 Mon Sep 17 00:00:00 2001 From: jihuayu Date: Mon, 14 Sep 2026 17:35:21 +0800 Subject: [PATCH 12/20] refactor(java): defer reader gap and fetch APIs --- bindings/java/README.md | 2 +- bindings/java/src/lib.rs | 37 +++--------- .../java/org/apache/opendal/Operator.java | 2 +- .../org/apache/opendal/OperatorReader.java | 32 +--------- .../org/apache/opendal/ReaderOptions.java | 13 +---- bindings/java/src/operator_reader.rs | 58 +------------------ .../opendal/test/OperatorReaderTest.java | 19 +----- .../opendal/test/ReaderOptionsTest.java | 31 ---------- core/core/src/blocking/read/reader.rs | 46 --------------- website/docs/20-bindings/java/04-tasks.md | 22 +------ 10 files changed, 16 insertions(+), 246 deletions(-) diff --git a/bindings/java/README.md b/bindings/java/README.md index b9179e150411..ce93cbc7e9cb 100644 --- a/bindings/java/README.md +++ b/bindings/java/README.md @@ -107,7 +107,7 @@ try (OperatorReader reader = op.reader("large.bin", options)) { ``` The [Java task guide](../../website/docs/20-bindings/java/04-tasks.md#read-part-of-a-file) -covers streams, multi-range `fetch`, and resource lifetimes. +covers streams and resource lifetimes. See [ReaderOptions](src/main/java/org/apache/opendal/ReaderOptions.java) for all supported options, defaults, and constraints. diff --git a/bindings/java/src/lib.rs b/bindings/java/src/lib.rs index a9610ef2c432..eaa5fda63cd3 100644 --- a/bindings/java/src/lib.rs +++ b/bindings/java/src/lib.rs @@ -318,7 +318,6 @@ fn make_reader_options( convert::read_int64_field(env, options, "chunk")?, convert::read_int_field(env, options, "prefetch")?, convert::read_int64_field(env, options, "contentLengthHint")?, - convert::read_int64_field(env, options, "gap")?, )? }) } @@ -328,7 +327,6 @@ fn build_reader_options( chunk: i64, prefetch: i32, content_length_hint: i64, - gap: i64, ) -> opendal::Result { use opendal::{Error, ErrorKind}; @@ -364,22 +362,12 @@ fn build_reader_options( ) })?), }; - let gap = match gap { - -1 => None, - value => Some(usize::try_from(value).map_err(|_| { - Error::new( - ErrorKind::ConfigInvalid, - "gap must be -1 or a non-negative native-sized integer", - ) - })?), - }; Ok(opendal::options::ReaderOptions { concurrent, chunk, prefetch, content_length_hint, - gap, ..Default::default() }) } @@ -391,42 +379,35 @@ mod reader_options_tests { #[test] fn default_reader_options() { - let options = build_reader_options(1, -1, 0, -1, -1).unwrap(); + let options = build_reader_options(1, -1, 0, -1).unwrap(); assert_eq!(options.concurrent, 1); assert_eq!(options.chunk, None); assert_eq!(options.prefetch, 0); assert_eq!(options.content_length_hint, None); - assert_eq!(options.gap, None); } #[test] fn tuned_reader_options() { - let options = build_reader_options(4, 8 * 1024 * 1024, 2, 128 * 1024 * 1024, 16).unwrap(); + let options = build_reader_options(4, 8 * 1024 * 1024, 2, 128 * 1024 * 1024).unwrap(); assert_eq!(options.concurrent, 4); assert_eq!(options.chunk, Some(8 * 1024 * 1024)); assert_eq!(options.prefetch, 2); assert_eq!(options.content_length_hint, Some(128 * 1024 * 1024)); - assert_eq!(options.gap, Some(16)); } #[test] fn empty_content_length_hint() { - let options = build_reader_options(1, 1, 0, 0, 0).unwrap(); + let options = build_reader_options(1, 1, 0, 0).unwrap(); assert_eq!(options.content_length_hint, Some(0)); - assert_eq!(options.gap, Some(0)); } #[test] - fn reader_sizes_respect_native_width() { - for (chunk, gap) in [(i64::MAX, -1), (-1, i64::MAX)] { - let options = build_reader_options(1, chunk, 0, -1, gap); - if usize::BITS < 64 { - assert_eq!(options.unwrap_err().kind(), ErrorKind::ConfigInvalid); - } else { - let options = options.unwrap(); - assert_eq!(options.chunk, usize::try_from(chunk).ok()); - assert_eq!(options.gap, usize::try_from(gap).ok()); - } + fn chunk_conversion_respects_native_width() { + let options = build_reader_options(1, i64::MAX, 0, -1); + if usize::BITS < 64 { + assert_eq!(options.unwrap_err().kind(), ErrorKind::ConfigInvalid); + } else { + assert_eq!(options.unwrap().chunk, usize::try_from(i64::MAX).ok()); } } } diff --git a/bindings/java/src/main/java/org/apache/opendal/Operator.java b/bindings/java/src/main/java/org/apache/opendal/Operator.java index 9c0a0489b9f5..7a6a5d1ed7c7 100644 --- a/bindings/java/src/main/java/org/apache/opendal/Operator.java +++ b/bindings/java/src/main/java/org/apache/opendal/Operator.java @@ -131,7 +131,7 @@ public OperatorReader reader(String path) { * each read selects its own byte range. The reader can outlive this operator. * Creation does not read file contents or guarantee that the file exists. * Version selection and conditions apply to all requests through the reader, including - * streams and fetches. All conditions must hold; a failed condition on an existing file + * streams. All conditions must hold; a failed condition on an existing file * fails with ConditionNotMatch and a missing file fails with NotFound. Depending on the * service, errors may surface at creation or during reading. * diff --git a/bindings/java/src/main/java/org/apache/opendal/OperatorReader.java b/bindings/java/src/main/java/org/apache/opendal/OperatorReader.java index bdee5f706f9d..7d5ba94cafe6 100644 --- a/bindings/java/src/main/java/org/apache/opendal/OperatorReader.java +++ b/bindings/java/src/main/java/org/apache/opendal/OperatorReader.java @@ -24,7 +24,7 @@ /** * Reads a file synchronously through a reusable Rust core reader. * Each read selects its own range and does not advance a shared cursor. - * Reader options apply to every request, including streams and fetches. + * Reader options apply to every request, including streams. * A reader does not snapshot the file; * changes to the file may be visible to subsequent reads. * @@ -81,34 +81,6 @@ public byte[] read(ReadOptions options) { return read(options.offset, options.length); } - /** - * Fetches multiple bounded ranges into byte arrays in the same order as the input. - * Overlapping and adjacent ranges are merged; {@link ReaderOptions#gap} also allows - * merging nearby ranges. Chunking, concurrency, prefetch, version, and conditions apply - * to all requests. Empty ranges produce empty arrays, and empty input performs no reads. - * A failed request fails the whole call. This method does not advance a shared cursor. - * - * @param ranges ranges with non-negative offsets and lengths; -1 lengths are not supported - * @return one byte array per input range, including duplicates and empty ranges - * @throws OpenDALException if a range is invalid (RangeNotSatisfied), a request returns - * fewer bytes than planned (Unexpected), a condition fails (ConditionNotMatch), or reading fails - * @throws IllegalStateException if this reader is closed - */ - public synchronized byte[][] fetch(ReadOptions... ranges) { - if (isDisposed()) { - throw new IllegalStateException("OperatorReader is closed"); - } - Objects.requireNonNull(ranges, "ranges"); - long[] offsets = new long[ranges.length]; - long[] lengths = new long[ranges.length]; - for (int i = 0; i < ranges.length; i++) { - ReadOptions range = Objects.requireNonNull(ranges[i], "range"); - offsets[i] = range.offset; - lengths[i] = range.length; - } - return fetchRanges(nativeHandle, offsets, lengths); - } - /** * Creates an independent stream over the whole file. * @@ -167,8 +139,6 @@ protected void disposeInternal(long handle) { private static native byte[] readBytes(long reader, long offset, long length); - private static native byte[][] fetchRanges(long reader, long[] offsets, long[] lengths); - private static native long createBytesIterator(long reader, long offset, long length); private static native void disposeReader(long reader); diff --git a/bindings/java/src/main/java/org/apache/opendal/ReaderOptions.java b/bindings/java/src/main/java/org/apache/opendal/ReaderOptions.java index 1ebfe7bba18a..d82183cd473f 100644 --- a/bindings/java/src/main/java/org/apache/opendal/ReaderOptions.java +++ b/bindings/java/src/main/java/org/apache/opendal/ReaderOptions.java @@ -27,7 +27,7 @@ * All conditions must hold for data to be returned. Missing files fail with NotFound; * failed conditions on existing files fail with ConditionNotMatch. Unsupported conditions * fail with Unsupported. Depending on the service, errors can surface when creating the - * reader or while reading. Conditions apply to every request, including streams and fetches. + * reader or while reading. Conditions apply to every request, including streams. * Null version and condition fields leave those options unset. * *

Execution options apply independently of each logical range. @@ -94,17 +94,6 @@ public final class ReaderOptions { @Builder.Default public final long chunk = -1L; - /** - * Maximum gap in bytes between ranges that {@link OperatorReader#fetch(ReadOptions...)} - * may merge into one request. The unrequested bytes are discarded from the returned arrays. - * Zero disables merging across gaps; overlapping and adjacent ranges still merge. - * The default of -1 uses the core default (1 MiB). Values below -1 are invalid, - * and non-negative values must fit the native platform's unsigned pointer-sized integer. - * This option does not affect read or createInputStream calls. - */ - @Builder.Default - public final long gap = -1L; - /** * Maximum number of completed chunks buffered ahead of consumption, not a byte count. * Must be non-negative. The default of zero applies strict backpressure. diff --git a/bindings/java/src/operator_reader.rs b/bindings/java/src/operator_reader.rs index 0b352b54bb77..5d9b1a59b1b7 100644 --- a/bindings/java/src/operator_reader.rs +++ b/bindings/java/src/operator_reader.rs @@ -16,12 +16,9 @@ // under the License. use jni::EnvUnowned; -use jni::jni_str; -use jni::objects::{JByteArray, JClass, JLongArray, JObject, JObjectArray, JString}; +use jni::objects::{JByteArray, JClass, JObject, JString}; use jni::sys::jlong; -use jni::sys::jsize; use opendal::blocking; -use std::ops::Bound; use crate::convert; use crate::error::ThrowException; @@ -87,59 +84,6 @@ pub unsafe extern "system" fn Java_org_apache_opendal_OperatorReader_createBytes .resolve::() } -/// # Safety -/// -/// `reader` must point to a live blocking reader for the duration of this call. -#[unsafe(no_mangle)] -pub unsafe extern "system" fn Java_org_apache_opendal_OperatorReader_fetchRanges<'local>( - mut env: EnvUnowned<'local>, - _: JClass<'local>, - reader: *const blocking::Reader, - offsets: JLongArray<'local>, - lengths: JLongArray<'local>, -) -> JObjectArray<'local> { - env.with_env(|env| -> crate::Result<_> { - let count = offsets.len(env)?; - if lengths.len(env)? != count { - return Err(opendal::Error::new( - opendal::ErrorKind::RangeNotSatisfied, - "offsets and lengths must have the same size", - ) - .into()); - } - let mut starts = vec![0; count]; - let mut sizes = vec![0; count]; - offsets.get_region(env, 0, &mut starts)?; - lengths.get_region(env, 0, &mut sizes)?; - let ranges = starts - .into_iter() - .zip(sizes) - .map( - |(offset, length)| match convert::offset_length_to_range(offset, length)? { - (Bound::Included(start), Bound::Excluded(end)) => Ok(start..end), - _ => Err(opendal::Error::new( - opendal::ErrorKind::RangeNotSatisfied, - "fetch requires non-negative lengths", - ) - .into()), - }, - ) - .collect::>>()?; - let reader = unsafe { &*reader }; - let buffers = reader.fetch(ranges)?; - let output = env.new_object_array(count as jsize, jni_str!("[B"), JObject::null())?; - for (index, buffer) in buffers.into_iter().enumerate() { - env.with_local_frame(2, |env| -> crate::Result<()> { - let bytes = convert::bytes_to_jbytearray(env, buffer.to_vec())?; - output.set_element(env, index, &bytes)?; - Ok(()) - })?; - } - Ok(output) - }) - .resolve::() -} - /// # Safety /// /// `reader` must be a live handle allocated by `Operator.reader`, with no calls in progress. diff --git a/bindings/java/src/test/java/org/apache/opendal/test/OperatorReaderTest.java b/bindings/java/src/test/java/org/apache/opendal/test/OperatorReaderTest.java index b2294ce141e8..fd03c1bd6a54 100644 --- a/bindings/java/src/test/java/org/apache/opendal/test/OperatorReaderTest.java +++ b/bindings/java/src/test/java/org/apache/opendal/test/OperatorReaderTest.java @@ -79,9 +79,6 @@ void testInvalidRange(long offset, long length) { .is(OpenDALExceptionCondition.ofSync(OpenDALException.Code.RangeNotSatisfied)); assertThatThrownBy(() -> reader.createInputStream(offset, length)) .is(OpenDALExceptionCondition.ofSync(OpenDALException.Code.RangeNotSatisfied)); - assertThatThrownBy(() -> reader.fetch( - ReadOptions.builder().offset(offset).length(length).build())) - .is(OpenDALExceptionCondition.ofSync(OpenDALException.Code.RangeNotSatisfied)); } } @@ -111,7 +108,6 @@ void testIndependentStreams() throws Exception { reader.close(); op.close(); assertThatThrownBy(reader::createInputStream).isInstanceOf(IllegalStateException.class); - assertThatThrownBy(reader::fetch).isInstanceOf(IllegalStateException.class); assertThat(first.read()).isEqualTo('0'); assertThat(second.read()).isEqualTo('4'); first.close(); @@ -138,16 +134,6 @@ void testZeroLengthStreamReads() { } } - @Test - void testFetchRequiresBoundedRanges() { - try (Operator op = Operator.of( - ServiceConfig.Fs.builder().root(tempDir.toString()).build()); - OperatorReader reader = op.reader("missing")) { - assertThatThrownBy(() -> reader.fetch(ReadOptions.builder().build())) - .is(OpenDALExceptionCondition.ofSync(OpenDALException.Code.RangeNotSatisfied)); - } - } - @Test void testUnsupportedReaderCondition() { try (Operator op = @@ -159,12 +145,9 @@ void testUnsupportedReaderCondition() { } @Test - void testInvalidGapAndTimestamp() { + void testInvalidTimestamp() { try (Operator op = Operator.of(ServiceConfig.Fs.builder().root(tempDir.toString()).build())) { - assertThatThrownBy(() -> - op.reader("missing", ReaderOptions.builder().gap(-2).build())) - .is(OpenDALExceptionCondition.ofSync(OpenDALException.Code.ConfigInvalid)); assertThatThrownBy(() -> op.reader( "missing", ReaderOptions.builder().ifModifiedSince(Instant.MIN).build())) diff --git a/bindings/java/src/test/java/org/apache/opendal/test/ReaderOptionsTest.java b/bindings/java/src/test/java/org/apache/opendal/test/ReaderOptionsTest.java index ac253f657fb4..c12ec792a1f5 100644 --- a/bindings/java/src/test/java/org/apache/opendal/test/ReaderOptionsTest.java +++ b/bindings/java/src/test/java/org/apache/opendal/test/ReaderOptionsTest.java @@ -35,7 +35,6 @@ import org.apache.opendal.Operator; import org.apache.opendal.OperatorInputStream; import org.apache.opendal.OperatorReader; -import org.apache.opendal.ReadOptions; import org.apache.opendal.ReaderOptions; import org.apache.opendal.ServiceConfig; import org.apache.opendal.test.condition.OpenDALExceptionCondition; @@ -43,8 +42,6 @@ import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.Timeout; -import org.junit.jupiter.params.ParameterizedTest; -import org.junit.jupiter.params.provider.CsvSource; /** Verifies Java-to-core option forwarding against recorded local HTTP requests. */ @Timeout(10) @@ -164,31 +161,6 @@ void testVersionConditionsReachReads() { assertThat(urls.get(0).getQuery()).contains("ifGenerationMatch=17", "ifGenerationNotMatch=18"); } - @ParameterizedTest - @CsvSource({"-1, 1", "0, 2", "2, 1"}) - void testFetchGapAndRangeOrdering(long gap, int requestCount) { - try (Operator op = Operator.of( - ServiceConfig.Http.builder().endpoint(endpoint).build()); - OperatorReader reader = - op.reader("file", ReaderOptions.builder().gap(gap).build())) { - byte[][] data = reader.fetch( - ReadOptions.builder().offset(6).length(2).build(), - ReadOptions.builder().offset(0).length(2).build(), - ReadOptions.builder().offset(2).length(2).build(), - ReadOptions.builder().offset(6).length(2).build(), - ReadOptions.builder().offset(4).length(0).build()); - assertThat(data).isDeepEqualTo(new byte[][] { - "67".getBytes(StandardCharsets.UTF_8), - "01".getBytes(StandardCharsets.UTF_8), - "23".getBytes(StandardCharsets.UTF_8), - "67".getBytes(StandardCharsets.UTF_8), - new byte[0] - }); - assertThat(reader.fetch()).isEmpty(); - } - assertThat(requests).hasSize(requestCount); - } - @Test void testConditionalErrorsKeepTheirCode() { try (Operator op = Operator.of( @@ -197,9 +169,6 @@ void testConditionalErrorsKeepTheirCode() { "file", ReaderOptions.builder().ifMatch("changed").build())) { assertThatThrownBy(() -> reader.read(0, 2)) .is(OpenDALExceptionCondition.ofSync(OpenDALException.Code.ConditionNotMatch)); - assertThatThrownBy( - () -> reader.fetch(ReadOptions.builder().length(2).build())) - .is(OpenDALExceptionCondition.ofSync(OpenDALException.Code.ConditionNotMatch)); try (OperatorInputStream in = reader.createInputStream(0, 2)) { assertThatThrownBy(in::read) .is(OpenDALExceptionCondition.ofSync(OpenDALException.Code.ConditionNotMatch)); diff --git a/core/core/src/blocking/read/reader.rs b/core/core/src/blocking/read/reader.rs index 31d21832e921..38d2240edc2d 100644 --- a/core/core/src/blocking/read/reader.rs +++ b/core/core/src/blocking/read/reader.rs @@ -15,8 +15,6 @@ // specific language governing permissions and limitations // under the License. -use std::ops::Range; - use bytes::BufMut; use super::BufferIterator; @@ -61,50 +59,6 @@ impl Reader { self.handle.block_on(inner.read(range)) } - /// Fetch multiple byte ranges into buffers in the same order as the input. - /// - /// Overlapping and adjacent ranges are merged. The reader's - /// [`options::ReaderOptions::gap`] also allows merging ranges separated by - /// at most that many bytes; `0` disables merging across gaps. The reader - /// splits merged requests using `chunk` and executes them with `concurrent` - /// and `prefetch`. Returned buffers contain only the requested bytes and - /// may share their underlying storage. - /// - /// Empty or reversed ranges return empty buffers. An empty input returns - /// an empty vector without reading. A failed request fails the whole call. - /// A request that returns fewer bytes than planned fails with - /// [`ErrorKind::Unexpected`]. The reader's version and conditions apply to - /// every request: a missing file returns [`ErrorKind::NotFound`], a failed - /// condition returns [`ErrorKind::ConditionNotMatch`], and an unsupported - /// condition returns [`ErrorKind::Unsupported`]. - /// - /// # Examples - /// - /// ``` - /// use opendal_core::{Operator, Result, blocking, services}; - /// - /// # fn main() -> Result<()> { - /// let runtime = tokio::runtime::Runtime::new().unwrap(); - /// let op = { - /// let _guard = runtime.enter(); - /// blocking::Operator::new(Operator::new(services::Memory::default())?)? - /// }; - /// op.write("file", "0123456789")?; - /// let reader = op.reader("file")?; - /// let buffers = reader.fetch(vec![6..8, 0..2])?; - /// assert_eq!(buffers[0].to_vec(), b"67"); - /// assert_eq!(buffers[1].to_vec(), b"01"); - /// # Ok(()) - /// # } - /// ``` - pub fn fetch(&self, ranges: Vec>) -> Result> { - let inner = self - .inner - .as_ref() - .ok_or_else(|| Error::new(ErrorKind::Unexpected, "reader has been dropped"))?; - self.handle.block_on(inner.fetch(ranges)) - } - /// /// This operation will copy and write bytes into given [`BufMut`]. Allocation happens while /// [`BufMut`] doesn't have enough space. diff --git a/website/docs/20-bindings/java/04-tasks.md b/website/docs/20-bindings/java/04-tasks.md index 869a8263321c..8205277aff05 100644 --- a/website/docs/20-bindings/java/04-tasks.md +++ b/website/docs/20-bindings/java/04-tasks.md @@ -63,29 +63,9 @@ file, so subsequent reads may observe changes. Version selection and conditions require service support. Unsupported options fail with `Unsupported`; a failed condition on an existing file fails with `ConditionNotMatch`. Conditions apply to all requests through the reader, -including its streams and fetches. Errors may surface when creating the reader +including its streams. Errors may surface when creating the reader or while reading from it. -For multiple bounded ranges, use `fetch`: - -```java -import org.apache.opendal.ReadOptions; - -ReaderOptions options = ReaderOptions.builder().gap(4096).build(); -try (OperatorReader reader = op.reader("path/to/file", options)) { - byte[][] parts = reader.fetch( - ReadOptions.builder().offset(0).length(1024).build(), - ReadOptions.builder().offset(2048).length(1024).build()); -} -``` - -The returned arrays follow input order. Fetch requires non-negative offsets -and lengths; it does not accept `-1` lengths. The `gap` option merges nearby -ranges to reduce requests while excluding gap bytes from the results. Set it -to `0` to disable merging across gaps, or leave it at `-1` for the core default -of 1 MiB. Overlapping and adjacent ranges still merge. `gap` does not affect -single-range reads or streams. - ## Stream a large file Don't load gigabytes into memory — read through an `InputStream` in chunks: From 3fad9ad36b61e4636ef511a5ff5e2d8b953ebc32 Mon Sep 17 00:00:00 2001 From: jihuayu Date: Tue, 15 Sep 2026 11:09:57 +0800 Subject: [PATCH 13/20] refactor(java): simplify reader creation APIs --- bindings/java/README.md | 4 +-- .../java/org/apache/opendal/Operator.java | 33 +++++-------------- .../apache/opendal/OperatorInputStream.java | 19 ++--------- .../org/apache/opendal/OperatorReader.java | 28 +--------------- bindings/java/src/operator_reader.rs | 2 +- .../test/OperatorInputOutputStreamTest.java | 17 +++++----- .../opendal/test/OperatorReaderTest.java | 21 +++++------- .../opendal/test/ReaderOptionsTest.java | 8 ++--- .../test/behavior/BlockingWriteTest.java | 6 ++-- 9 files changed, 40 insertions(+), 98 deletions(-) diff --git a/bindings/java/README.md b/bindings/java/README.md index ce93cbc7e9cb..9b9d479fae98 100644 --- a/bindings/java/README.md +++ b/bindings/java/README.md @@ -94,13 +94,13 @@ Use the synchronous `Operator` for blocking calls, or `AsyncOperator` for ## Reuse a reader -`Operator.reader(path, readerOptions)` creates an `OperatorReader` for repeated +`Operator.createReader(path, readerOptions)` creates an `OperatorReader` for repeated reads. `ReaderOptions` selects versions, conditions, and execution controls; each call selects its own byte range. ```java ReaderOptions options = ReaderOptions.builder().chunk(8 * 1024 * 1024L).build(); -try (OperatorReader reader = op.reader("large.bin", options)) { +try (OperatorReader reader = op.createReader("large.bin", options)) { byte[] first = reader.read(0, 1024); byte[] next = reader.read(1024, 1024); } diff --git a/bindings/java/src/main/java/org/apache/opendal/Operator.java b/bindings/java/src/main/java/org/apache/opendal/Operator.java index 7a6a5d1ed7c7..8f1de909a4c8 100644 --- a/bindings/java/src/main/java/org/apache/opendal/Operator.java +++ b/bindings/java/src/main/java/org/apache/opendal/Operator.java @@ -120,10 +120,10 @@ public byte[] read(String path, ReadOptions options) { * * @param path file path * @return a reader that the caller must close - * @see #reader(String, ReaderOptions) + * @see #createReader(String, ReaderOptions) */ - public OperatorReader reader(String path) { - return reader(path, ReaderOptions.builder().build()); + public OperatorReader createReader(String path) { + return createReader(path, ReaderOptions.builder().build()); } /** @@ -142,40 +142,23 @@ public OperatorReader reader(String path) { * (IsADirectory), or the service does not support reads or a requested option (Unsupported) * @throws IllegalStateException if this operator is closed */ - public OperatorReader reader(String path, ReaderOptions options) { + public OperatorReader createReader(String path, ReaderOptions options) { if (isDisposed()) { throw new IllegalStateException("Operator is closed"); } Objects.requireNonNull(path, "path"); Objects.requireNonNull(options, "options"); - return new OperatorReader(reader(nativeHandle, path, options)); + return new OperatorReader(createReader(nativeHandle, path, options)); } - private static native long reader(long operator, String path, ReaderOptions options); + private static native long createReader(long operator, String path, ReaderOptions options); public OperatorInputStream createInputStream(String path) { - return createInputStream( - path, ReadOptions.builder().build(), ReaderOptions.builder().build()); + return createInputStream(path, ReadOptions.builder().build()); } public OperatorInputStream createInputStream(String path, ReadOptions options) { - return createInputStream(path, options, ReaderOptions.builder().build()); - } - - /** - * Creates a stream over the requested range using the supplied reader execution options. - * The stream ends at the range boundary. Closing it releases its native reader. - * - * @param path object path - * @param readOptions logical offset and length - * @param readerOptions internal chunk request and buffering controls - * @return a stream that the caller must close - * @throws OpenDALException if reader options are invalid (ConfigInvalid) or creation fails - * @see #reader(String, ReaderOptions) - * @see OperatorReader#createInputStream(ReadOptions) - */ - public OperatorInputStream createInputStream(String path, ReadOptions readOptions, ReaderOptions readerOptions) { - return new OperatorInputStream(this, path, readOptions, readerOptions); + return new OperatorInputStream(this, path, options); } public void delete(String path) { diff --git a/bindings/java/src/main/java/org/apache/opendal/OperatorInputStream.java b/bindings/java/src/main/java/org/apache/opendal/OperatorInputStream.java index dfb97c11e890..47895482c083 100644 --- a/bindings/java/src/main/java/org/apache/opendal/OperatorInputStream.java +++ b/bindings/java/src/main/java/org/apache/opendal/OperatorInputStream.java @@ -49,22 +49,9 @@ protected void disposeInternal(long handle) { } public OperatorInputStream(Operator operator, String path, ReadOptions options) { - this(operator, path, options, ReaderOptions.builder().build()); - } - - /** - * Creates a stream with a logical range and independent execution controls. - * - * @param operator operator that reads the object - * @param path object path - * @param readOptions logical offset and length - * @param readerOptions internal chunk request and buffering controls - * @throws OpenDALException if reader options are invalid (ConfigInvalid) or creation fails - */ - public OperatorInputStream(Operator operator, String path, ReadOptions readOptions, ReaderOptions readerOptions) { - Objects.requireNonNull(readOptions, "readOptions"); - try (OperatorReader source = operator.reader(path, readerOptions)) { - this.reader = new BytesIterator(source.createBytesIterator(readOptions.offset, readOptions.length)); + Objects.requireNonNull(options, "options"); + try (OperatorReader source = operator.createReader(path)) { + this.reader = new BytesIterator(source.createBytesIterator(options.offset, options.length)); } } diff --git a/bindings/java/src/main/java/org/apache/opendal/OperatorReader.java b/bindings/java/src/main/java/org/apache/opendal/OperatorReader.java index 7d5ba94cafe6..5c9de1399af7 100644 --- a/bindings/java/src/main/java/org/apache/opendal/OperatorReader.java +++ b/bindings/java/src/main/java/org/apache/opendal/OperatorReader.java @@ -19,8 +19,6 @@ package org.apache.opendal; -import java.util.Objects; - /** * Reads a file synchronously through a reusable Rust core reader. * Each read selects its own range and does not advance a shared cursor. @@ -32,7 +30,7 @@ * Closing the operator that created it does not close the reader. * Calls on one reader are serialized, including close. * - * @see Operator#reader(String, ReaderOptions) + * @see Operator#createReader(String, ReaderOptions) * @see ReaderOptions */ public final class OperatorReader extends NativeObject { @@ -69,18 +67,6 @@ public synchronized byte[] read(long offset, long length) { return readBytes(nativeHandle, offset, length); } - /** - * Reads the range selected by the supplied options. - * - * @param options logical offset and length - * @return contents of the requested range - * @see #read(long, long) - */ - public byte[] read(ReadOptions options) { - Objects.requireNonNull(options, "options"); - return read(options.offset, options.length); - } - /** * Creates an independent stream over the whole file. * @@ -106,18 +92,6 @@ public OperatorInputStream createInputStream(long offset, long length) { return new OperatorInputStream(createBytesIterator(offset, length)); } - /** - * Creates a stream over the range selected by the supplied options. - * - * @param options logical offset and length - * @return a stream that the caller must close - * @see #createInputStream(long, long) - */ - public OperatorInputStream createInputStream(ReadOptions options) { - Objects.requireNonNull(options, "options"); - return createInputStream(options.offset, options.length); - } - // The caller owns the returned iterator independently of this reader. synchronized long createBytesIterator(long offset, long length) { if (isDisposed()) { diff --git a/bindings/java/src/operator_reader.rs b/bindings/java/src/operator_reader.rs index 5d9b1a59b1b7..944461e3adfc 100644 --- a/bindings/java/src/operator_reader.rs +++ b/bindings/java/src/operator_reader.rs @@ -27,7 +27,7 @@ use crate::error::ThrowException; /// /// `op` must point to a live blocking operator for the duration of this call. #[unsafe(no_mangle)] -pub unsafe extern "system" fn Java_org_apache_opendal_Operator_reader<'local>( +pub unsafe extern "system" fn Java_org_apache_opendal_Operator_createReader<'local>( mut env: EnvUnowned<'local>, _: JClass<'local>, op: *const blocking::Operator, diff --git a/bindings/java/src/test/java/org/apache/opendal/test/OperatorInputOutputStreamTest.java b/bindings/java/src/test/java/org/apache/opendal/test/OperatorInputOutputStreamTest.java index c70479f20184..59decae2fb3e 100644 --- a/bindings/java/src/test/java/org/apache/opendal/test/OperatorInputOutputStreamTest.java +++ b/bindings/java/src/test/java/org/apache/opendal/test/OperatorInputOutputStreamTest.java @@ -33,6 +33,7 @@ import org.apache.opendal.Operator; import org.apache.opendal.OperatorInputStream; import org.apache.opendal.OperatorOutputStream; +import org.apache.opendal.OperatorReader; import org.apache.opendal.ReadOptions; import org.apache.opendal.ReaderOptions; import org.apache.opendal.ServiceConfig; @@ -110,8 +111,8 @@ void testChunkedInputStream() throws Exception { .chunk(64 * 1024L) .prefetch(2) .build(); - try (final OperatorInputStream in = - op.createInputStream(path, ReadOptions.builder().build(), options)) { + try (final OperatorReader reader = op.createReader(path, options); + final OperatorInputStream in = reader.createInputStream()) { assertThat(IOUtils.toByteArray(in)).isEqualTo(content); assertThat(in.read()).isEqualTo(-1); } @@ -125,14 +126,14 @@ void testRangeWithReaderOptions() throws Exception { try (final Operator op = Operator.of(fs)) { final String path = "chunked-range.txt"; op.write(path, "0123456789"); - final ReadOptions range = ReadOptions.builder().offset(4).length(5).build(); final ReaderOptions options = ReaderOptions.builder() .concurrent(2) .chunk(2) .prefetch(1) .contentLengthHint(10) .build(); - try (final OperatorInputStream in = op.createInputStream(path, range, options)) { + try (final OperatorReader reader = op.createReader(path, options); + final OperatorInputStream in = reader.createInputStream(4, 5)) { assertThat(IOUtils.toByteArray(in)).isEqualTo("45678".getBytes(StandardCharsets.UTF_8)); assertThat(in.read()).isEqualTo(-1); } @@ -148,8 +149,8 @@ void testEmptyInputStreamWithReaderOptions() throws Exception { op.write(path, new byte[0]); final ReaderOptions options = ReaderOptions.builder().chunk(2).contentLengthHint(0).build(); - try (final OperatorInputStream in = - op.createInputStream(path, ReadOptions.builder().build(), options)) { + try (final OperatorReader reader = op.createReader(path, options); + final OperatorInputStream in = reader.createInputStream()) { assertThat(in.read()).isEqualTo(-1); } } @@ -172,8 +173,8 @@ void testInvalidReaderOptions(ReaderOptions options, String field) { ServiceConfig.Fs.builder().root(tempDir.toString()).build(); try (final Operator op = Operator.of(fs)) { assertThatThrownBy(() -> { - try (final OperatorInputStream in = op.createInputStream( - "invalid-options", ReadOptions.builder().build(), options)) { + try (final OperatorReader reader = op.createReader("invalid-options", options); + final OperatorInputStream in = reader.createInputStream()) { in.read(); } }) diff --git a/bindings/java/src/test/java/org/apache/opendal/test/OperatorReaderTest.java b/bindings/java/src/test/java/org/apache/opendal/test/OperatorReaderTest.java index fd03c1bd6a54..d002825ce0f9 100644 --- a/bindings/java/src/test/java/org/apache/opendal/test/OperatorReaderTest.java +++ b/bindings/java/src/test/java/org/apache/opendal/test/OperatorReaderTest.java @@ -29,7 +29,6 @@ import org.apache.opendal.Operator; import org.apache.opendal.OperatorInputStream; import org.apache.opendal.OperatorReader; -import org.apache.opendal.ReadOptions; import org.apache.opendal.ReaderOptions; import org.apache.opendal.ServiceConfig; import org.apache.opendal.test.condition.OpenDALExceptionCondition; @@ -48,7 +47,7 @@ void testReusableReaderOutlivesOperator() { try (Operator op = Operator.of(ServiceConfig.Fs.builder().root(tempDir.toString()).build())) { op.write("file", "0123456789"); - reader = op.reader( + reader = op.createReader( "file", ReaderOptions.builder() .concurrent(2) @@ -59,8 +58,7 @@ void testReusableReaderOutlivesOperator() { } try (OperatorReader r = reader) { assertThat(r.read(4, 5)).isEqualTo("45678".getBytes(StandardCharsets.UTF_8)); - assertThat(r.read(ReadOptions.builder().offset(1).length(2).build())) - .isEqualTo("12".getBytes(StandardCharsets.UTF_8)); + assertThat(r.read(1, 2)).isEqualTo("12".getBytes(StandardCharsets.UTF_8)); assertThat(r.read(8, -1)).isEqualTo("89".getBytes(StandardCharsets.UTF_8)); assertThat(r.read(0, 0)).isEmpty(); assertThat(r.read()).isEqualTo("0123456789".getBytes(StandardCharsets.UTF_8)); @@ -74,7 +72,7 @@ void testReusableReaderOutlivesOperator() { void testInvalidRange(long offset, long length) { try (Operator op = Operator.of( ServiceConfig.Fs.builder().root(tempDir.toString()).build()); - OperatorReader reader = op.reader("missing")) { + OperatorReader reader = op.createReader("missing")) { assertThatThrownBy(() -> reader.read(offset, length)) .is(OpenDALExceptionCondition.ofSync(OpenDALException.Code.RangeNotSatisfied)); assertThatThrownBy(() -> reader.createInputStream(offset, length)) @@ -86,7 +84,7 @@ void testInvalidRange(long offset, long length) { void testMissingFileFailsOnRead() { try (Operator op = Operator.of( ServiceConfig.Fs.builder().root(tempDir.toString()).build()); - OperatorReader reader = op.reader("missing")) { + OperatorReader reader = op.createReader("missing")) { assertThatThrownBy(reader::read).is(OpenDALExceptionCondition.ofSync(OpenDALException.Code.NotFound)); } } @@ -97,9 +95,8 @@ void testIndependentStreams() throws Exception { Operator.of(ServiceConfig.Fs.builder().root(tempDir.toString()).build())) { op.write("file", "0123456789"); try (OperatorReader reader = - op.reader("file", ReaderOptions.builder().chunk(2).build())) { - try (OperatorInputStream in = reader.createInputStream( - ReadOptions.builder().offset(4).length(3).build())) { + op.createReader("file", ReaderOptions.builder().chunk(2).build())) { + try (OperatorInputStream in = reader.createInputStream(4, 3)) { assertThat(IOUtils.toByteArray(in)).isEqualTo("456".getBytes(StandardCharsets.UTF_8)); } assertThat(reader.read(0, 2)).isEqualTo("01".getBytes(StandardCharsets.UTF_8)); @@ -125,7 +122,7 @@ void testIndependentStreams() throws Exception { void testZeroLengthStreamReads() { try (Operator op = Operator.of( ServiceConfig.Fs.builder().root(tempDir.toString()).build()); - OperatorReader reader = op.reader("missing"); + OperatorReader reader = op.createReader("missing"); OperatorInputStream in = reader.createInputStream(0, 0)) { byte[] bytes = new byte[1]; assertThat(in.read(bytes, 0, 0)).isZero(); @@ -138,7 +135,7 @@ void testZeroLengthStreamReads() { void testUnsupportedReaderCondition() { try (Operator op = Operator.of(ServiceConfig.Fs.builder().root(tempDir.toString()).build())) { - assertThatThrownBy(() -> op.reader( + assertThatThrownBy(() -> op.createReader( "missing", ReaderOptions.builder().ifMatch("etag").build())) .is(OpenDALExceptionCondition.ofSync(OpenDALException.Code.Unsupported)); } @@ -148,7 +145,7 @@ void testUnsupportedReaderCondition() { void testInvalidTimestamp() { try (Operator op = Operator.of(ServiceConfig.Fs.builder().root(tempDir.toString()).build())) { - assertThatThrownBy(() -> op.reader( + assertThatThrownBy(() -> op.createReader( "missing", ReaderOptions.builder().ifModifiedSince(Instant.MIN).build())) .is(OpenDALExceptionCondition.ofSync(OpenDALException.Code.Unexpected)); diff --git a/bindings/java/src/test/java/org/apache/opendal/test/ReaderOptionsTest.java b/bindings/java/src/test/java/org/apache/opendal/test/ReaderOptionsTest.java index c12ec792a1f5..418089a3b6e6 100644 --- a/bindings/java/src/test/java/org/apache/opendal/test/ReaderOptionsTest.java +++ b/bindings/java/src/test/java/org/apache/opendal/test/ReaderOptionsTest.java @@ -105,7 +105,7 @@ void testConditionsReachEveryChunkAndStream() throws Exception { .build(); try (Operator op = Operator.of( ServiceConfig.Http.builder().endpoint(endpoint).build()); - OperatorReader reader = op.reader("file", options)) { + OperatorReader reader = op.createReader("file", options)) { assertThat(reader.read(0, 4)).isEqualTo("0123".getBytes(StandardCharsets.UTF_8)); try (OperatorInputStream in = reader.createInputStream(4, 4)) { assertThat(IOUtils.toByteArray(in)).isEqualTo("4567".getBytes(StandardCharsets.UTF_8)); @@ -129,7 +129,7 @@ void testVersionReachesReadsAndStreams() throws Exception { .endpoint(endpoint) .skipSignature(true) .build()); - OperatorReader reader = op.reader( + OperatorReader reader = op.createReader( "file", ReaderOptions.builder().version("version-one").build())) { assertThat(reader.read(0, 2)).isEqualTo("01".getBytes(StandardCharsets.UTF_8)); try (OperatorInputStream in = reader.createInputStream(4, 2)) { @@ -149,7 +149,7 @@ void testVersionConditionsReachReads() { .endpoint(endpoint) .skipSignature(true) .build()); - OperatorReader reader = op.reader( + OperatorReader reader = op.createReader( "file", ReaderOptions.builder() .ifVersionMatch("17") @@ -165,7 +165,7 @@ void testVersionConditionsReachReads() { void testConditionalErrorsKeepTheirCode() { try (Operator op = Operator.of( ServiceConfig.Http.builder().endpoint(endpoint).build()); - OperatorReader reader = op.reader( + OperatorReader reader = op.createReader( "file", ReaderOptions.builder().ifMatch("changed").build())) { assertThatThrownBy(() -> reader.read(0, 2)) .is(OpenDALExceptionCondition.ofSync(OpenDALException.Code.ConditionNotMatch)); diff --git a/bindings/java/src/test/java/org/apache/opendal/test/behavior/BlockingWriteTest.java b/bindings/java/src/test/java/org/apache/opendal/test/behavior/BlockingWriteTest.java index 7b2296e0c84c..50e148285b33 100644 --- a/bindings/java/src/test/java/org/apache/opendal/test/behavior/BlockingWriteTest.java +++ b/bindings/java/src/test/java/org/apache/opendal/test/behavior/BlockingWriteTest.java @@ -29,7 +29,7 @@ import org.apache.opendal.Metadata; import org.apache.opendal.OpenDALException; import org.apache.opendal.OperatorInputStream; -import org.apache.opendal.ReadOptions; +import org.apache.opendal.OperatorReader; import org.apache.opendal.ReaderOptions; import org.apache.opendal.test.condition.OpenDALExceptionCondition; import org.junit.jupiter.api.BeforeAll; @@ -79,8 +79,8 @@ public void testBlockingInputStreamWithReaderOptions() throws Exception { .chunk(256 * 1024L) .prefetch(2) .build(); - try (final OperatorInputStream in = - op().createInputStream(path, ReadOptions.builder().build(), options)) { + try (final OperatorReader reader = op().createReader(path, options); + final OperatorInputStream in = reader.createInputStream()) { assertThat(IOUtils.toByteArray(in)).isEqualTo(content); assertThat(in.read()).isEqualTo(-1); } From 061db7e083515caff7c7af31897f7d7c2305889c Mon Sep 17 00:00:00 2001 From: jihuayu Date: Tue, 15 Sep 2026 11:17:25 +0800 Subject: [PATCH 14/20] refactor(java): defer unrelated input stream behavior changes --- .../org/apache/opendal/OperatorInputStream.java | 12 ++++++++---- .../org/apache/opendal/test/OperatorReaderTest.java | 13 ------------- 2 files changed, 8 insertions(+), 17 deletions(-) diff --git a/bindings/java/src/main/java/org/apache/opendal/OperatorInputStream.java b/bindings/java/src/main/java/org/apache/opendal/OperatorInputStream.java index 47895482c083..1294dd5898e4 100644 --- a/bindings/java/src/main/java/org/apache/opendal/OperatorInputStream.java +++ b/bindings/java/src/main/java/org/apache/opendal/OperatorInputStream.java @@ -60,7 +60,7 @@ public synchronized int read() { if (reader.isDisposed()) { throw new IllegalStateException("OperatorInputStream is closed"); } - while (bytes != null && offset >= bytes.length) { + if (bytes != null && offset >= bytes.length) { bytes = readNextBytes(reader.nativeHandle); offset = 0; } @@ -85,7 +85,7 @@ public synchronized int read(byte[] b, int off, int len) { } int read = 0; while (len > 0) { - while (bytes != null && offset >= bytes.length) { + if (bytes != null && offset >= bytes.length) { bytes = readNextBytes(reader.nativeHandle); offset = 0; } @@ -102,13 +102,17 @@ public synchronized int read(byte[] b, int off, int len) { len -= n; } - return read; + if (bytes != null && offset >= bytes.length) { + bytes = readNextBytes(reader.nativeHandle); + offset = 0; + } + + return bytes != null ? read : (read != 0 ? read : -1); } @Override public synchronized void close() { reader.close(); - bytes = null; } private static native void disposeReader(long reader); diff --git a/bindings/java/src/test/java/org/apache/opendal/test/OperatorReaderTest.java b/bindings/java/src/test/java/org/apache/opendal/test/OperatorReaderTest.java index d002825ce0f9..d42ebdf05358 100644 --- a/bindings/java/src/test/java/org/apache/opendal/test/OperatorReaderTest.java +++ b/bindings/java/src/test/java/org/apache/opendal/test/OperatorReaderTest.java @@ -118,19 +118,6 @@ void testIndependentStreams() throws Exception { } } - @Test - void testZeroLengthStreamReads() { - try (Operator op = Operator.of( - ServiceConfig.Fs.builder().root(tempDir.toString()).build()); - OperatorReader reader = op.createReader("missing"); - OperatorInputStream in = reader.createInputStream(0, 0)) { - byte[] bytes = new byte[1]; - assertThat(in.read(bytes, 0, 0)).isZero(); - assertThat(in.read()).isEqualTo(-1); - assertThat(in.read(bytes, 0, 0)).isZero(); - } - } - @Test void testUnsupportedReaderCondition() { try (Operator op = From f58960e0e623dde46b314c42ee0e1f70c33994ce Mon Sep 17 00:00:00 2001 From: jihuayu Date: Tue, 15 Sep 2026 11:23:08 +0800 Subject: [PATCH 15/20] refactor(java): clarify native iterator handle naming --- .../java/org/apache/opendal/OperatorInputStream.java | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/bindings/java/src/main/java/org/apache/opendal/OperatorInputStream.java b/bindings/java/src/main/java/org/apache/opendal/OperatorInputStream.java index 1294dd5898e4..de306c350adf 100644 --- a/bindings/java/src/main/java/org/apache/opendal/OperatorInputStream.java +++ b/bindings/java/src/main/java/org/apache/opendal/OperatorInputStream.java @@ -28,8 +28,8 @@ * Reading a closed stream throws {@link IllegalStateException}. */ public class OperatorInputStream extends InputStream { - private static class BytesIterator extends NativeObject { - private BytesIterator(long nativeHandle) { + private static class NativeIteratorHandle extends NativeObject { + private NativeIteratorHandle(long nativeHandle) { super(nativeHandle); } @@ -39,19 +39,19 @@ protected void disposeInternal(long handle) { } } - private final BytesIterator reader; + private final NativeIteratorHandle reader; private int offset = 0; private byte[] bytes = new byte[0]; OperatorInputStream(long nativeHandle) { - this.reader = new BytesIterator(nativeHandle); + this.reader = new NativeIteratorHandle(nativeHandle); } public OperatorInputStream(Operator operator, String path, ReadOptions options) { Objects.requireNonNull(options, "options"); try (OperatorReader source = operator.createReader(path)) { - this.reader = new BytesIterator(source.createBytesIterator(options.offset, options.length)); + this.reader = new NativeIteratorHandle(source.createBytesIterator(options.offset, options.length)); } } From c945b6ff62b4418080be26a6d715a508d1a7f67d Mon Sep 17 00:00:00 2001 From: jihuayu Date: Tue, 15 Sep 2026 11:37:00 +0800 Subject: [PATCH 16/20] test(java): trim redundant reader stream coverage --- .../test/OperatorInputOutputStreamTest.java | 33 ++----------------- 1 file changed, 3 insertions(+), 30 deletions(-) diff --git a/bindings/java/src/test/java/org/apache/opendal/test/OperatorInputOutputStreamTest.java b/bindings/java/src/test/java/org/apache/opendal/test/OperatorInputOutputStreamTest.java index 59decae2fb3e..2cb903a0558a 100644 --- a/bindings/java/src/test/java/org/apache/opendal/test/OperatorInputOutputStreamTest.java +++ b/bindings/java/src/test/java/org/apache/opendal/test/OperatorInputOutputStreamTest.java @@ -23,7 +23,6 @@ import static org.assertj.core.api.Assertions.assertThatThrownBy; import java.io.BufferedReader; import java.io.InputStreamReader; -import java.nio.charset.StandardCharsets; import java.nio.file.Path; import java.util.Random; import java.util.concurrent.atomic.AtomicLong; @@ -99,7 +98,7 @@ void testCreateInputStreamWithOptions() { @Test void testChunkedInputStream() throws Exception { - final byte[] content = new byte[4 * 1024 * 1024 + 13]; + final byte[] content = new byte[4 * 1024 + 13]; new Random(8252).nextBytes(content); final ServiceConfig.Fs fs = ServiceConfig.Fs.builder().root(tempDir.toString()).build(); @@ -108,7 +107,7 @@ void testChunkedInputStream() throws Exception { op.write(path, content); final ReaderOptions options = ReaderOptions.builder() .concurrent(4) - .chunk(64 * 1024L) + .chunk(1024) .prefetch(2) .build(); try (final OperatorReader reader = op.createReader(path, options); @@ -119,27 +118,6 @@ void testChunkedInputStream() throws Exception { } } - @Test - void testRangeWithReaderOptions() throws Exception { - final ServiceConfig.Fs fs = - ServiceConfig.Fs.builder().root(tempDir.toString()).build(); - try (final Operator op = Operator.of(fs)) { - final String path = "chunked-range.txt"; - op.write(path, "0123456789"); - final ReaderOptions options = ReaderOptions.builder() - .concurrent(2) - .chunk(2) - .prefetch(1) - .contentLengthHint(10) - .build(); - try (final OperatorReader reader = op.createReader(path, options); - final OperatorInputStream in = reader.createInputStream(4, 5)) { - assertThat(IOUtils.toByteArray(in)).isEqualTo("45678".getBytes(StandardCharsets.UTF_8)); - assertThat(in.read()).isEqualTo(-1); - } - } - } - @Test void testEmptyInputStreamWithReaderOptions() throws Exception { final ServiceConfig.Fs fs = @@ -172,12 +150,7 @@ void testInvalidReaderOptions(ReaderOptions options, String field) { final ServiceConfig.Fs fs = ServiceConfig.Fs.builder().root(tempDir.toString()).build(); try (final Operator op = Operator.of(fs)) { - assertThatThrownBy(() -> { - try (final OperatorReader reader = op.createReader("invalid-options", options); - final OperatorInputStream in = reader.createInputStream()) { - in.read(); - } - }) + assertThatThrownBy(() -> op.createReader("invalid-options", options)) .is(OpenDALExceptionCondition.ofSync(OpenDALException.Code.ConfigInvalid)) .hasMessageContaining(field); } From 79b958dbe8a2aadeea5ae9b65fc8152a6876951a Mon Sep 17 00:00:00 2001 From: jihuayu Date: Tue, 15 Sep 2026 11:37:44 +0800 Subject: [PATCH 17/20] refactor(java): align native iterator handle names --- .../apache/opendal/OperatorInputStream.java | 24 +++++++++---------- bindings/java/src/operator_input_stream.rs | 18 +++++++------- 2 files changed, 22 insertions(+), 20 deletions(-) diff --git a/bindings/java/src/main/java/org/apache/opendal/OperatorInputStream.java b/bindings/java/src/main/java/org/apache/opendal/OperatorInputStream.java index de306c350adf..15bbc37ce0f5 100644 --- a/bindings/java/src/main/java/org/apache/opendal/OperatorInputStream.java +++ b/bindings/java/src/main/java/org/apache/opendal/OperatorInputStream.java @@ -35,33 +35,33 @@ private NativeIteratorHandle(long nativeHandle) { @Override protected void disposeInternal(long handle) { - disposeReader(handle); + disposeIterator(handle); } } - private final NativeIteratorHandle reader; + private final NativeIteratorHandle iteratorHandle; private int offset = 0; private byte[] bytes = new byte[0]; OperatorInputStream(long nativeHandle) { - this.reader = new NativeIteratorHandle(nativeHandle); + this.iteratorHandle = new NativeIteratorHandle(nativeHandle); } public OperatorInputStream(Operator operator, String path, ReadOptions options) { Objects.requireNonNull(options, "options"); try (OperatorReader source = operator.createReader(path)) { - this.reader = new NativeIteratorHandle(source.createBytesIterator(options.offset, options.length)); + this.iteratorHandle = new NativeIteratorHandle(source.createBytesIterator(options.offset, options.length)); } } @Override public synchronized int read() { - if (reader.isDisposed()) { + if (iteratorHandle.isDisposed()) { throw new IllegalStateException("OperatorInputStream is closed"); } if (bytes != null && offset >= bytes.length) { - bytes = readNextBytes(reader.nativeHandle); + bytes = readNextBytes(iteratorHandle.nativeHandle); offset = 0; } @@ -80,13 +80,13 @@ public synchronized int read(byte[] b, int off, int len) { throw new IndexOutOfBoundsException( String.format("Range [%s, % 0) { if (bytes != null && offset >= bytes.length) { - bytes = readNextBytes(reader.nativeHandle); + bytes = readNextBytes(iteratorHandle.nativeHandle); offset = 0; } @@ -103,7 +103,7 @@ public synchronized int read(byte[] b, int off, int len) { } if (bytes != null && offset >= bytes.length) { - bytes = readNextBytes(reader.nativeHandle); + bytes = readNextBytes(iteratorHandle.nativeHandle); offset = 0; } @@ -112,10 +112,10 @@ public synchronized int read(byte[] b, int off, int len) { @Override public synchronized void close() { - reader.close(); + iteratorHandle.close(); } - private static native void disposeReader(long reader); + private static native void disposeIterator(long iteratorHandle); - private static native byte[] readNextBytes(long reader); + private static native byte[] readNextBytes(long iteratorHandle); } diff --git a/bindings/java/src/operator_input_stream.rs b/bindings/java/src/operator_input_stream.rs index ebe10245f67e..e03761795ab3 100644 --- a/bindings/java/src/operator_input_stream.rs +++ b/bindings/java/src/operator_input_stream.rs @@ -24,30 +24,32 @@ use crate::error::ThrowException; /// # Safety /// -/// `reader` must point to a live iterator, with no other calls in progress. +/// `iterator` must point to a live iterator, with no other calls in progress. #[unsafe(no_mangle)] -pub unsafe extern "system" fn Java_org_apache_opendal_OperatorInputStream_disposeReader<'local>( +pub unsafe extern "system" fn Java_org_apache_opendal_OperatorInputStream_disposeIterator< + 'local, +>( _: EnvUnowned<'local>, _: JClass<'local>, - reader: *mut StdBytesIterator, + iterator: *mut StdBytesIterator, ) { unsafe { - drop(Box::from_raw(reader)); + drop(Box::from_raw(iterator)); } } /// # Safety /// -/// `reader` must point to a live iterator, with no other calls in progress. +/// `iterator` must point to a live iterator, with no other calls in progress. #[unsafe(no_mangle)] pub unsafe extern "system" fn Java_org_apache_opendal_OperatorInputStream_readNextBytes<'local>( mut env: EnvUnowned<'local>, _: JClass<'local>, - reader: *mut StdBytesIterator, + iterator: *mut StdBytesIterator, ) -> JByteArray<'local> { env.with_env(|env| -> crate::Result<_> { - let reader = unsafe { &mut *reader }; - match reader.next().transpose().map_err(|err| { + let iterator = unsafe { &mut *iterator }; + match iterator.next().transpose().map_err(|err| { err.downcast::().unwrap_or_else(|err| { opendal::Error::new(opendal::ErrorKind::Unexpected, err.to_string()) }) From e7c92c5df870a7ba49c2e759244df83affb929ca Mon Sep 17 00:00:00 2001 From: jihuayu Date: Tue, 15 Sep 2026 17:23:35 +0800 Subject: [PATCH 18/20] refactor(java): simplify native iterator naming --- .../apache/opendal/OperatorInputStream.java | 26 +++++++++---------- 1 file changed, 13 insertions(+), 13 deletions(-) diff --git a/bindings/java/src/main/java/org/apache/opendal/OperatorInputStream.java b/bindings/java/src/main/java/org/apache/opendal/OperatorInputStream.java index 15bbc37ce0f5..15ab89d0ba6a 100644 --- a/bindings/java/src/main/java/org/apache/opendal/OperatorInputStream.java +++ b/bindings/java/src/main/java/org/apache/opendal/OperatorInputStream.java @@ -28,8 +28,8 @@ * Reading a closed stream throws {@link IllegalStateException}. */ public class OperatorInputStream extends InputStream { - private static class NativeIteratorHandle extends NativeObject { - private NativeIteratorHandle(long nativeHandle) { + private static class NativeIterator extends NativeObject { + private NativeIterator(long nativeHandle) { super(nativeHandle); } @@ -39,29 +39,29 @@ protected void disposeInternal(long handle) { } } - private final NativeIteratorHandle iteratorHandle; + private final NativeIterator iterator; private int offset = 0; private byte[] bytes = new byte[0]; OperatorInputStream(long nativeHandle) { - this.iteratorHandle = new NativeIteratorHandle(nativeHandle); + this.iterator = new NativeIterator(nativeHandle); } public OperatorInputStream(Operator operator, String path, ReadOptions options) { Objects.requireNonNull(options, "options"); try (OperatorReader source = operator.createReader(path)) { - this.iteratorHandle = new NativeIteratorHandle(source.createBytesIterator(options.offset, options.length)); + this.iterator = new NativeIterator(source.createBytesIterator(options.offset, options.length)); } } @Override public synchronized int read() { - if (iteratorHandle.isDisposed()) { + if (iterator.isDisposed()) { throw new IllegalStateException("OperatorInputStream is closed"); } if (bytes != null && offset >= bytes.length) { - bytes = readNextBytes(iteratorHandle.nativeHandle); + bytes = readNextBytes(iterator.nativeHandle); offset = 0; } @@ -80,13 +80,13 @@ public synchronized int read(byte[] b, int off, int len) { throw new IndexOutOfBoundsException( String.format("Range [%s, % 0) { if (bytes != null && offset >= bytes.length) { - bytes = readNextBytes(iteratorHandle.nativeHandle); + bytes = readNextBytes(iterator.nativeHandle); offset = 0; } @@ -103,7 +103,7 @@ public synchronized int read(byte[] b, int off, int len) { } if (bytes != null && offset >= bytes.length) { - bytes = readNextBytes(iteratorHandle.nativeHandle); + bytes = readNextBytes(iterator.nativeHandle); offset = 0; } @@ -112,10 +112,10 @@ public synchronized int read(byte[] b, int off, int len) { @Override public synchronized void close() { - iteratorHandle.close(); + iterator.close(); } - private static native void disposeIterator(long iteratorHandle); + private static native void disposeIterator(long nativeHandle); - private static native byte[] readNextBytes(long iteratorHandle); + private static native byte[] readNextBytes(long nativeHandle); } From ab9ef727108df7a8af67439eb060dc641c58a635 Mon Sep 17 00:00:00 2001 From: jihuayu Date: Tue, 15 Sep 2026 17:25:09 +0800 Subject: [PATCH 19/20] docs(java): correct reader creation example --- website/docs/20-bindings/java/04-tasks.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/website/docs/20-bindings/java/04-tasks.md b/website/docs/20-bindings/java/04-tasks.md index 8205277aff05..46ec60442456 100644 --- a/website/docs/20-bindings/java/04-tasks.md +++ b/website/docs/20-bindings/java/04-tasks.md @@ -46,7 +46,7 @@ ReaderOptions options = ReaderOptions.builder() .concurrent(4) .prefetch(2) .build(); -try (OperatorReader reader = op.reader("path/to/file", options)) { +try (OperatorReader reader = op.createReader("path/to/file", options)) { byte[] first = reader.read(0, 1024); byte[] next = reader.read(1024, 1024); } From 80af3fe7275eee0a27bcdb04e91b262ce93396fa Mon Sep 17 00:00:00 2001 From: jihuayu Date: Tue, 15 Sep 2026 17:29:27 +0800 Subject: [PATCH 20/20] refactor(java): inline reader option conversion --- bindings/java/src/lib.rs | 103 ++++++--------------------- bindings/java/src/operator_reader.rs | 2 +- 2 files changed, 24 insertions(+), 81 deletions(-) diff --git a/bindings/java/src/lib.rs b/bindings/java/src/lib.rs index eaa5fda63cd3..d552a95a6dda 100644 --- a/bindings/java/src/lib.rs +++ b/bindings/java/src/lib.rs @@ -297,44 +297,13 @@ fn make_reader_options( env: &mut Env, options: &JObject, ) -> Result { - Ok(opendal::options::ReaderOptions { - version: convert::read_string_field(env, options, "version")?, - if_match: convert::read_string_field(env, options, "ifMatch")?, - if_none_match: convert::read_string_field(env, options, "ifNoneMatch")?, - if_version_match: convert::read_string_field(env, options, "ifVersionMatch")?, - if_version_not_match: convert::read_string_field(env, options, "ifVersionNotMatch")?, - if_modified_since: convert::read_instant_field_to_timestamp( - env, - options, - "ifModifiedSince", - )?, - if_unmodified_since: convert::read_instant_field_to_timestamp( - env, - options, - "ifUnmodifiedSince", - )?, - ..build_reader_options( - convert::read_int_field(env, options, "concurrent")?, - convert::read_int64_field(env, options, "chunk")?, - convert::read_int_field(env, options, "prefetch")?, - convert::read_int64_field(env, options, "contentLengthHint")?, - )? - }) -} - -fn build_reader_options( - concurrent: i32, - chunk: i64, - prefetch: i32, - content_length_hint: i64, -) -> opendal::Result { - use opendal::{Error, ErrorKind}; + let concurrent = convert::read_int_field(env, options, "concurrent")?; + let chunk = convert::read_int64_field(env, options, "chunk")?; + let prefetch = convert::read_int_field(env, options, "prefetch")?; + let content_length_hint = convert::read_int64_field(env, options, "contentLengthHint")?; if concurrent <= 0 { - return Err(Error::new( - ErrorKind::ConfigInvalid, - "concurrent must be positive", - )); + return Err(Error::new(ErrorKind::ConfigInvalid, "concurrent must be positive").into()); } let concurrent = usize::try_from(concurrent) .map_err(|_| Error::new(ErrorKind::ConfigInvalid, "concurrent is too large"))?; @@ -345,10 +314,9 @@ fn build_reader_options( .map_err(|_| Error::new(ErrorKind::ConfigInvalid, "chunk is too large"))?, ), _ => { - return Err(Error::new( - ErrorKind::ConfigInvalid, - "chunk must be -1 or positive", - )); + return Err( + Error::new(ErrorKind::ConfigInvalid, "chunk must be -1 or positive").into(), + ); } }; let prefetch = usize::try_from(prefetch) @@ -364,6 +332,21 @@ fn build_reader_options( }; Ok(opendal::options::ReaderOptions { + version: convert::read_string_field(env, options, "version")?, + if_match: convert::read_string_field(env, options, "ifMatch")?, + if_none_match: convert::read_string_field(env, options, "ifNoneMatch")?, + if_version_match: convert::read_string_field(env, options, "ifVersionMatch")?, + if_version_not_match: convert::read_string_field(env, options, "ifVersionNotMatch")?, + if_modified_since: convert::read_instant_field_to_timestamp( + env, + options, + "ifModifiedSince", + )?, + if_unmodified_since: convert::read_instant_field_to_timestamp( + env, + options, + "ifUnmodifiedSince", + )?, concurrent, chunk, prefetch, @@ -371,43 +354,3 @@ fn build_reader_options( ..Default::default() }) } - -#[cfg(test)] -mod reader_options_tests { - use super::build_reader_options; - use opendal::ErrorKind; - - #[test] - fn default_reader_options() { - let options = build_reader_options(1, -1, 0, -1).unwrap(); - assert_eq!(options.concurrent, 1); - assert_eq!(options.chunk, None); - assert_eq!(options.prefetch, 0); - assert_eq!(options.content_length_hint, None); - } - - #[test] - fn tuned_reader_options() { - let options = build_reader_options(4, 8 * 1024 * 1024, 2, 128 * 1024 * 1024).unwrap(); - assert_eq!(options.concurrent, 4); - assert_eq!(options.chunk, Some(8 * 1024 * 1024)); - assert_eq!(options.prefetch, 2); - assert_eq!(options.content_length_hint, Some(128 * 1024 * 1024)); - } - - #[test] - fn empty_content_length_hint() { - let options = build_reader_options(1, 1, 0, 0).unwrap(); - assert_eq!(options.content_length_hint, Some(0)); - } - - #[test] - fn chunk_conversion_respects_native_width() { - let options = build_reader_options(1, i64::MAX, 0, -1); - if usize::BITS < 64 { - assert_eq!(options.unwrap_err().kind(), ErrorKind::ConfigInvalid); - } else { - assert_eq!(options.unwrap().chunk, usize::try_from(i64::MAX).ok()); - } - } -} diff --git a/bindings/java/src/operator_reader.rs b/bindings/java/src/operator_reader.rs index 944461e3adfc..cd2e2e8cdc3d 100644 --- a/bindings/java/src/operator_reader.rs +++ b/bindings/java/src/operator_reader.rs @@ -86,7 +86,7 @@ pub unsafe extern "system" fn Java_org_apache_opendal_OperatorReader_createBytes /// # Safety /// -/// `reader` must be a live handle allocated by `Operator.reader`, with no calls in progress. +/// `reader` must be a live handle allocated by `Operator.createReader`, with no calls in progress. /// It must not be used after this call. #[unsafe(no_mangle)] pub unsafe extern "system" fn Java_org_apache_opendal_OperatorReader_disposeReader<'local>(