Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
9 changes: 9 additions & 0 deletions CHANGELOG.md
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,15 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0

## [Unreleased]

### Added

- `ParquetExporter` (`parquet` module): writes a Vortex file to Parquet, the inverse of `ParquetImporter`. Flat schemas only — `Bool`, non-`F16` `Primitive`, `Utf8`, `Binary`, and `vortex.timestamp` (MILLIS/MICROS/NANOS); `Struct`/`List`/`Map` top-level columns throw `UnsupportedOperationException`. Built on Hardwood 1.1's new `ColumnWriter`/`ParquetFileWriter` write API. The CLI's `export` subcommand now dispatches to it when the output path ends `.parquet` (previously CSV-only). ([#362](https://github.com/dfa1/vortex-java/pull/362))
- `ParquetExporter.exportParquet(VortexHandle, Path[, ExportOptions])` and `ParquetImporter.importParquet(URI, Path[, ImportOptions])`: both directions now work against a remote source over HTTP(S) — export from an already-open `VortexHttpReader` handle (no intervening local copy), import from a Parquet file served over HTTP(S) (new `HttpInputFile`, fetched entirely through targeted Range requests, mirroring `VortexHttpReader`'s own range-fetch pattern). The CLI's `export`/`import` subcommands accept a `url` source too, Parquet-only in both directions. ([#362](https://github.com/dfa1/vortex-java/pull/362))

### Changed

- `dev.hardwood:hardwood-core` 1.0.0.Final → 1.1.0.Beta1: faster `DELTA_BINARY_PACKED` and dictionary-index decoding plus a fixed-length `LIST` fast path speed up `ParquetImporter`'s read path; its new Parquet write support now backs `ParquetExporter`. ([e66fb6eb](https://github.com/dfa1/vortex-java/commit/e66fb6eb))

## [0.13.4] — 2026-09-01

### Fixed
Expand Down
80 changes: 66 additions & 14 deletions cli/src/main/java/io/github/dfa1/vortex/cli/ExportCommand.java
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,8 @@
import io.github.dfa1.vortex.csv.CsvExporter;
import io.github.dfa1.vortex.inspect.ByteSize;
import io.github.dfa1.vortex.csv.ExportOptions;
import io.github.dfa1.vortex.parquet.ParquetExporter;
import io.github.dfa1.vortex.reader.VortexHandle;

import java.io.IOException;
import java.io.OutputStreamWriter;
Expand All @@ -19,31 +21,53 @@ private ExportCommand() {

static int run(String[] args) {
if (args.length < 2 || args.length > 3) {
System.err.println("usage: export <file.vortex> [out.csv | -]");
System.err.println("usage: export <file.vortex|url> [out.csv | out.parquet | -]");
return ExitStatus.USAGE_ERROR;
}
Path inputPath = Path.of(args[1]);
String target = args[1];
boolean toStdout = args.length == 3 && "-".equals(args[2]);
boolean remote = target.startsWith("http://") || target.startsWith("https://");
if (remote) {
return runRemote(target, args, toStdout);
}
Path inputPath = Path.of(target);
if (!Files.exists(inputPath)) {
System.err.println("file not found: " + inputPath);
return ExitStatus.FILE_NOT_FOUND;
}
boolean toStdout = args.length == 3 && "-".equals(args[2]);
Path outputPath = (args.length == 3 && !toStdout)
? Path.of(args[2])
: deriveOutputPath(inputPath);
try {
ExportOptions options = ExportOptions.defaults()
.withProgressListener(ProgressBar::render);
if (toStdout) {
Writer stdout = new OutputStreamWriter(System.out, StandardCharsets.UTF_8);
CsvExporter.exportCsv(inputPath, stdout, options);
stdout.flush();
ProgressBar.clear();
} else {
CsvExporter.exportCsv(inputPath, outputPath, options);
ProgressBar.clear();
printResult(inputPath, outputPath);
if (!toStdout && outputPath.getFileName().toString().endsWith(".parquet")) {
return runParquet(inputPath, outputPath);
}
return runCsv(inputPath, outputPath, toStdout);
} catch (IOException e) {
ProgressBar.clear();
System.err.println("error: " + e.getMessage());
return ExitStatus.ERROR;
}
}

/// Handles an `http(s)://` source: Parquet output only (an explicit `out.parquet` path is
/// required — CSV export and stdout streaming from a remote source aren't supported yet).
private static int runRemote(String target, String[] args, boolean toStdout) {
if (toStdout || args.length != 3 || !args[2].endsWith(".parquet")) {
System.err.println("usage: export <url> out.parquet (CSV/stdout export from a URL isn't supported yet)");
return ExitStatus.USAGE_ERROR;
}
Path outputPath = Path.of(args[2]);
try (VortexHandle handle = CliHandles.openTarget(target)) {
if (handle == null) {
return ExitStatus.FILE_NOT_FOUND;
}
io.github.dfa1.vortex.parquet.ExportOptions options =
io.github.dfa1.vortex.parquet.ExportOptions.defaults()
.withProgressListener(ProgressBar::render);
ParquetExporter.exportParquet(handle, outputPath, options);
ProgressBar.clear();
System.out.printf("written: %s (%s)%n", outputPath, ByteSize.format(Files.size(outputPath)));
return ExitStatus.OK;
} catch (IOException e) {
ProgressBar.clear();
Expand All @@ -52,6 +76,34 @@ static int run(String[] args) {
}
}

private static int runCsv(Path inputPath, Path outputPath, boolean toStdout) throws IOException {
ExportOptions options = ExportOptions.defaults()
.withProgressListener(ProgressBar::render);
if (toStdout) {
Writer stdout = new OutputStreamWriter(System.out, StandardCharsets.UTF_8);
CsvExporter.exportCsv(inputPath, stdout, options);
stdout.flush();
ProgressBar.clear();
} else {
CsvExporter.exportCsv(inputPath, outputPath, options);
ProgressBar.clear();
printResult(inputPath, outputPath);
}
return ExitStatus.OK;
}

private static int runParquet(Path inputPath, Path outputPath) throws IOException {
io.github.dfa1.vortex.parquet.ExportOptions options =
io.github.dfa1.vortex.parquet.ExportOptions.defaults()
.withProgressListener(ProgressBar::render);
ParquetExporter.exportParquet(inputPath, outputPath, options);
ProgressBar.clear();
printResult(inputPath, outputPath);
return ExitStatus.OK;
}

/// Defaults to `.csv` — a Parquet destination must be named explicitly (`out.parquet`),
/// matching [ExportCommand#run]'s extension-on-the-output-path dispatch.
private static Path deriveOutputPath(Path inputPath) {
String name = inputPath.getFileName().toString();
if (name.endsWith(".vortex")) {
Expand Down
71 changes: 56 additions & 15 deletions cli/src/main/java/io/github/dfa1/vortex/cli/ImportCommand.java
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,7 @@
import io.github.dfa1.vortex.parquet.ParquetImporter;

import java.io.IOException;
import java.net.URI;
import java.nio.file.Files;
import java.nio.file.Path;
import java.util.ArrayList;
Expand All @@ -14,7 +15,10 @@
@SuppressWarnings("java:S106")
final class ImportCommand {

private record ParsedArgs(Path inputPath, Path outputPath, Character delimiter) {
/// `outputTarget` is the raw second positional argument, or `null` when omitted — kept
/// unresolved here because deriving a default depends on whether `inputTarget` turns out to
/// be a local path or a URL, decided in [#run].
private record ParsedArgs(String inputTarget, String outputTarget, Character delimiter) {
}

private ImportCommand() {
Expand All @@ -26,20 +30,28 @@ static int run(String[] args) {
parsedArgs = parseArgs(args);
} catch (IllegalArgumentException e) {
System.err.println(e.getMessage());
System.err.println("usage: import [--delimiter <char>] <file.csv|file.parquet> [out.vortex]");
System.err.println("usage: import [--delimiter <char>] <file.csv|file.parquet|url> [out.vortex]");
return ExitStatus.USAGE_ERROR;
}
Path inputPath = parsedArgs.inputPath();
if (!Files.exists(inputPath)) {
System.err.println("file not found: " + inputPath);
return ExitStatus.FILE_NOT_FOUND;
}
String target = parsedArgs.inputTarget();
boolean remote = target.startsWith("http://") || target.startsWith("https://");
try {
if (remote) {
return runRemote(target, parsedArgs.outputTarget());
}
Path inputPath = Path.of(target);
if (!Files.exists(inputPath)) {
System.err.println("file not found: " + inputPath);
return ExitStatus.FILE_NOT_FOUND;
}
String name = inputPath.getFileName().toString();
Path outputPath = parsedArgs.outputTarget() != null
? Path.of(parsedArgs.outputTarget())
: inputPath.resolveSibling(vortexName(name));
if (name.endsWith(".parquet")) {
return runParquet(inputPath, parsedArgs.outputPath());
return runParquet(inputPath, outputPath);
} else {
return runCsv(inputPath, parsedArgs.outputPath(), parsedArgs.delimiter());
return runCsv(inputPath, outputPath, parsedArgs.delimiter());
}
} catch (IOException e) {
ProgressBar.clear();
Expand All @@ -48,6 +60,26 @@ static int run(String[] args) {
}
}

/// Handles an `http(s)://` source: Parquet import only (CSV import from a URL isn't
/// supported yet — CSV has no schema of its own, and type inference over a remote stream
/// needs a design of its own).
private static int runRemote(String parquetUrl, String outputTarget) throws IOException {
if (!parquetUrl.endsWith(".parquet")) {
System.err.println("only Parquet import is supported from a URL");
return ExitStatus.USAGE_ERROR;
}
Path vortexPath = outputTarget != null
? Path.of(outputTarget)
: Path.of(vortexName(lastPathSegment(parquetUrl)));
io.github.dfa1.vortex.parquet.ImportOptions options =
io.github.dfa1.vortex.parquet.ImportOptions.defaults()
.withProgressListener(ImportCommand::renderProgress);
ParquetImporter.importParquet(URI.create(parquetUrl), vortexPath, options);
ProgressBar.clear();
System.out.printf("written: %s (%s)%n", vortexPath, ByteSize.format(Files.size(vortexPath)));
return ExitStatus.OK;
}

private static ParsedArgs parseArgs(String[] args) {
if (args.length < 2) {
throw new IllegalArgumentException("missing import arguments");
Expand All @@ -72,9 +104,8 @@ private static ParsedArgs parseArgs(String[] args) {
if (positional.size() < 1 || positional.size() > 2) {
throw new IllegalArgumentException("expected input path and optional output path");
}
Path inputPath = Path.of(positional.getFirst());
Path outputPath = positional.size() == 2 ? Path.of(positional.get(1)) : deriveOutputPath(inputPath);
return new ParsedArgs(inputPath, outputPath, delimiter);
String outputTarget = positional.size() == 2 ? positional.get(1) : null;
return new ParsedArgs(positional.getFirst(), outputTarget, delimiter);
}

private static int runCsv(Path csvPath, Path vortexPath, Character delimiter) throws IOException {
Expand Down Expand Up @@ -125,13 +156,23 @@ private static void renderProgress(long done, long total) {
}
}

private static Path deriveOutputPath(Path inputPath) {
String name = inputPath.getFileName().toString();
/// Strips a known `.csv`/`.parquet` suffix from `inputFileName` and appends `.vortex`.
private static String vortexName(String inputFileName) {
String name = inputFileName;
if (name.endsWith(".csv")) {
name = name.substring(0, name.length() - 4);
} else if (name.endsWith(".parquet")) {
name = name.substring(0, name.length() - 8);
}
return inputPath.resolveSibling(name + ".vortex");
return name + ".vortex";
}

/// The last `/`-separated segment of a URL's path, used as the file name a downloaded
/// source is named after (mirroring what a browser would save the URL as).
private static String lastPathSegment(String url) {
String path = URI.create(url).getPath();
int slash = path.lastIndexOf('/');
String name = slash < 0 ? path : path.substring(slash + 1);
return name.isEmpty() ? "output.parquet" : name;
}
}
2 changes: 1 addition & 1 deletion cli/src/main/java/io/github/dfa1/vortex/cli/VortexCli.java
Original file line number Diff line number Diff line change
Expand Up @@ -47,7 +47,7 @@ static void printUsage(PrintStream out) {
out.println(" inspect <file|url> print file structure; url is http(s)://");
out.println(" tui <file|url> open interactive inspector; url is http(s)://");
out.println(" view <file|url> open scrollable data grid; url is http(s)://");
out.println(" export <file.vortex> [out.csv|-] write CSV; default output is <name>.csv, `-` for stdout");
out.println(" export <file.vortex> [out.csv|out.parquet|-] write CSV or Parquet; default is <name>.csv, `-` for stdout");
out.println(" import <file.csv|file.parquet> [out.vortex] convert CSV or Parquet to Vortex");
out.println(" schema <file.vortex> print dtype (machine-readable)");
out.println(" count <file.vortex> print row count");
Expand Down
51 changes: 51 additions & 0 deletions cli/src/test/java/io/github/dfa1/vortex/cli/ExportCommandTest.java
Original file line number Diff line number Diff line change
Expand Up @@ -49,4 +49,55 @@ void validFile_emitsCsvHeaderAndRows(@TempDir Path tmp) throws IOException {
assertThat(result.stdout()).startsWith("id");
assertThat(result.stdout().lines().count()).isEqualTo(4);
}

@Test
void parquetOutputPath_dispatchesToParquetExport(@TempDir Path tmp) throws IOException {
// Given — a `.parquet` destination, dispatching away from the CSV default
Path file = writeSmallVortex(tmp, "export.vortex");
Path outputPath = tmp.resolve("export.parquet");

// When
CliTestSupport.Captured result = capture(
() -> ExportCommand.run(new String[]{"export", file.toString(), outputPath.toString()}));

// Then
assertThat(result.status()).isEqualTo(ExitStatus.OK);
assertThat(outputPath).exists();
assertThat(result.stdout()).contains("written:").contains("export.parquet");
}

// ── URL source: argument validation only — these fail before any network call, so no
// mocked HTTP client is needed (mirrors ImportCommandTest's equivalent coverage). ──

@Test
void urlInput_missingOutputPath_returnsUsageError() {
// Given / When — a remote source needs an explicit `out.parquet` path
CliTestSupport.Captured result = capture(
() -> ExportCommand.run(new String[]{"export", "http://example.com/data.vortex"}));

// Then
assertThat(result.status()).isEqualTo(ExitStatus.USAGE_ERROR);
assertThat(result.stderr()).contains("out.parquet");
}

@Test
void urlInput_stdoutRequested_returnsUsageError() {
// Given / When — stdout streaming from a URL isn't supported
CliTestSupport.Captured result = capture(
() -> ExportCommand.run(new String[]{"export", "http://example.com/data.vortex", "-"}));

// Then
assertThat(result.status()).isEqualTo(ExitStatus.USAGE_ERROR);
}

@Test
void urlInput_csvOutputRequested_returnsUsageError(@TempDir Path tmp) {
// Given / When — CSV export from a URL isn't supported, only `.parquet`
Path outputPath = tmp.resolve("out.csv");
CliTestSupport.Captured result = capture(() ->
ExportCommand.run(new String[]{"export", "http://example.com/data.vortex", outputPath.toString()}));

// Then
assertThat(result.status()).isEqualTo(ExitStatus.USAGE_ERROR);
}
}
12 changes: 12 additions & 0 deletions cli/src/test/java/io/github/dfa1/vortex/cli/ImportCommandTest.java
Original file line number Diff line number Diff line change
Expand Up @@ -61,6 +61,18 @@ void tooManyPositional_returnsUsageError() {
assertThat(result.status()).isEqualTo(ExitStatus.USAGE_ERROR);
assertThat(result.stderr()).contains("input path");
}

@Test
void urlInput_nonParquetExtension_returnsUsageError() {
// Given / When — CSV import from a URL isn't supported; caught before any network
// call is made, so this needs no mocked HTTP client
CliTestSupport.Captured result = capture(() ->
ImportCommand.run(new String[]{"import", "http://example.com/data.csv"}));

// Then
assertThat(result.status()).isEqualTo(ExitStatus.USAGE_ERROR);
assertThat(result.stderr()).contains("only Parquet import is supported from a URL");
}
}

@Nested
Expand Down
Loading
Loading