Add remote CSV import + CSV-to-Parquet chaining in the CLI

CHANGELOG.md

@@ -10,6 +10,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 ### Added
 
 - `ParquetExporter` (`parquet` module): writes a Vortex file to Parquet, the inverse of `ParquetImporter`. Flat schemas only — `Bool`, non-`F16` `Primitive`, `Utf8`, `Binary`, and `vortex.timestamp` (MILLIS/MICROS/NANOS); `Struct`/`List`/`Map` top-level columns throw `UnsupportedOperationException`. Built on Hardwood 1.1's new `ColumnWriter`/`ParquetFileWriter` write API. The CLI's `export` subcommand now dispatches to it when the output path ends `.parquet` (previously CSV-only). ([#362](https://github.com/dfa1/vortex-java/pull/362))
+- `CsvImporter.importCsv(URI, Path[, ImportOptions])`: imports a CSV file served over HTTP(S), streaming the response body directly (CSV is read front to back in one pass, so unlike Parquet's random-access format this needs no Range requests). The CLI's `import` subcommand now accepts a `.csv` URL too, and — for both local and remote CSV — a `.parquet` output path chains CSV → temp Vortex → Parquet internally (Vortex stays the hub; Parquet is never a direct CSV-import target), deleting the temp file afterward. ([#363](https://github.com/dfa1/vortex-java/pull/363))
 - `ParquetExporter.exportParquet(VortexHandle, Path[, ExportOptions])` and `ParquetImporter.importParquet(URI, Path[, ImportOptions])`: both directions now work against a remote source over HTTP(S) — export from an already-open `VortexHttpReader` handle (no intervening local copy), import from a Parquet file served over HTTP(S) (new `HttpInputFile`, fetched entirely through targeted Range requests, mirroring `VortexHttpReader`'s own range-fetch pattern). The CLI's `export`/`import` subcommands accept a `url` source too, Parquet-only in both directions. ([#362](https://github.com/dfa1/vortex-java/pull/362))
 
 ### Changed

cli/src/main/java/io/github/dfa1/vortex/cli/ImportCommand.java

@@ -3,6 +3,7 @@
 import io.github.dfa1.vortex.csv.CsvImporter;
 import io.github.dfa1.vortex.inspect.ByteSize;
 import io.github.dfa1.vortex.csv.ImportOptions;
+import io.github.dfa1.vortex.parquet.ParquetExporter;
 import io.github.dfa1.vortex.parquet.ParquetImporter;
 
 import java.io.IOException;
@@ -30,14 +31,15 @@ static int run(String[] args) {
             parsedArgs = parseArgs(args);
         } catch (IllegalArgumentException e) {
             System.err.println(e.getMessage());
-            System.err.println("usage: import [--delimiter <char>] <file.csv|file.parquet|url> [out.vortex]");
+            System.err.println(
+                    "usage: import [--delimiter <char>] <file.csv|file.parquet|url> [out.vortex|out.parquet]");
             return ExitStatus.USAGE_ERROR;
         }
         String target = parsedArgs.inputTarget();
         boolean remote = target.startsWith("http://") || target.startsWith("https://");
         try {
             if (remote) {
-                return runRemote(target, parsedArgs.outputTarget());
+                return runRemote(target, parsedArgs.outputTarget(), parsedArgs.delimiter());
             }
             Path inputPath = Path.of(target);
             if (!Files.exists(inputPath)) {
@@ -49,6 +51,11 @@ static int run(String[] args) {
                     ? Path.of(parsedArgs.outputTarget())
                     : inputPath.resolveSibling(vortexName(name));
             if (name.endsWith(".parquet")) {
+                if (isParquetTarget(outputPath)) {
+                    System.err.println("import always converts Parquet to Vortex; "
+                            + "a Parquet source cannot import to a .parquet output");
+                    return ExitStatus.USAGE_ERROR;
+                }
                 return runParquet(inputPath, outputPath);
             } else {
                 return runCsv(inputPath, outputPath, parsedArgs.delimiter());
@@ -60,23 +67,53 @@ static int run(String[] args) {
         }
     }
 
-    /// Handles an `http(s)://` source: Parquet import only (CSV import from a URL isn't
-    /// supported yet — CSV has no schema of its own, and type inference over a remote stream
-    /// needs a design of its own).
-    private static int runRemote(String parquetUrl, String outputTarget) throws IOException {
-        if (!parquetUrl.endsWith(".parquet")) {
-            System.err.println("only Parquet import is supported from a URL");
-            return ExitStatus.USAGE_ERROR;
+    /// Handles an `http(s)://` source, dispatching by the *source* extension (Parquet or CSV;
+    /// nothing else is supported from a URL). The output target may independently be `.vortex`
+    /// or `.parquet` — see [#runCsv] / [#runRemoteCsv] for the CSV-to-Parquet chain.
+    private static int runRemote(String url, String outputTarget, Character delimiter) throws IOException {
+        if (url.endsWith(".parquet")) {
+            Path vortexPath = outputTarget != null
+                    ? Path.of(outputTarget)
+                    : Path.of(vortexName(lastPathSegment(url, "output.parquet")));
+            if (isParquetTarget(vortexPath)) {
+                System.err.println("import always converts Parquet to Vortex; "
+                        + "a Parquet source cannot import to a .parquet output");
+                return ExitStatus.USAGE_ERROR;
+            }
+            return runRemoteParquet(url, vortexPath);
+        }
+        if (url.endsWith(".csv")) {
+            Path outputPath = outputTarget != null
+                    ? Path.of(outputTarget)
+                    : Path.of(vortexName(lastPathSegment(url, "output.csv")));
+            return runRemoteCsv(url, outputPath, delimiter);
         }
-        Path vortexPath = outputTarget != null
-                ? Path.of(outputTarget)
-                : Path.of(vortexName(lastPathSegment(parquetUrl)));
+        System.err.println("only Parquet or CSV import is supported from a URL");
+        return ExitStatus.USAGE_ERROR;
+    }
+
+    private static int runRemoteParquet(String parquetUrl, Path vortexPath) throws IOException {
         io.github.dfa1.vortex.parquet.ImportOptions options =
                 io.github.dfa1.vortex.parquet.ImportOptions.defaults()
                         .withProgressListener(ImportCommand::renderProgress);
         ParquetImporter.importParquet(URI.create(parquetUrl), vortexPath, options);
         ProgressBar.clear();
-        System.out.printf("written: %s  (%s)%n", vortexPath, ByteSize.format(Files.size(vortexPath)));
+        printSimpleResult(vortexPath);
+        return ExitStatus.OK;
+    }
+
+    /// Imports a remote CSV, same as [#runCsv] but with no local input file to size for the
+    /// progress/result print, so the result line reports only the output size.
+    private static int runRemoteCsv(String csvUrl, Path outputPath, Character delimiter) throws IOException {
+        ImportOptions options = csvOptions(delimiter);
+        if (isParquetTarget(outputPath)) {
+            chainCsvToParquet(tempVortex -> CsvImporter.importCsv(URI.create(csvUrl), tempVortex, options),
+                    outputPath);
+        } else {
+            CsvImporter.importCsv(URI.create(csvUrl), outputPath, options);
+        }
+        ProgressBar.clear();
+        printSimpleResult(outputPath);
         return ExitStatus.OK;
     }
 
@@ -108,15 +145,21 @@ private static ParsedArgs parseArgs(String[] args) {
         return new ParsedArgs(positional.getFirst(), outputTarget, delimiter);
     }
 
-    private static int runCsv(Path csvPath, Path vortexPath, Character delimiter) throws IOException {
-        ImportOptions options = ImportOptions.defaults()
-                                        .withProgressListener(ImportCommand::renderProgress);
-        if (delimiter != null) {
-            options = options.withDelimiter(delimiter);
+    /// Imports a local CSV file. When `outputPath` ends `.parquet`, the CSV is imported to a
+    /// temp Vortex file first, then exported to Parquet and the temp file discarded — Vortex is
+    /// always the hub, Parquet is never a direct CSV-import target.
+    private static int runCsv(Path csvPath, Path outputPath, Character delimiter) throws IOException {
+        ImportOptions options = csvOptions(delimiter);
+        if (isParquetTarget(outputPath)) {
+            chainCsvToParquet(tempVortex -> CsvImporter.importCsv(csvPath, tempVortex, options), outputPath);
+            ProgressBar.clear();
+            // cascading depth doesn't apply to a Parquet destination — suppressed via 0.
+            printResult(csvPath, outputPath, 0);
+            return ExitStatus.OK;
         }
-        CsvImporter.importCsv(csvPath, vortexPath, options);
+        CsvImporter.importCsv(csvPath, outputPath, options);
         ProgressBar.clear();
-        printResult(csvPath, vortexPath, options.writeOptions().allowedCascading());
+        printResult(csvPath, outputPath, options.writeOptions().allowedCascading());
         return ExitStatus.OK;
     }
 
@@ -130,6 +173,33 @@ private static int runParquet(Path parquetPath, Path vortexPath) throws IOExcept
         return ExitStatus.OK;
     }
 
+    private static ImportOptions csvOptions(Character delimiter) {
+        ImportOptions options = ImportOptions.defaults()
+                                         .withProgressListener(ImportCommand::renderProgress);
+        return delimiter != null ? options.withDelimiter(delimiter) : options;
+    }
+
+    @FunctionalInterface
+    private interface CsvToVortex {
+        void importTo(Path tempVortex) throws IOException;
+    }
+
+    /// Imports CSV to a temp Vortex file via `importer`, exports that to `parquetOut`, then
+    /// discards the temp file — the CSV-to-Parquet chain shared by [#runCsv] and [#runRemoteCsv].
+    private static void chainCsvToParquet(CsvToVortex importer, Path parquetOut) throws IOException {
+        Path tempVortex = Files.createTempFile("vortex-cli-import-", ".vortex");
+        try {
+            importer.importTo(tempVortex);
+            ParquetExporter.exportParquet(tempVortex, parquetOut);
+        } finally {
+            Files.deleteIfExists(tempVortex);
+        }
+    }
+
+    private static boolean isParquetTarget(Path path) {
+        return path.getFileName().toString().endsWith(".parquet");
+    }
+
     private static void printResult(Path inputPath, Path vortexPath, int cascadingDepth) throws IOException {
         long inputBytes = Files.size(inputPath);
         long vortexBytes = Files.size(vortexPath);
@@ -145,6 +215,12 @@ private static void printResult(Path inputPath, Path vortexPath, int cascadingDe
                 sizeChange, cascadingInfo);
     }
 
+    /// Result line for a remote source, which has no cheaply-known local input size to compare
+    /// against — just the output path and its size.
+    private static void printSimpleResult(Path outputPath) throws IOException {
+        System.out.printf("written: %s  (%s)%n", outputPath, ByteSize.format(Files.size(outputPath)));
+    }
+
     /// Progress callback for imports. An indeterminate `total` (`< 0`, e.g. a streamed source with
     /// no known row count) shows just the running row count; otherwise delegates to the shared bar.
     private static void renderProgress(long done, long total) {
@@ -168,11 +244,12 @@ private static String vortexName(String inputFileName) {
     }
 
     /// The last `/`-separated segment of a URL's path, used as the file name a downloaded
-    /// source is named after (mirroring what a browser would save the URL as).
-    private static String lastPathSegment(String url) {
+    /// source is named after (mirroring what a browser would save the URL as). Falls back to
+    /// `fallback` when the path has no final segment (e.g. `https://host` with no path).
+    private static String lastPathSegment(String url, String fallback) {
         String path = URI.create(url).getPath();
         int slash = path.lastIndexOf('/');
         String name = slash < 0 ? path : path.substring(slash + 1);
-        return name.isEmpty() ? "output.parquet" : name;
+        return name.isEmpty() ? fallback : name;
     }
 }

cli/src/main/java/io/github/dfa1/vortex/cli/VortexCli.java

@@ -48,7 +48,8 @@ static void printUsage(PrintStream out) {
         out.println("  tui     <file|url>                  open interactive inspector; url is http(s)://");
         out.println("  view    <file|url>                  open scrollable data grid; url is http(s)://");
         out.println("  export  <file.vortex> [out.csv|out.parquet|-]  write CSV or Parquet; default is <name>.csv, `-` for stdout");
-        out.println("  import  <file.csv|file.parquet> [out.vortex]  convert CSV or Parquet to Vortex");
+        out.println("  import  [--delimiter <char>] <file.csv|file.parquet|url> [out.vortex|out.parquet]");
+        out.println("                                       convert CSV or Parquet (local or url) to Vortex or Parquet");
         out.println("  schema  <file.vortex>               print dtype (machine-readable)");
         out.println("  count   <file.vortex>               print row count");
         out.println("  select  <file.vortex> <col> [...]   project columns to CSV on stdout");

cli/src/test/java/io/github/dfa1/vortex/cli/ImportCommandTest.java

@@ -63,15 +63,15 @@ void tooManyPositional_returnsUsageError() {
         }
 
         @Test
-        void urlInput_nonParquetExtension_returnsUsageError() {
-            // Given / When — CSV import from a URL isn't supported; caught before any network
-            // call is made, so this needs no mocked HTTP client
+        void urlInput_unsupportedExtension_returnsUsageError() {
+            // Given / When — only Parquet and CSV sources are supported from a URL; caught
+            // before any network call is made, so this needs no mocked HTTP client
             CliTestSupport.Captured result = capture(() ->
-                    ImportCommand.run(new String[]{"import", "http://example.com/data.csv"}));
+                    ImportCommand.run(new String[]{"import", "http://example.com/data.json"}));
 
             // Then
             assertThat(result.status()).isEqualTo(ExitStatus.USAGE_ERROR);
-            assertThat(result.stderr()).contains("only Parquet import is supported from a URL");
+            assertThat(result.stderr()).contains("only Parquet or CSV import is supported from a URL");
         }
     }
 
@@ -143,5 +143,46 @@ void csvWithCustomDelimiter_imports(@TempDir Path tmp) throws IOException {
             assertThat(result.status()).isEqualTo(ExitStatus.OK);
             assertThat(tmp.resolve("data.tsv.vortex")).exists();
         }
+
+        @Test
+        void csvWithParquetOutputPath_chainsThroughTempVortex(@TempDir Path tmp) throws IOException {
+            // Given — a `.parquet` destination; Vortex is always the hub, so this chains
+            // CSV -> temp Vortex -> Parquet internally and discards the temp file
+            Path csv = tmp.resolve("in.csv");
+            Files.writeString(csv, "id,name\n1,Ada\n2,Grace\n", StandardCharsets.UTF_8);
+            Path out = tmp.resolve("out.parquet");
+
+            // When
+            CliTestSupport.Captured result = capture(() ->
+                    ImportCommand.run(new String[]{"import", csv.toString(), out.toString()}));
+
+            // Then — the real Parquet output exists; the temp Vortex file (system temp dir, not
+            // this directory) leaves nothing behind in the working directory
+            assertThat(result.status()).isEqualTo(ExitStatus.OK);
+            assertThat(out).exists();
+            assertThat(result.stdout()).contains(out.toString());
+            try (var files = Files.list(tmp)) {
+                assertThat(files.map(p -> p.getFileName().toString())).containsExactlyInAnyOrder("in.csv", "out.parquet");
+            }
+        }
+
+        @Test
+        void parquetInputWithParquetOutputPath_returnsUsageError(@TempDir Path tmp) throws IOException {
+            // Given — a Parquet source always produces Vortex; a `.parquet`-named output would
+            // silently write a Vortex-format file under a misleading name if left unchecked
+            Path vortex = CliTestSupport.writeSmallVortex(tmp, "src.vortex");
+            Path parquetIn = tmp.resolve("src.parquet");
+            io.github.dfa1.vortex.parquet.ParquetExporter.exportParquet(vortex, parquetIn);
+            Path parquetOut = tmp.resolve("out.parquet");
+
+            // When
+            CliTestSupport.Captured result = capture(() ->
+                    ImportCommand.run(new String[]{"import", parquetIn.toString(), parquetOut.toString()}));
+
+            // Then
+            assertThat(result.status()).isEqualTo(ExitStatus.USAGE_ERROR);
+            assertThat(result.stderr()).contains("Parquet source cannot import to a .parquet output");
+            assertThat(parquetOut).doesNotExist();
+        }
     }
 }