From 7934f0280667b74dbadb02666d6d9ee0af0be068 Mon Sep 17 00:00:00 2001 From: Davide Angelocola Date: Sun, 18 Oct 2026 20:27:05 +0200 Subject: [PATCH 01/10] feat(calcite): expose a directory of .vortex files as SQL tables VortexCalciteOptions (schema name, directory, explicit tables) replaces passing a raw table map; connect(Path) is the one-call shortcut. The columns still come from each file's dtype. Existing connect(String, Map) and new VortexSchema(Map) delegate to the options, so nothing breaks. Part of #566. Co-Authored-By: Claude Sonnet 5 --- CHANGELOG.md | 1 + .../dfa1/vortex/calcite/VortexCalcite.java | 29 +++- .../vortex/calcite/VortexCalciteOptions.java | 143 +++++++++++++++++ .../dfa1/vortex/calcite/VortexSchema.java | 9 +- .../calcite/VortexCalciteOptionsTest.java | 150 ++++++++++++++++++ docs/how-to.md | 9 ++ docs/reference.md | 2 + 7 files changed, 339 insertions(+), 4 deletions(-) create mode 100644 calcite/src/main/java/io/github/dfa1/vortex/calcite/VortexCalciteOptions.java create mode 100644 calcite/src/test/java/io/github/dfa1/vortex/calcite/VortexCalciteOptionsTest.java diff --git a/CHANGELOG.md b/CHANGELOG.md index 097b89df9..bef3b1553 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,6 +8,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] ### Added +- `VortexCalcite.connect(Path)` and `VortexCalciteOptions` expose every `.vortex` file in a directory as a SQL table. ([#566](https://github.com/dfa1/vortex-java/issues/566)) - `DecimalArray.getUnscaledLong(long)` reads a decimal's unscaled value without building a `BigDecimal`. ([#563](https://github.com/dfa1/vortex-java/pull/563)) - `Compute.groupedAggregate(Chunk, ColumnName, RowFilter, ColumnName)` folds an aggregate per value of a dictionary-encoded string, boolean or integer column. ([#555](https://github.com/dfa1/vortex-java/pull/555)) - `Chunk.slice(offset, length)` gives a zero-copy view of a row range. ([#550](https://github.com/dfa1/vortex-java/pull/550)) diff --git a/calcite/src/main/java/io/github/dfa1/vortex/calcite/VortexCalcite.java b/calcite/src/main/java/io/github/dfa1/vortex/calcite/VortexCalcite.java index ce564e8f0..deed55fa4 100644 --- a/calcite/src/main/java/io/github/dfa1/vortex/calcite/VortexCalcite.java +++ b/calcite/src/main/java/io/github/dfa1/vortex/calcite/VortexCalcite.java @@ -44,8 +44,7 @@ private VortexCalcite() { /// /// Each map entry binds a SQL table name to the `.vortex` file backing it; the tables are /// reachable as `schemaName.tableName`. The returned connection is the caller's to close, which - /// releases the schema's readers. If schema registration fails the connection is closed before - /// the failure propagates, so no connection leaks. + /// releases the schema's readers. /// /// @param schemaName the schema name the tables are registered under /// @param tables map from SQL table name to the `.vortex` file path @@ -54,6 +53,30 @@ private VortexCalcite() { public static Connection connect(String schemaName, Map tables) throws SQLException { Objects.requireNonNull(schemaName, "schemaName"); Objects.requireNonNull(tables, "tables"); + return connect(VortexCalciteOptions.defaults().withSchemaName(schemaName).withTables(tables)); + } + + /// Opens a Calcite JDBC connection exposing every `*.vortex` file in `directory` as a table of + /// the default schema `vtx`, so `ohlc.vortex` is queried as `vtx.ohlc`. + /// + /// @param directory the directory to scan for `.vortex` files + /// @return an open Calcite JDBC connection with the Vortex schema registered + /// @throws SQLException if the JDBC connection cannot be opened or the schema cannot be registered + public static Connection connect(Path directory) throws SQLException { + return connect(VortexCalciteOptions.defaults().withDirectory(directory)); + } + + /// Opens a Calcite JDBC connection exposing the tables `options` describes. + /// + /// The returned connection is the caller's to close, which releases the schema's readers. If + /// schema registration fails the connection is closed before the failure propagates, so no + /// connection leaks. + /// + /// @param options the schema name and the files to expose + /// @return an open Calcite JDBC connection with the Vortex schema registered + /// @throws SQLException if the JDBC connection cannot be opened or the schema cannot be registered + public static Connection connect(VortexCalciteOptions options) throws SQLException { + Objects.requireNonNull(options, "options"); Properties info = new Properties(); // JAVA lex: case-sensitive identifiers, back-tick quoting, unquoted case preserved — so an // unquoted `ohlc` matches the registered table name exactly. @@ -64,7 +87,7 @@ public static Connection connect(String schemaName, Map tables) th Connection connection = DriverManager.getConnection("jdbc:calcite:", info); try { SchemaPlus root = connection.unwrap(CalciteConnection.class).getRootSchema(); - root.add(schemaName, new VortexSchema(tables)); + root.add(options.schemaName(), new VortexSchema(options)); return connection; } catch (SQLException | RuntimeException e) { try { diff --git a/calcite/src/main/java/io/github/dfa1/vortex/calcite/VortexCalciteOptions.java b/calcite/src/main/java/io/github/dfa1/vortex/calcite/VortexCalciteOptions.java new file mode 100644 index 000000000..dbc134b5f --- /dev/null +++ b/calcite/src/main/java/io/github/dfa1/vortex/calcite/VortexCalciteOptions.java @@ -0,0 +1,143 @@ +package io.github.dfa1.vortex.calcite; + +import java.io.IOException; +import java.io.UncheckedIOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.LinkedHashMap; +import java.util.Map; +import java.util.Objects; +import java.util.Optional; +import java.util.stream.Stream; + +/// Settings for [VortexCalcite#connect(VortexCalciteOptions)]: which schema name to register and +/// which `.vortex` files become tables. +/// +/// Start from [#defaults()] and adjust with the `withXxx` methods, each of which returns a copy: +/// +/// ```java +/// VortexCalciteOptions options = VortexCalciteOptions.defaults() +/// .withDirectory(Path.of("/data/vortex")) // every *.vortex becomes a table +/// .withTable("trades", Path.of("/other/t.vortex")); +/// ``` +/// +/// A table is named after its file without the `.vortex` extension (`ohlc.vortex` is `ohlc`); an +/// explicit [#withTable(String, Path)] entry wins over a discovered file of the same name. Columns +/// are never declared: they come from each file's embedded dtype. +public final class VortexCalciteOptions { + + private static final String DEFAULT_SCHEMA_NAME = "vtx"; + private static final String EXTENSION = ".vortex"; + + private final String schemaName; + private final Path directory; + private final Map tables; + + private VortexCalciteOptions(String schemaName, Path directory, Map tables) { + this.schemaName = schemaName; + this.directory = directory; + this.tables = tables; + } + + /// Returns the starting point: schema name `vtx`, no directory, no explicit tables. + /// + /// @return the default options + public static VortexCalciteOptions defaults() { + return new VortexCalciteOptions(DEFAULT_SCHEMA_NAME, null, Map.of()); + } + + /// Returns a copy registering the tables under `schemaName`. + /// + /// @param schemaName the schema name + /// @return the updated options + public VortexCalciteOptions withSchemaName(String schemaName) { + Objects.requireNonNull(schemaName, "schemaName"); + return new VortexCalciteOptions(schemaName, directory, tables); + } + + /// Returns a copy that exposes every `*.vortex` file directly inside `directory` as a table. + /// + /// The directory is scanned when the schema is created, so a file added later appears only + /// on the next connection. + /// + /// @param directory the directory to scan + /// @return the updated options + public VortexCalciteOptions withDirectory(Path directory) { + Objects.requireNonNull(directory, "directory"); + return new VortexCalciteOptions(schemaName, directory, tables); + } + + /// Returns a copy that also binds the SQL table `name` to `file`. + /// + /// @param name the SQL table name + /// @param file the `.vortex` file backing it + /// @return the updated options + public VortexCalciteOptions withTable(String name, Path file) { + Objects.requireNonNull(name, "name"); + Objects.requireNonNull(file, "file"); + Map copy = new LinkedHashMap<>(tables); + copy.put(name, file); + return new VortexCalciteOptions(schemaName, directory, Map.copyOf(copy)); + } + + /// Returns a copy that also binds every entry of `tables`, as [#withTable(String, Path)] would. + /// + /// @param tables map from SQL table name to the `.vortex` file path + /// @return the updated options + public VortexCalciteOptions withTables(Map tables) { + Objects.requireNonNull(tables, "tables"); + VortexCalciteOptions result = this; + for (Map.Entry entry : tables.entrySet()) { + result = result.withTable(entry.getKey(), entry.getValue()); + } + return result; + } + + /// Returns the schema name the tables are registered under. + /// + /// @return the schema name + public String schemaName() { + return schemaName; + } + + /// Returns the directory scanned for `*.vortex` files, if one was set. + /// + /// @return the directory + public Optional directory() { + return Optional.ofNullable(directory); + } + + /// Returns the explicit table bindings, not including files found in the directory. + /// + /// @return an unmodifiable map from SQL table name to file + public Map tables() { + return tables; + } + + /// Resolves the full table set: files found in the directory, then the explicit bindings on top. + /// + /// @return map from SQL table name to the `.vortex` file path + /// @throws IllegalArgumentException if the directory is not a directory + /// @throws UncheckedIOException if the directory cannot be listed + Map resolveTables() { + Map resolved = new LinkedHashMap<>(); + if (directory != null) { + if (!Files.isDirectory(directory)) { + throw new IllegalArgumentException("not a directory: " + directory); + } + try (Stream entries = Files.list(directory)) { + entries.filter(Files::isRegularFile) + .filter(p -> p.getFileName().toString().endsWith(EXTENSION)) + .sorted() + .forEach(p -> { + String name = p.getFileName().toString(); + resolved.put(name.substring(0, name.length() - EXTENSION.length()), p); + }); + } catch (IOException e) { + throw new UncheckedIOException("cannot list " + directory, e); + } + } + resolved.putAll(tables); + return resolved; + } +} diff --git a/calcite/src/main/java/io/github/dfa1/vortex/calcite/VortexSchema.java b/calcite/src/main/java/io/github/dfa1/vortex/calcite/VortexSchema.java index df8be495b..19bc336ae 100644 --- a/calcite/src/main/java/io/github/dfa1/vortex/calcite/VortexSchema.java +++ b/calcite/src/main/java/io/github/dfa1/vortex/calcite/VortexSchema.java @@ -22,7 +22,14 @@ public final class VortexSchema extends AbstractSchema { /// /// @param files map from table name to the `.vortex` file path public VortexSchema(Map files) { - this.tables = files.entrySet().stream() + this(VortexCalciteOptions.defaults().withTables(files)); + } + + /// Creates a schema from `options`: the files found in its directory plus its explicit tables. + /// + /// @param options which files become tables + public VortexSchema(VortexCalciteOptions options) { + this.tables = options.resolveTables().entrySet().stream() .collect(java.util.stream.Collectors.toUnmodifiableMap( Map.Entry::getKey, e -> new VortexTable(e.getValue()))); } diff --git a/calcite/src/test/java/io/github/dfa1/vortex/calcite/VortexCalciteOptionsTest.java b/calcite/src/test/java/io/github/dfa1/vortex/calcite/VortexCalciteOptionsTest.java new file mode 100644 index 000000000..16e64d1de --- /dev/null +++ b/calcite/src/test/java/io/github/dfa1/vortex/calcite/VortexCalciteOptionsTest.java @@ -0,0 +1,150 @@ +package io.github.dfa1.vortex.calcite; + +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; + +import java.nio.file.Files; +import java.nio.file.Path; +import java.sql.Connection; +import java.sql.ResultSet; +import java.sql.Statement; +import java.util.Map; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatIllegalArgumentException; +import static org.assertj.core.api.Assertions.assertThatNullPointerException; + +/// Covers [VortexCalciteOptions] and directory discovery through [VortexCalcite#connect(Path)]: +/// a table is named after its file, columns come from the file's own dtype, and an explicit +/// binding wins over a discovered file. +class VortexCalciteOptionsTest { + + @TempDir + static Path tmp; + private static Path dir; + private static Path other; + + @BeforeAll + static void writeFiles() throws Exception { + dir = Files.createDirectory(tmp.resolve("data")); + // Different row counts tell the discovered file from the explicitly bound one. + OhlcGenerator.write(dir.resolve("ohlc.vortex"), 20_000, 10_000); + OhlcGenerator.write(dir.resolve("small.vortex"), 100, 100); + Files.writeString(dir.resolve("notes.txt"), "not a vortex file"); + other = tmp.resolve("other.vortex"); + OhlcGenerator.write(other, 7, 7); + } + + @Test + void defaults_useVtxSchemaAndNoTables() { + // Given / When + VortexCalciteOptions result = VortexCalciteOptions.defaults(); + + // Then + assertThat(result.schemaName()).isEqualTo("vtx"); + assertThat(result.directory()).isEmpty(); + assertThat(result.tables()).isEmpty(); + } + + @Test + void withMethods_returnCopiesAndLeaveTheOriginalUntouched() { + // Given + VortexCalciteOptions sut = VortexCalciteOptions.defaults(); + + // When + VortexCalciteOptions result = sut.withSchemaName("s").withDirectory(dir).withTable("t", other); + + // Then + assertThat(sut.schemaName()).isEqualTo("vtx"); + assertThat(sut.directory()).isEmpty(); + assertThat(sut.tables()).isEmpty(); + assertThat(result.schemaName()).isEqualTo("s"); + assertThat(result.directory()).contains(dir); + assertThat(result.tables()).containsExactly(Map.entry("t", other)); + } + + @Test + void resolveTables_namesTablesAfterFilesAndIgnoresOtherExtensions() { + // Given + VortexCalciteOptions sut = VortexCalciteOptions.defaults().withDirectory(dir); + + // When + Map result = sut.resolveTables(); + + // Then + assertThat(result).containsOnlyKeys("ohlc", "small"); + } + + @Test + void resolveTables_explicitTableWinsOverDiscoveredFile() { + // Given a directory holding ohlc.vortex and an explicit binding for the same name + VortexCalciteOptions sut = VortexCalciteOptions.defaults().withDirectory(dir).withTable("ohlc", other); + + // When + Map result = sut.resolveTables(); + + // Then + assertThat(result).containsEntry("ohlc", other).containsKey("small"); + } + + @Test + void resolveTables_rejectsAPathThatIsNotADirectory() { + // Given + VortexCalciteOptions sut = VortexCalciteOptions.defaults().withDirectory(other); + + // When / Then + assertThatIllegalArgumentException().isThrownBy(sut::resolveTables).withMessageContaining("not a directory"); + } + + @Test + void withMethods_rejectNulls() { + // Given + VortexCalciteOptions sut = VortexCalciteOptions.defaults(); + + // When / Then + assertThatNullPointerException().isThrownBy(() -> sut.withSchemaName(null)); + assertThatNullPointerException().isThrownBy(() -> sut.withDirectory(null)); + assertThatNullPointerException().isThrownBy(() -> sut.withTable(null, other)); + assertThatNullPointerException().isThrownBy(() -> sut.withTable("t", null)); + } + + @Test + void connectDirectory_queriesEveryDiscoveredFileWithNoDeclaredColumns() throws Exception { + // Given a directory connection: no table map, no column list + try (Connection conn = VortexCalcite.connect(dir); + Statement st = conn.createStatement()) { + + // When both discovered tables are counted + long ohlc = count(st, "vtx.ohlc"); + long small = count(st, "vtx.small"); + + // Then each is named after its file and holds that file's rows + assertThat(ohlc).isEqualTo(20_000); + assertThat(small).isEqualTo(100); + } + } + + @Test + void connectOptions_usesTheConfiguredSchemaNameAndExplicitOverride() throws Exception { + // Given + VortexCalciteOptions options = VortexCalciteOptions.defaults() + .withSchemaName("data").withDirectory(dir).withTable("ohlc", other); + + // When + try (Connection conn = VortexCalcite.connect(options); + Statement st = conn.createStatement()) { + long result = count(st, "data.ohlc"); + + // Then the explicit 7-row file shadowed the discovered 20,000-row one + assertThat(result).isEqualTo(7); + } + } + + private static long count(Statement st, String table) throws Exception { + try (ResultSet rs = st.executeQuery("select count(*) from " + table)) { + rs.next(); + return rs.getLong(1); + } + } +} diff --git a/docs/how-to.md b/docs/how-to.md index b25ca567f..7d59d37de 100644 --- a/docs/how-to.md +++ b/docs/how-to.md @@ -572,6 +572,15 @@ try (Connection conn = VortexCalcite.connect("vtx", tables)) { } ``` +Or point at a directory — every `*.vortex` file in it becomes a table named after the file, and the +columns come from each file's own schema: + +```java +try (Connection conn = VortexCalcite.connect(Path.of("/data/vortex"))) { + // select * from vtx.ohlc (ohlc.vortex) +} +``` + Whole-table `min`/`max`/`sum` aggregates are answered straight from zone-map statistics — no data segment is decoded — on files with the legacy `vortex.stats` zone map (written with `WriteOptions.defaults().withEdition(Editions.CORE_2025_10_0)`). Files with the default diff --git a/docs/reference.md b/docs/reference.md index cda5fe8a5..3dcdac5d2 100644 --- a/docs/reference.md +++ b/docs/reference.md @@ -605,6 +605,8 @@ Calcite](https://calcite.apache.org/). `connect(String, Map)` fold | Method | Notes | |--------------------------------------------------------------------|------------------------------------------------------------------------| | `static connect(String schemaName, Map tables)` | Opens a Calcite JDBC connection with `tables` registered under `schemaName` (`schemaName.tableName` in SQL); connection is the caller's to close | +| `static connect(Path directory)` | Every `*.vortex` file in `directory` becomes a table of schema `vtx`, named after the file (`ohlc.vortex` is `vtx.ohlc`); scanned once at connect time | +| `static connect(VortexCalciteOptions options)` | The general form: schema name, a directory to scan, and explicit `withTable(name, path)` bindings that win over a discovered file of the same name | The connection uses Calcite's `JAVA` lexical policy (case-sensitive identifiers, unquoted case preserved) and the **Babel** SQL parser, so a column named after a reserved word — `close`, `open`, From fb92450c8613db3e31799fb6fa37211beee79ab4 Mon Sep 17 00:00:00 2001 From: Davide Angelocola Date: Sun, 18 Oct 2026 20:30:57 +0200 Subject: [PATCH 02/10] feat(calcite): VortexSchemaFactory builds the schema from a Calcite model Lets a jdbc:calcite:model=... URL, as an IDE data source holds it, expose a directory of .vortex files. operand takes directory and files; any other key is rejected so a typo cannot silently expose no tables. The key is files, not tables: Calcite fills operand.tables with the model's own empty list. A model cannot set connection properties, so lex=JAVA must be in the URL. Part of #566. Co-Authored-By: Claude Sonnet 5 --- CHANGELOG.md | 1 + .../vortex/calcite/VortexSchemaFactory.java | 74 +++++++++++++++ .../calcite/VortexSchemaFactoryTest.java | 91 +++++++++++++++++++ docs/reference.md | 16 ++++ 4 files changed, 182 insertions(+) create mode 100644 calcite/src/main/java/io/github/dfa1/vortex/calcite/VortexSchemaFactory.java create mode 100644 calcite/src/test/java/io/github/dfa1/vortex/calcite/VortexSchemaFactoryTest.java diff --git a/CHANGELOG.md b/CHANGELOG.md index bef3b1553..6202a74ef 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,6 +8,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] ### Added +- `VortexSchemaFactory` builds the Calcite schema from a `jdbc:calcite:model=...` URL, so SQL tools can query a directory of `.vortex` files without Java code. ([#566](https://github.com/dfa1/vortex-java/issues/566)) - `VortexCalcite.connect(Path)` and `VortexCalciteOptions` expose every `.vortex` file in a directory as a SQL table. ([#566](https://github.com/dfa1/vortex-java/issues/566)) - `DecimalArray.getUnscaledLong(long)` reads a decimal's unscaled value without building a `BigDecimal`. ([#563](https://github.com/dfa1/vortex-java/pull/563)) - `Compute.groupedAggregate(Chunk, ColumnName, RowFilter, ColumnName)` folds an aggregate per value of a dictionary-encoded string, boolean or integer column. ([#555](https://github.com/dfa1/vortex-java/pull/555)) diff --git a/calcite/src/main/java/io/github/dfa1/vortex/calcite/VortexSchemaFactory.java b/calcite/src/main/java/io/github/dfa1/vortex/calcite/VortexSchemaFactory.java new file mode 100644 index 000000000..0b8d88dd6 --- /dev/null +++ b/calcite/src/main/java/io/github/dfa1/vortex/calcite/VortexSchemaFactory.java @@ -0,0 +1,74 @@ +package io.github.dfa1.vortex.calcite; + +import org.apache.calcite.schema.Schema; +import org.apache.calcite.schema.SchemaFactory; +import org.apache.calcite.schema.SchemaPlus; + +import java.nio.file.Path; +import java.util.LinkedHashSet; +import java.util.Map; +import java.util.Set; + +/// Builds a [VortexSchema] from a Calcite model, for `jdbc:calcite:model=...` URLs and tools that +/// only take a URL: +/// +/// ```json +/// { "version": "1.0", "defaultSchema": "vtx", +/// "schemas": [{ "name": "vtx", "type": "custom", +/// "factory": "io.github.dfa1.vortex.calcite.VortexSchemaFactory", +/// "operand": { "directory": "/data/vortex", +/// "files": { "trades": "/other/trades.vortex" } } }] } +/// ``` +/// +/// The schema name is the model's `name`. `operand` takes the same two settings as +/// [VortexCalciteOptions]: `directory` (every `*.vortex` in it becomes a table) and `files` (a +/// table name to path map that wins over a discovered file); any other key is rejected, so a typo does not +/// silently expose no tables. +/// +/// A model cannot set connection properties. To query reserved-word columns such as `close` +/// unquoted, add `lex=JAVA;parserFactory=org.apache.calcite.sql.parser.babel.SqlBabelParserImpl#FACTORY` +/// to the URL, or connect through [VortexCalcite], which sets both. +public final class VortexSchemaFactory implements SchemaFactory { + + private static final String DIRECTORY = "directory"; + // Not "tables": Calcite fills that key with the model's own (empty) table list. + private static final String FILES = "files"; + // Calcite adds these to every custom schema's operand; they are not ours to reject. + private static final Set CALCITE_KEYS = Set.of("modelUri", "baseDirectory", "tables"); + + /// Creates the factory Calcite instantiates by class name. + public VortexSchemaFactory() { + } + + @Override + public Schema create(SchemaPlus parent, String name, Map operand) { + return new VortexSchema(options(name, operand)); + } + + static VortexCalciteOptions options(String name, Map operand) { + Set unknown = new LinkedHashSet<>(operand.keySet()); + unknown.removeAll(Set.of(DIRECTORY, FILES)); + unknown.removeAll(CALCITE_KEYS); + if (!unknown.isEmpty()) { + throw new IllegalArgumentException( + "unknown VortexSchemaFactory operand " + unknown + "; expected " + DIRECTORY + ", " + FILES); + } + VortexCalciteOptions options = VortexCalciteOptions.defaults().withSchemaName(name); + if (operand.get(DIRECTORY) instanceof String directory) { + options = options.withDirectory(Path.of(directory)); + } else if (operand.containsKey(DIRECTORY)) { + throw new IllegalArgumentException("operand '" + DIRECTORY + "' must be a string"); + } + if (operand.get(FILES) instanceof Map files) { + for (Map.Entry entry : files.entrySet()) { + if (!(entry.getKey() instanceof String table) || !(entry.getValue() instanceof String file)) { + throw new IllegalArgumentException("operand '" + FILES + "' must map table names to path strings"); + } + options = options.withTable(table, Path.of(file)); + } + } else if (operand.containsKey(FILES)) { + throw new IllegalArgumentException("operand '" + FILES + "' must be an object"); + } + return options; + } +} diff --git a/calcite/src/test/java/io/github/dfa1/vortex/calcite/VortexSchemaFactoryTest.java b/calcite/src/test/java/io/github/dfa1/vortex/calcite/VortexSchemaFactoryTest.java new file mode 100644 index 000000000..8006976cc --- /dev/null +++ b/calcite/src/test/java/io/github/dfa1/vortex/calcite/VortexSchemaFactoryTest.java @@ -0,0 +1,91 @@ +package io.github.dfa1.vortex.calcite; + +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; + +import java.nio.file.Files; +import java.nio.file.Path; +import java.sql.Connection; +import java.sql.DriverManager; +import java.sql.ResultSet; +import java.sql.Statement; +import java.util.Map; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatIllegalArgumentException; + +/// Covers [VortexSchemaFactory]: the model `operand` maps onto [VortexCalciteOptions] with the same +/// rules as the Java API, and a plain `jdbc:calcite:model=inline:` URL, as an IDE would use, works. +class VortexSchemaFactoryTest { + + @TempDir + static Path tmp; + private static Path dir; + private static Path other; + + @BeforeAll + static void writeFiles() throws Exception { + dir = Files.createDirectory(tmp.resolve("data")); + OhlcGenerator.write(dir.resolve("ohlc.vortex"), 20_000, 10_000); + other = tmp.resolve("other.vortex"); + OhlcGenerator.write(other, 7, 7); + } + + @Test + void options_mapsDirectoryFilesAndTheModelSchemaName() { + // Given + Map operand = Map.of("directory", dir.toString(), "files", Map.of("extra", other.toString())); + + // When + VortexCalciteOptions result = VortexSchemaFactory.options("lake", operand); + + // Then + assertThat(result.schemaName()).isEqualTo("lake"); + assertThat(result.directory()).contains(dir); + assertThat(result.tables()).containsExactly(Map.entry("extra", other)); + } + + @Test + void options_rejectsAnUnknownKeyInsteadOfIgnoringATypo() { + // Given a misspelled key: silently ignoring it would expose no tables at all + Map operand = Map.of("directry", dir.toString()); + + // When / Then + assertThatIllegalArgumentException().isThrownBy(() -> VortexSchemaFactory.options("vtx", operand)) + .withMessageContaining("directry"); + } + + @Test + void options_rejectsWrongValueTypes() { + // Given / When / Then + assertThatIllegalArgumentException() + .isThrownBy(() -> VortexSchemaFactory.options("vtx", Map.of("directory", 1))); + assertThatIllegalArgumentException() + .isThrownBy(() -> VortexSchemaFactory.options("vtx", Map.of("files", "x"))); + assertThatIllegalArgumentException() + .isThrownBy(() -> VortexSchemaFactory.options("vtx", Map.of("files", Map.of("t", 1)))); + } + + @Test + void inlineModelUrl_queriesTheDirectory() throws Exception { + // Given a URL carrying the whole configuration, as pasted into an IDE data source. A model + // cannot set connection properties, so lex=JAVA must be in the URL: without it Calcite + // uppercases the unquoted table name and finds nothing. + String url = "jdbc:calcite:lex=JAVA;model=inline:{version:'1.0',defaultSchema:'vtx',schemas:[{name:'vtx'," + + "type:'custom',factory:'" + VortexSchemaFactory.class.getName() + "'," + + "operand:{directory:'" + dir + "'}}]}"; + + // When + long result; + try (Connection conn = DriverManager.getConnection(url); + Statement st = conn.createStatement(); + ResultSet rs = st.executeQuery("select count(*) from ohlc")) { + rs.next(); + result = rs.getLong(1); + } + + // Then + assertThat(result).isEqualTo(20_000); + } +} diff --git a/docs/reference.md b/docs/reference.md index 3dcdac5d2..a18a28025 100644 --- a/docs/reference.md +++ b/docs/reference.md @@ -608,6 +608,22 @@ Calcite](https://calcite.apache.org/). `connect(String, Map)` fold | `static connect(Path directory)` | Every `*.vortex` file in `directory` becomes a table of schema `vtx`, named after the file (`ohlc.vortex` is `vtx.ohlc`); scanned once at connect time | | `static connect(VortexCalciteOptions options)` | The general form: schema name, a directory to scan, and explicit `withTable(name, path)` bindings that win over a discovered file of the same name | +### `VortexSchemaFactory` (`io.github.dfa1.vortex.calcite.VortexSchemaFactory`) + +Builds the schema from a Calcite model, so a `jdbc:calcite:` URL (or an IDE data source) needs no +Java code. `operand` takes `directory` (every `*.vortex` becomes a table) and `files` (table name to +path, wins over a discovered file); any other key is rejected. The schema name is the model's +`name`. + +``` +jdbc:calcite:lex=JAVA;model=inline:{version:'1.0',defaultSchema:'vtx',schemas:[{name:'vtx',type:'custom',factory:'io.github.dfa1.vortex.calcite.VortexSchemaFactory',operand:{directory:'/data/vortex'}}]} +``` + +A model cannot set connection properties, so put `lex=JAVA` in the URL (without it Calcite +uppercases unquoted names and `vtx.ohlc` is not found). For reserved-word columns such as `close` +unquoted, also add `parserFactory=org.apache.calcite.sql.parser.babel.SqlBabelParserImpl#FACTORY`; +[`VortexCalcite`] sets both for you. + The connection uses Calcite's `JAVA` lexical policy (case-sensitive identifiers, unquoted case preserved) and the **Babel** SQL parser, so a column named after a reserved word — `close`, `open`, `value`, `year`, … — is queryable unquoted (`select close, open from vtx.ohlc`). The exception is From b322e55356486233d9170008a946f09b6eabd421 Mon Sep 17 00:00:00 2001 From: Davide Angelocola Date: Sun, 18 Oct 2026 20:38:05 +0200 Subject: [PATCH 03/10] fix(calcite): connect without DriverManager so a separately loaded jar works VortexCalcite.connect called DriverManager.getConnection("jdbc:calcite:"), which only finds drivers already registered. A jar loaded by its own classloader (an IDE data source, an application server) never had Calcite's driver registered, so connect failed with "No suitable driver". Instantiate Calcite's driver directly. Found by the shaded-jar integration test added with the jdbc-driver module. Part of #566. Co-Authored-By: Claude Sonnet 5 --- CHANGELOG.md | 1 + .../java/io/github/dfa1/vortex/calcite/VortexCalcite.java | 6 ++++-- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 6202a74ef..81a662ffa 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -50,6 +50,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - Frame-of-reference encoding takes the unsigned minimum of a `u64` column whose values span 2^63 as its reference, as Rust does, instead of a reference that left those values at full width. ([#571](https://github.com/dfa1/vortex-java/pull/571)) - **Breaking:** `DType.Decimal` refuses a precision outside 1 to 76 or a scale above the precision, as Rust does, so the writer no longer writes decimal columns that Rust and vortex-java cannot read, and a variant's decimal is checked as it is read. ([#571](https://github.com/dfa1/vortex-java/pull/571)) - SQL `ORDER BY ... LIMIT` and `GROUP BY` with a few thousand result cells no longer fail with "Code grows beyond 64 KB"; past the limit Calcite scans instead. ([#569](https://github.com/dfa1/vortex-java/pull/569)) +- `VortexCalcite.connect` no longer fails with "No suitable driver" when the jar is loaded by its own classloader, as an IDE or application server does. ([#566](https://github.com/dfa1/vortex-java/issues/566)) - A file whose array FlatBuffer holds a corrupt offset fails the read with `VortexException` instead of `IndexOutOfBoundsException`. ([#493](https://github.com/dfa1/vortex-java/issues/493)) - A column whose dtype does not fit its bit-packed, delta or RLE encoding fails with `VortexException` instead of `ClassCastException`, and an empty RLE boolean chunk reads. ([#493](https://github.com/dfa1/vortex-java/issues/493)) - A struct dtype missing its field names or types, or a struct layout whose children do not match its fields, is rejected when the file opens, as Rust does, instead of failing the scan with `ArrayIndexOutOfBoundsException`. ([#493](https://github.com/dfa1/vortex-java/issues/493)) diff --git a/calcite/src/main/java/io/github/dfa1/vortex/calcite/VortexCalcite.java b/calcite/src/main/java/io/github/dfa1/vortex/calcite/VortexCalcite.java index deed55fa4..00a24b1cf 100644 --- a/calcite/src/main/java/io/github/dfa1/vortex/calcite/VortexCalcite.java +++ b/calcite/src/main/java/io/github/dfa1/vortex/calcite/VortexCalcite.java @@ -1,12 +1,12 @@ package io.github.dfa1.vortex.calcite; import org.apache.calcite.jdbc.CalciteConnection; +import org.apache.calcite.jdbc.Driver; import org.apache.calcite.schema.SchemaPlus; import org.apache.calcite.sql.parser.babel.SqlBabelParserImpl; import java.nio.file.Path; import java.sql.Connection; -import java.sql.DriverManager; import java.sql.SQLException; import java.util.Map; import java.util.Objects; @@ -84,7 +84,9 @@ public static Connection connect(VortexCalciteOptions options) throws SQLExcepti // Babel parser: accept SQL reserved words (date, close, open, value, …) as identifiers, so // columnar files whose column names collide with keywords are queryable unquoted. info.setProperty("parserFactory", SqlBabelParserImpl.class.getName() + "#FACTORY"); - Connection connection = DriverManager.getConnection("jdbc:calcite:", info); + // Not DriverManager: it only sees drivers already registered, and a jar loaded by its own + // classloader (an IDE data source, an application server) has not had Calcite's registered. + Connection connection = new Driver().connect("jdbc:calcite:", info); try { SchemaPlus root = connection.unwrap(CalciteConnection.class).getRootSchema(); root.add(options.schemaName(), new VortexSchema(options)); From 67ee89c0a56aab5f7a41bd1feaec6b4b122a4fe5 Mon Sep 17 00:00:00 2001 From: Davide Angelocola Date: Sun, 18 Oct 2026 20:38:05 +0200 Subject: [PATCH 04/10] feat(jdbc-driver): jdbc:vortex: driver in a self-contained jar for SQL tools New module vortex-jdbc-driver: a java.sql.Driver registered through META-INF/services that parses jdbc:vortex:?schema=..&table.= into VortexCalciteOptions and connects through VortexCalcite. Unknown keys fail with SQLState 08001 so a typo cannot silently expose no tables. The -all shaded jar merges the java.sql.Driver service entries, strips signature files and bundles the zstd natives. ShadedJarIT loads it alone in an isolated classloader after package and runs a query. Closes #566. Co-Authored-By: Claude Sonnet 5 --- CHANGELOG.md | 1 + bom/pom.xml | 5 + docs/how-to.md | 27 +++ docs/reference.md | 23 +++ jdbc-driver/pom.xml | 128 ++++++++++++ .../dfa1/vortex/jdbc/driver/VortexDriver.java | 143 +++++++++++++ .../META-INF/services/java.sql.Driver | 1 + .../dfa1/vortex/jdbc/driver/Fixtures.java | 39 ++++ .../dfa1/vortex/jdbc/driver/ShadedJarIT.java | 82 ++++++++ .../vortex/jdbc/driver/VortexDriverTest.java | 195 ++++++++++++++++++ pom.xml | 6 + 11 files changed, 650 insertions(+) create mode 100644 jdbc-driver/pom.xml create mode 100644 jdbc-driver/src/main/java/io/github/dfa1/vortex/jdbc/driver/VortexDriver.java create mode 100644 jdbc-driver/src/main/resources/META-INF/services/java.sql.Driver create mode 100644 jdbc-driver/src/test/java/io/github/dfa1/vortex/jdbc/driver/Fixtures.java create mode 100644 jdbc-driver/src/test/java/io/github/dfa1/vortex/jdbc/driver/ShadedJarIT.java create mode 100644 jdbc-driver/src/test/java/io/github/dfa1/vortex/jdbc/driver/VortexDriverTest.java diff --git a/CHANGELOG.md b/CHANGELOG.md index 81a662ffa..c979ac84b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,6 +8,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] ### Added +- `vortex-jdbc-driver` is a `jdbc:vortex:` JDBC driver in one self-contained jar, so IntelliJ, DBeaver and other SQL tools can query a directory of `.vortex` files. ([#566](https://github.com/dfa1/vortex-java/issues/566)) - `VortexSchemaFactory` builds the Calcite schema from a `jdbc:calcite:model=...` URL, so SQL tools can query a directory of `.vortex` files without Java code. ([#566](https://github.com/dfa1/vortex-java/issues/566)) - `VortexCalcite.connect(Path)` and `VortexCalciteOptions` expose every `.vortex` file in a directory as a SQL table. ([#566](https://github.com/dfa1/vortex-java/issues/566)) - `DecimalArray.getUnscaledLong(long)` reads a decimal's unscaled value without building a `BigDecimal`. ([#563](https://github.com/dfa1/vortex-java/pull/563)) diff --git a/bom/pom.xml b/bom/pom.xml index 363808254..0540b19bb 100644 --- a/bom/pom.xml +++ b/bom/pom.xml @@ -75,6 +75,11 @@ vortex-calcite ${project.version} + + io.github.dfa1.vortex + vortex-jdbc-driver + ${project.version} + diff --git a/docs/how-to.md b/docs/how-to.md index 7d59d37de..7af92df4c 100644 --- a/docs/how-to.md +++ b/docs/how-to.md @@ -592,6 +592,33 @@ policy. --- +## Query Vortex files from an SQL tool (IntelliJ, DBeaver) + +`vortex-jdbc-driver` is a JDBC driver: one self-contained jar (Calcite and the Vortex modules +inside), so a tool that only takes a driver jar and a URL can query `.vortex` files. It is an +embedded engine like DuckDB or SQLite: the query runs inside the tool's JVM, which must be JDK 25 or +later. + +1. Build or download `vortex-jdbc-driver--all.jar`. +2. In IntelliJ: *Database* → *+* → *Driver*; add that jar, and set the class to + `io.github.dfa1.vortex.jdbc.driver.VortexDriver`. In DBeaver: *Database* → *Driver Manager* → + *New*, same jar and class. +3. Create a data source with the URL: + +``` +jdbc:vortex:/data/vortex +jdbc:vortex:/data/vortex?schema=lake&table.trades=/other/trades.vortex +``` + +Every `*.vortex` file in the directory becomes a table named after the file (`ohlc.vortex` is +`vtx.ohlc`); columns come from the file. `schema` renames `vtx`, and `table.=` adds or +overrides one file. Any other key is an error, so a typo cannot silently expose no tables. Leave user +and password empty. The directory is scanned when the connection opens, so a file added later needs a +reconnect. + +On JDK 25 the first query prints a `sun.misc.Unsafe` warning from protobuf, a Calcite dependency; +it is harmless. + ## Query a Vortex file with DuckDB [DuckDB](https://duckdb.org) reads Vortex files through its `vortex` extension, which is built from the Rust reference diff --git a/docs/reference.md b/docs/reference.md index a18a28025..5e7951a3b 100644 --- a/docs/reference.md +++ b/docs/reference.md @@ -683,6 +683,29 @@ by hand instead of going through `connect`. --- +## JDBC driver + +### `VortexDriver` (`io.github.dfa1.vortex.jdbc.driver.VortexDriver`) + +Artifact `vortex-jdbc-driver`: a `java.sql.Driver` registered through +`META-INF/services/java.sql.Driver`, for SQL tools that take a URL. It parses the URL into a +[`VortexCalciteOptions`](#vortexcalcite-iogithubdfa1vortexcalcitevortexcalcite) and connects through +`VortexCalcite.connect(options)`, so it behaves exactly like the Java API. + +| URL | Meaning | +|------------------------------------------------------|---------------------------------------------------------------------------------| +| `jdbc:vortex:` | Every `*.vortex` in `` is a table of schema `vtx`, named after the file | +| `...?schema=` | Registers the tables under `` instead of `vtx` | +| `...?table.=` | Binds one file; wins over a discovered file of the same name. `` may be empty if at least one is given | + +Values are URL-decoded. An unknown key, a pair without `=`, or a URL exposing no files throws an +`SQLException` with state `08001`. Driver properties (`user`, `password`) are ignored. + +The `-all` classifier is the jar to hand to a tool: it merges `META-INF/services` (ours and +Calcite's `java.sql.Driver` entries), strips signature files and sets `Enable-Native-Access`. +`ShadedJarIT` loads it alone in an isolated classloader and runs a query, so a lost service entry or a +stale signature fails `verify`. Needs JDK 25 or later. + ## CLI The `-all` uber-jar is self-contained: pure-Java code plus the native `libzstd` for all diff --git a/jdbc-driver/pom.xml b/jdbc-driver/pom.xml new file mode 100644 index 000000000..e60da8b09 --- /dev/null +++ b/jdbc-driver/pom.xml @@ -0,0 +1,128 @@ + + + 4.0.0 + + io.github.dfa1.vortex + vortex-java + 0.17.3-SNAPSHOT + + + vortex-jdbc-driver + + vortex-jdbc-driver + JDBC driver (jdbc:vortex:) to query Vortex files with SQL from any JDBC tool, shipped as a self-contained jar. + + + + --sun-misc-unsafe-memory-access=allow + + + + + + io.github.dfa1.vortex + vortex-calcite + + + + io.github.dfa1.zstd + zstd + + + io.github.dfa1.zstd + zstd-platform + runtime + + + + io.github.dfa1.vortex + vortex-core + test-jar + test + + + io.github.dfa1.vortex + vortex-writer + test + + + org.junit.jupiter + junit-jupiter + test + + + org.assertj + assertj-core + test + + + + + + + + org.apache.maven.plugins + maven-failsafe-plugin + + + **/*IT.java + + @{argLine} --sun-misc-unsafe-memory-access=allow + + + + + integration-test + verify + + + + + + org.apache.maven.plugins + maven-shade-plugin + + + package + + shade + + + true + all + false + + + vortex-jdbc-driver + + + + + + ALL-UNNAMED + + + + + + + + *:* + + module-info.class + META-INF/*.SF + META-INF/*.DSA + META-INF/*.RSA + + + + + + + + + + diff --git a/jdbc-driver/src/main/java/io/github/dfa1/vortex/jdbc/driver/VortexDriver.java b/jdbc-driver/src/main/java/io/github/dfa1/vortex/jdbc/driver/VortexDriver.java new file mode 100644 index 000000000..5fe9d0006 --- /dev/null +++ b/jdbc-driver/src/main/java/io/github/dfa1/vortex/jdbc/driver/VortexDriver.java @@ -0,0 +1,143 @@ +package io.github.dfa1.vortex.jdbc.driver; + +import io.github.dfa1.vortex.calcite.VortexCalcite; +import io.github.dfa1.vortex.calcite.VortexCalciteOptions; + +import java.net.URLDecoder; +import java.nio.charset.StandardCharsets; +import java.nio.file.Path; +import java.sql.Connection; +import java.sql.Driver; +import java.sql.DriverManager; +import java.sql.DriverPropertyInfo; +import java.sql.SQLException; +import java.sql.SQLFeatureNotSupportedException; +import java.util.Properties; +import java.util.logging.Logger; + +/// JDBC driver that queries Vortex files with SQL, for IDEs and tools that only accept a URL: +/// +/// ``` +/// jdbc:vortex:/data/vortex +/// jdbc:vortex:/data/vortex?schema=lake&table.trades=/other/trades.vortex +/// ``` +/// +/// The path is a directory whose `*.vortex` files become tables named after the file (`ohlc.vortex` +/// is `vtx.ohlc`); columns come from each file, never declared. The query string takes `schema` +/// (default `vtx`) and `table.=` (an explicit file, which wins over a discovered one). +/// Any other key is rejected, so a typo cannot silently expose no tables. Values are URL-decoded. +/// +/// The URL maps one-to-one onto [VortexCalciteOptions]; the connection is +/// [VortexCalcite#connect(VortexCalciteOptions)], so identifiers are case-sensitive and reserved +/// words such as `close` work unquoted. Driver properties are ignored (IDEs always pass `user` and +/// `password`). +public final class VortexDriver implements Driver { + + static final String PREFIX = "jdbc:vortex:"; + private static final String SCHEMA = "schema"; + private static final String TABLE_PREFIX = "table."; + + static { + try { + DriverManager.registerDriver(new VortexDriver()); + } catch (SQLException e) { + throw new ExceptionInInitializerError(e); + } + } + + /// Creates the driver; `DriverManager` does this through `META-INF/services/java.sql.Driver`. + public VortexDriver() { + } + + @Override + public Connection connect(String url, Properties info) throws SQLException { + if (!acceptsURL(url)) { + return null; + } + VortexCalciteOptions options = options(url); + try { + return VortexCalcite.connect(options); + } catch (IllegalArgumentException e) { + throw new SQLException(e.getMessage(), "08001", e); + } + } + + @Override + public boolean acceptsURL(String url) { + return url != null && url.startsWith(PREFIX); + } + + /// Parses `url` into the options it describes. + /// + /// @param url a `jdbc:vortex:` URL + /// @return the options + /// @throws SQLException if the URL has an unknown key, a malformed pair or no files to expose + static VortexCalciteOptions options(String url) throws SQLException { + String rest = url.substring(PREFIX.length()); + int query = rest.indexOf('?'); + String path = query < 0 ? rest : rest.substring(0, query); + VortexCalciteOptions options = VortexCalciteOptions.defaults(); + boolean hasFiles = !path.isEmpty(); + if (hasFiles) { + options = options.withDirectory(Path.of(decode(path))); + } + if (query >= 0) { + for (String pair : rest.substring(query + 1).split("&")) { + if (pair.isEmpty()) { + continue; + } + int eq = pair.indexOf('='); + if (eq <= 0) { + throw invalid(url, "expected key=value, got '" + pair + "'"); + } + String key = decode(pair.substring(0, eq)); + String value = decode(pair.substring(eq + 1)); + if (key.equals(SCHEMA)) { + options = options.withSchemaName(value); + } else if (key.startsWith(TABLE_PREFIX) && key.length() > TABLE_PREFIX.length()) { + options = options.withTable(key.substring(TABLE_PREFIX.length()), Path.of(value)); + hasFiles = true; + } else { + throw invalid(url, "unknown key '" + key + "'; expected schema or table."); + } + } + } + if (!hasFiles) { + throw invalid(url, "give a directory (jdbc:vortex:/path) or at least one table.="); + } + return options; + } + + private static String decode(String value) { + return URLDecoder.decode(value, StandardCharsets.UTF_8); + } + + private static SQLException invalid(String url, String reason) { + return new SQLException("invalid Vortex URL '" + url + "': " + reason, "08001"); + } + + @Override + public DriverPropertyInfo[] getPropertyInfo(String url, Properties info) { + return new DriverPropertyInfo[0]; + } + + @Override + public int getMajorVersion() { + return 0; + } + + @Override + public int getMinorVersion() { + return 0; + } + + @Override + public boolean jdbcCompliant() { + return false; + } + + @Override + public Logger getParentLogger() throws SQLFeatureNotSupportedException { + throw new SQLFeatureNotSupportedException("the Vortex driver does not use java.util.logging"); + } +} diff --git a/jdbc-driver/src/main/resources/META-INF/services/java.sql.Driver b/jdbc-driver/src/main/resources/META-INF/services/java.sql.Driver new file mode 100644 index 000000000..bef5aa932 --- /dev/null +++ b/jdbc-driver/src/main/resources/META-INF/services/java.sql.Driver @@ -0,0 +1 @@ +io.github.dfa1.vortex.jdbc.driver.VortexDriver diff --git a/jdbc-driver/src/test/java/io/github/dfa1/vortex/jdbc/driver/Fixtures.java b/jdbc-driver/src/test/java/io/github/dfa1/vortex/jdbc/driver/Fixtures.java new file mode 100644 index 000000000..753570af0 --- /dev/null +++ b/jdbc-driver/src/test/java/io/github/dfa1/vortex/jdbc/driver/Fixtures.java @@ -0,0 +1,39 @@ +package io.github.dfa1.vortex.jdbc.driver; + +import io.github.dfa1.vortex.core.model.ColumnName; +import io.github.dfa1.vortex.core.model.Editions; +import io.github.dfa1.vortex.core.testing.OhlcData; +import io.github.dfa1.vortex.writer.VortexWriter; +import io.github.dfa1.vortex.writer.WriteOptions; + +import java.io.IOException; +import java.nio.channels.FileChannel; +import java.nio.file.Path; +import java.nio.file.StandardOpenOption; +import java.util.LinkedHashMap; +import java.util.Map; + +/// Writes a small single-chunk OHLC Vortex file for the driver tests. +final class Fixtures { + + private Fixtures() { + } + + static void write(Path file, int rows) throws IOException { + WriteOptions options = WriteOptions.cascading(0).withEdition(Editions.CORE_2025_10_0); + try (var channel = FileChannel.open(file, StandardOpenOption.CREATE, StandardOpenOption.WRITE); + var writer = VortexWriter.create(channel, OhlcData.SCHEMA, options)) { + for (OhlcData.Batch batch : OhlcData.generate(rows, rows)) { + Map columns = new LinkedHashMap<>(); + columns.put(ColumnName.of("date"), batch.date()); + columns.put(ColumnName.of("symbol"), batch.symbol()); + columns.put(ColumnName.of("open"), batch.open()); + columns.put(ColumnName.of("high"), batch.high()); + columns.put(ColumnName.of("low"), batch.low()); + columns.put(ColumnName.of("close"), batch.close()); + columns.put(ColumnName.of("volume"), batch.volume()); + writer.writeChunk(columns); + } + } + } +} diff --git a/jdbc-driver/src/test/java/io/github/dfa1/vortex/jdbc/driver/ShadedJarIT.java b/jdbc-driver/src/test/java/io/github/dfa1/vortex/jdbc/driver/ShadedJarIT.java new file mode 100644 index 000000000..43955aa0d --- /dev/null +++ b/jdbc-driver/src/test/java/io/github/dfa1/vortex/jdbc/driver/ShadedJarIT.java @@ -0,0 +1,82 @@ +package io.github.dfa1.vortex.jdbc.driver; + +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; + +import java.io.IOException; +import java.net.URL; +import java.net.URLClassLoader; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; +import java.sql.Connection; +import java.sql.Driver; +import java.sql.ResultSet; +import java.sql.Statement; +import java.util.Properties; +import java.util.jar.JarFile; +import java.util.stream.Stream; + +import static org.assertj.core.api.Assertions.assertThat; + +/// Exercises the shaded `-all` jar the way an IDE does: alone, with none of this module's own +/// classpath. A plain unit test cannot catch what shading loses (a clobbered `java.sql.Driver` +/// service file, leftover signature files that make the JVM reject the jar), so this runs after +/// `package`, under failsafe. The jar is loaded through a classloader whose parent is the JDK +/// platform loader, so no test dependency (Calcite, the Vortex modules) can leak in. +class ShadedJarIT { + + @TempDir + Path tmp; + + @Test + void shadedJar_mergesDriverEntriesAndCarriesNoSignatures() throws Exception { + // Given + Path jar = shadedJar(); + + // When + String services; + long signatures; + try (JarFile jarFile = new JarFile(jar.toFile())) { + services = new String(jarFile.getInputStream(jarFile.getEntry("META-INF/services/java.sql.Driver")) + .readAllBytes(), StandardCharsets.UTF_8); + signatures = jarFile.stream().filter(e -> e.getName().matches("META-INF/.*\\.(SF|RSA|DSA)")).count(); + } + + // Then ServicesResourceTransformer kept ours and Calcite's; a stale signature would make the JVM + // reject the whole jar with a SecurityException + assertThat(services).contains(VortexDriver.class.getName(), "org.apache.calcite.jdbc.Driver"); + assertThat(signatures).isZero(); + } + + @Test + void shadedJar_runsSqlWithNothingElseOnTheClasspath() throws Exception { + // Given a Vortex file and only the shaded jar on an isolated classloader + Path dir = Files.createDirectory(tmp.resolve("data")); + Fixtures.write(dir.resolve("ohlc.vortex"), 500); + URL[] urls = {shadedJar().toUri().toURL()}; + try (URLClassLoader isolated = new URLClassLoader(urls, ClassLoader.getPlatformClassLoader())) { + Driver driver = (Driver) isolated.loadClass(VortexDriver.class.getName()) + .getDeclaredConstructor().newInstance(); + + // When a query runs through it + long result; + try (Connection conn = driver.connect("jdbc:vortex:" + dir, new Properties()); + Statement st = conn.createStatement(); + ResultSet rs = st.executeQuery("select count(*) from vtx.ohlc")) { + rs.next(); + result = rs.getLong(1); + } + + // Then every dependency the query needed came from the jar + assertThat(result).isEqualTo(500); + } + } + + private static Path shadedJar() throws IOException { + try (Stream files = Files.list(Path.of("target"))) { + return files.filter(p -> p.getFileName().toString().endsWith("-all.jar")).findFirst() + .orElseThrow(() -> new AssertionError("no -all.jar in target/: run `verify`, not `test`")); + } + } +} diff --git a/jdbc-driver/src/test/java/io/github/dfa1/vortex/jdbc/driver/VortexDriverTest.java b/jdbc-driver/src/test/java/io/github/dfa1/vortex/jdbc/driver/VortexDriverTest.java new file mode 100644 index 000000000..4c4c513c6 --- /dev/null +++ b/jdbc-driver/src/test/java/io/github/dfa1/vortex/jdbc/driver/VortexDriverTest.java @@ -0,0 +1,195 @@ +package io.github.dfa1.vortex.jdbc.driver; + +import io.github.dfa1.vortex.calcite.VortexCalciteOptions; + +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.ValueSource; + +import java.nio.file.Files; +import java.nio.file.Path; +import java.sql.Connection; +import java.sql.Driver; +import java.sql.DriverManager; +import java.sql.ResultSet; +import java.sql.SQLException; +import java.sql.Statement; +import java.util.Map; +import java.util.Properties; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; + +/// Covers [VortexDriver]: URL parsing onto [VortexCalciteOptions], rejection of malformed URLs, and +/// that `DriverManager` finds the driver through its service entry and runs real SQL. +class VortexDriverTest { + + @TempDir + static Path tmp; + private static Path dir; + private static Path other; + + @BeforeAll + static void writeFiles() throws Exception { + dir = Files.createDirectory(tmp.resolve("data")); + Fixtures.write(dir.resolve("ohlc.vortex"), 2_000); + other = tmp.resolve("other.vortex"); + Fixtures.write(other, 7); + } + + @Test + void options_directoryOnly() throws Exception { + // Given + String url = "jdbc:vortex:" + dir; + + // When + VortexCalciteOptions result = VortexDriver.options(url); + + // Then + assertThat(result.directory()).contains(dir); + assertThat(result.schemaName()).isEqualTo("vtx"); + assertThat(result.tables()).isEmpty(); + } + + @Test + void options_schemaAndExplicitTables() throws Exception { + // Given a URL with an encoded space, which an IDE produces for a path with one + String url = "jdbc:vortex:" + dir + "?schema=lake&table.trades=" + other.toString().replace(" ", "%20"); + + // When + VortexCalciteOptions result = VortexDriver.options(url); + + // Then + assertThat(result.schemaName()).isEqualTo("lake"); + assertThat(result.tables()).containsExactly(Map.entry("trades", other)); + } + + @Test + void options_tablesWithoutDirectoryAreEnough() throws Exception { + // Given + String url = "jdbc:vortex:?table.t=" + other; + + // When + VortexCalciteOptions result = VortexDriver.options(url); + + // Then + assertThat(result.directory()).isEmpty(); + assertThat(result.tables()).containsKey("t"); + } + + @ParameterizedTest + @ValueSource(strings = { + "jdbc:vortex:", // nothing to expose + "jdbc:vortex:?schema=lake", // a schema with no files + "jdbc:vortex:/d?directry=/x", // typo: silently ignoring it would expose no tables + "jdbc:vortex:/d?schema", // no '=' + "jdbc:vortex:/d?=v", // empty key + "jdbc:vortex:/d?table.=/x", // empty table name + }) + void options_rejectsMalformedUrls(String url) { + // Given / When / Then + assertThatThrownBy(() -> VortexDriver.options(url)) + .isInstanceOf(SQLException.class) + .hasMessageContaining("invalid Vortex URL") + .extracting(e -> ((SQLException) e).getSQLState()).isEqualTo("08001"); + } + + @Test + void acceptsUrl_onlyOwnScheme() { + // Given + VortexDriver sut = new VortexDriver(); + + // When / Then + assertThat(sut.acceptsURL("jdbc:vortex:/x")).isTrue(); + assertThat(sut.acceptsURL("jdbc:postgresql://h/db")).isFalse(); + assertThat(sut.acceptsURL(null)).isFalse(); + } + + @Test + void connect_returnsNullForAForeignUrl() throws Exception { + // Given + VortexDriver sut = new VortexDriver(); + + // When + Connection result = sut.connect("jdbc:postgresql://h/db", new Properties()); + + // Then the JDBC contract: another driver gets its turn + assertThat(result).isNull(); + } + + @Test + void connect_wrapsAMissingDirectoryAsSqlException() { + // Given + VortexDriver sut = new VortexDriver(); + + // When / Then + assertThatThrownBy(() -> sut.connect("jdbc:vortex:" + tmp.resolve("missing"), new Properties())) + .isInstanceOf(SQLException.class) + .hasMessageContaining("not a directory"); + } + + @Test + void driverManager_findsTheDriverAndRunsSql() throws Exception { + // Given the driver registered only through META-INF/services, as in an IDE + String url = "jdbc:vortex:" + dir + "?table.small=" + other; + + // When + Driver driver = DriverManager.getDriver(url); + long ohlc; + long small; + try (Connection conn = DriverManager.getConnection(url); + Statement st = conn.createStatement()) { + ohlc = count(st, "vtx.ohlc"); + small = count(st, "vtx.small"); + } + + // Then + assertThat(driver).isInstanceOf(VortexDriver.class); + assertThat(ohlc).isEqualTo(2_000); + assertThat(small).isEqualTo(7); + } + + @Test + void driverManager_ideStyleCredentialsAreIgnored() throws Exception { + // Given IDEs always send user and password properties + Properties info = new Properties(); + info.setProperty("user", ""); + info.setProperty("password", ""); + + // When + try (Connection conn = DriverManager.getConnection("jdbc:vortex:" + dir, info); + Statement st = conn.createStatement()) { + long result = count(st, "vtx.ohlc"); + + // Then + assertThat(result).isEqualTo(2_000); + } + } + + @Test + void reservedWordColumnsWorkUnquoted() throws Exception { + // Given the connection goes through VortexCalcite, which sets JAVA lex and the Babel parser + try (Connection conn = DriverManager.getConnection("jdbc:vortex:" + dir); + Statement st = conn.createStatement(); + ResultSet rs = st.executeQuery("select close, open from vtx.ohlc")) { + + // When + long result = 0; + while (rs.next()) { + result++; + } + + // Then + assertThat(result).isEqualTo(2_000); + } + } + + private static long count(Statement st, String table) throws SQLException { + try (ResultSet rs = st.executeQuery("select count(*) from " + table)) { + rs.next(); + return rs.getLong(1); + } + } +} diff --git a/pom.xml b/pom.xml index af66b231a..3c69602d0 100644 --- a/pom.xml +++ b/pom.xml @@ -51,6 +51,7 @@ cli inspector calcite + jdbc-driver demo integration fuzz @@ -204,6 +205,11 @@ vortex-calcite ${project.version} + + io.github.dfa1.vortex + vortex-jdbc-driver + ${project.version} + io.github.dfa1.vortex vortex-server From 91f5e4ca83ce9da35de22476aebbb96f65d7ad7a Mon Sep 17 00:00:00 2001 From: Davide Angelocola Date: Sun, 18 Oct 2026 21:02:00 +0200 Subject: [PATCH 05/10] build(jdbc-driver): leave out Calcite dependencies the driver never reaches Drops the Avatica HTTP client, YAML, sketches, Hilbert-curve and connection pooling jars: the shaded jar goes from 32.4 to 28.7 MB. Each was excluded on the calcite module's classpath and its whole SQL suite re-run; httpcore5, json-path and JTS cannot go, the planner initializes with them. Part of #566. Co-Authored-By: Claude Sonnet 5 --- docs/reference.md | 4 +++- jdbc-driver/pom.xml | 52 +++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 55 insertions(+), 1 deletion(-) diff --git a/docs/reference.md b/docs/reference.md index 5e7951a3b..9bfc76e33 100644 --- a/docs/reference.md +++ b/docs/reference.md @@ -704,7 +704,9 @@ Values are URL-decoded. An unknown key, a pair without `=`, or a URL exposing no The `-all` classifier is the jar to hand to a tool: it merges `META-INF/services` (ours and Calcite's `java.sql.Driver` entries), strips signature files and sets `Enable-Native-Access`. `ShadedJarIT` loads it alone in an isolated classloader and runs a query, so a lost service entry or a -stale signature fails `verify`. Needs JDK 25 or later. +stale signature fails `verify`. Needs JDK 25 or later. The jar leaves out the Calcite dependencies the driver never reaches +(the Avatica HTTP client, YAML, sketches, connection pooling), so Calcite model URLs +(`jdbc:calcite:model=...`) are not available through it; use `jdbc:vortex:`, or depend on `vortex-calcite`. ## CLI diff --git a/jdbc-driver/pom.xml b/jdbc-driver/pom.xml index e60da8b09..9fe5516c6 100644 --- a/jdbc-driver/pom.xml +++ b/jdbc-driver/pom.xml @@ -22,6 +22,58 @@ io.github.dfa1.vortex vortex-calcite + + + + org.apache.httpcomponents.client5 + httpclient5 + + + org.apache.httpcomponents.core5 + httpcore5-h2 + + + com.fasterxml.jackson.dataformat + jackson-dataformat-yaml + + + org.yaml + snakeyaml + + + com.google.uzaygezen + uzaygezen-core + + + com.yahoo.datasketches + sketches-core + + + com.yahoo.datasketches + memory + + + org.apache.commons + commons-dbcp2 + + + org.apache.commons + commons-pool2 + + + jakarta.transaction + jakarta.transaction-api + + + net.hydromatic + aggdesigner-algorithm + + - ALL-UNNAMED - - diff --git a/jdbc-driver/src/main/java/io/github/dfa1/vortex/jdbc/driver/VortexDriver.java b/jdbc-driver/src/main/java/io/github/dfa1/vortex/jdbc/driver/VortexDriver.java index 5fe9d0006..c2e0f7de0 100644 --- a/jdbc-driver/src/main/java/io/github/dfa1/vortex/jdbc/driver/VortexDriver.java +++ b/jdbc-driver/src/main/java/io/github/dfa1/vortex/jdbc/driver/VortexDriver.java @@ -3,6 +3,7 @@ import io.github.dfa1.vortex.calcite.VortexCalcite; import io.github.dfa1.vortex.calcite.VortexCalciteOptions; +import java.io.UncheckedIOException; import java.net.URLDecoder; import java.nio.charset.StandardCharsets; import java.nio.file.Path; @@ -25,7 +26,7 @@ /// The path is a directory whose `*.vortex` files become tables named after the file (`ohlc.vortex` /// is `vtx.ohlc`); columns come from each file, never declared. The query string takes `schema` /// (default `vtx`) and `table.=` (an explicit file, which wins over a discovered one). -/// Any other key is rejected, so a typo cannot silently expose no tables. Values are URL-decoded. +/// Any other key is rejected, so a typo cannot silently expose no tables. Values are percent-decoded (a `+` is a plus). /// /// The URL maps one-to-one onto [VortexCalciteOptions]; the connection is /// [VortexCalcite#connect(VortexCalciteOptions)], so identifiers are case-sensitive and reserved @@ -54,10 +55,10 @@ public Connection connect(String url, Properties info) throws SQLException { if (!acceptsURL(url)) { return null; } - VortexCalciteOptions options = options(url); try { - return VortexCalcite.connect(options); - } catch (IllegalArgumentException e) { + return VortexCalcite.connect(options(url)); + } catch (IllegalArgumentException | UncheckedIOException e) { + // a bad escape, an invalid path, a directory that is not one or cannot be listed throw new SQLException(e.getMessage(), "08001", e); } } @@ -93,8 +94,14 @@ static VortexCalciteOptions options(String url) throws SQLException { String key = decode(pair.substring(0, eq)); String value = decode(pair.substring(eq + 1)); if (key.equals(SCHEMA)) { + if (value.isEmpty()) { + throw invalid(url, "schema is empty"); + } options = options.withSchemaName(value); } else if (key.startsWith(TABLE_PREFIX) && key.length() > TABLE_PREFIX.length()) { + if (value.isEmpty()) { + throw invalid(url, key + " has no file"); + } options = options.withTable(key.substring(TABLE_PREFIX.length()), Path.of(value)); hasFiles = true; } else { @@ -108,8 +115,10 @@ static VortexCalciteOptions options(String url) throws SQLException { return options; } + /// Percent-decodes `value`. A `+` stays a plus: `URLDecoder` follows form encoding and would turn it + /// into a space, so `/data/c++/x` would name a directory that does not exist. private static String decode(String value) { - return URLDecoder.decode(value, StandardCharsets.UTF_8); + return URLDecoder.decode(value.replace("+", "%2B"), StandardCharsets.UTF_8); } private static SQLException invalid(String url, String reason) { diff --git a/jdbc-driver/src/test/java/io/github/dfa1/vortex/jdbc/driver/VortexDriverTest.java b/jdbc-driver/src/test/java/io/github/dfa1/vortex/jdbc/driver/VortexDriverTest.java index 4c4c513c6..84202d36f 100644 --- a/jdbc-driver/src/test/java/io/github/dfa1/vortex/jdbc/driver/VortexDriverTest.java +++ b/jdbc-driver/src/test/java/io/github/dfa1/vortex/jdbc/driver/VortexDriverTest.java @@ -87,6 +87,8 @@ void options_tablesWithoutDirectoryAreEnough() throws Exception { "jdbc:vortex:/d?schema", // no '=' "jdbc:vortex:/d?=v", // empty key "jdbc:vortex:/d?table.=/x", // empty table name + "jdbc:vortex:/d?schema=", // would register the tables under a name SQL cannot write + "jdbc:vortex:/d?table.t=", // would bind t to the working directory }) void options_rejectsMalformedUrls(String url) { // Given / When / Then @@ -96,6 +98,29 @@ void options_rejectsMalformedUrls(String url) { .extracting(e -> ((SQLException) e).getSQLState()).isEqualTo("08001"); } + @Test + void options_aPlusInAPathStaysAPlus() throws Exception { + // Given a directory whose name has a '+', pasted raw as the docs show + Path plus = Files.createDirectory(tmp.resolve("c++ data")); + + // When + VortexCalciteOptions result = VortexDriver.options("jdbc:vortex:" + plus.toString().replace(" ", "%20")); + + // Then URLDecoder alone would have turned the '+' into a space and named a directory that is not there + assertThat(result.directory()).contains(plus); + } + + @Test + void connect_aBadEscapeIsAnSqlExceptionNotAnUncheckedOne() { + // Given + VortexDriver sut = new VortexDriver(); + + // When / Then + assertThatThrownBy(() -> sut.connect("jdbc:vortex:/data/100%", new Properties())) + .isInstanceOf(SQLException.class) + .extracting(e -> ((SQLException) e).getSQLState()).isEqualTo("08001"); + } + @Test void acceptsUrl_onlyOwnScheme() { // Given From a26480a669baad378fed8f4644e224b32a9f6bca Mon Sep 17 00:00:00 2001 From: Davide Angelocola Date: Mon, 19 Oct 2026 07:21:05 +0200 Subject: [PATCH 09/10] build(integration): depend on vortex-jdbc-driver so the docs check resolves VortexDriver DocsConsistencyTest resolves the fully qualified class names the living docs mention; the integration module could not see the new driver class. Part of #566. Co-Authored-By: Claude Sonnet 5 --- integration/pom.xml | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/integration/pom.xml b/integration/pom.xml index 5badc2461..c8f4c9a17 100644 --- a/integration/pom.xml +++ b/integration/pom.xml @@ -61,6 +61,12 @@ vortex-calcite test + + + io.github.dfa1.vortex + vortex-jdbc-driver + test + dev.hardwood hardwood-core From f2e65b1ac923163c46f921593481e5f63cd03def Mon Sep 17 00:00:00 2001 From: Davide Angelocola Date: Mon, 19 Oct 2026 07:26:03 +0200 Subject: [PATCH 10/10] test(calcite): write the inline model's path with forward slashes On Windows the temp directory's backslashes read as JSON escape sequences in the inline model ('\U'). Document the same for users writing a model there. Part of #566. Co-Authored-By: Claude Sonnet 5 --- .../github/dfa1/vortex/calcite/VortexSchemaFactoryTest.java | 5 +++-- docs/reference.md | 3 ++- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/calcite/src/test/java/io/github/dfa1/vortex/calcite/VortexSchemaFactoryTest.java b/calcite/src/test/java/io/github/dfa1/vortex/calcite/VortexSchemaFactoryTest.java index 0353c623e..678cc8891 100644 --- a/calcite/src/test/java/io/github/dfa1/vortex/calcite/VortexSchemaFactoryTest.java +++ b/calcite/src/test/java/io/github/dfa1/vortex/calcite/VortexSchemaFactoryTest.java @@ -104,12 +104,13 @@ void options_resolvesRelativePathsAgainstTheModelFile() { @Test void inlineModelUrl_queriesTheDirectory() throws Exception { - // Given a URL carrying the whole configuration, as pasted into an IDE data source. A model + // Given a URL carrying the whole configuration, as pasted into an IDE data source. The path uses + // forward slashes: in a JSON model a Windows backslash starts an escape sequence ('\\U'). A model // cannot set connection properties, so lex=JAVA must be in the URL: without it Calcite // uppercases the unquoted table name and finds nothing. String url = "jdbc:calcite:lex=JAVA;model=inline:{version:'1.0',defaultSchema:'vtx',schemas:[{name:'vtx'," + "type:'custom',factory:'" + VortexSchemaFactory.class.getName() + "'," - + "operand:{directory:'" + dir + "'}}]}"; + + "operand:{directory:'" + dir.toString().replace('\\', '/') + "'}}]}"; // When long result; diff --git a/docs/reference.md b/docs/reference.md index cef320394..2027a1284 100644 --- a/docs/reference.md +++ b/docs/reference.md @@ -619,7 +619,8 @@ path, wins over a discovered file); any other key is rejected. The schema name i jdbc:calcite:lex=JAVA;model=inline:{version:'1.0',defaultSchema:'vtx',schemas:[{name:'vtx',type:'custom',factory:'io.github.dfa1.vortex.calcite.VortexSchemaFactory',operand:{directory:'/data/vortex'}}]} ``` -A model cannot set connection properties, so put `lex=JAVA` in the URL (without it Calcite +In a model, write a Windows path with forward slashes (`C:/data`) or doubled backslashes: a single backslash +starts a JSON escape sequence. A model cannot set connection properties, so put `lex=JAVA` in the URL (without it Calcite uppercases unquoted names and `vtx.ohlc` is not found). For reserved-word columns such as `close` unquoted, also add `parserFactory=org.apache.calcite.sql.parser.babel.SqlBabelParserImpl#FACTORY`; [`VortexCalcite`] sets both for you.