diff --git a/CHANGELOG.md b/CHANGELOG.md index 097b89df9..c979ac84b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,6 +8,9 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] ### Added +- `vortex-jdbc-driver` is a `jdbc:vortex:` JDBC driver in one self-contained jar, so IntelliJ, DBeaver and other SQL tools can query a directory of `.vortex` files. ([#566](https://github.com/dfa1/vortex-java/issues/566)) +- `VortexSchemaFactory` builds the Calcite schema from a `jdbc:calcite:model=...` URL, so SQL tools can query a directory of `.vortex` files without Java code. ([#566](https://github.com/dfa1/vortex-java/issues/566)) +- `VortexCalcite.connect(Path)` and `VortexCalciteOptions` expose every `.vortex` file in a directory as a SQL table. ([#566](https://github.com/dfa1/vortex-java/issues/566)) - `DecimalArray.getUnscaledLong(long)` reads a decimal's unscaled value without building a `BigDecimal`. ([#563](https://github.com/dfa1/vortex-java/pull/563)) - `Compute.groupedAggregate(Chunk, ColumnName, RowFilter, ColumnName)` folds an aggregate per value of a dictionary-encoded string, boolean or integer column. ([#555](https://github.com/dfa1/vortex-java/pull/555)) - `Chunk.slice(offset, length)` gives a zero-copy view of a row range. ([#550](https://github.com/dfa1/vortex-java/pull/550)) @@ -48,6 +51,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - Frame-of-reference encoding takes the unsigned minimum of a `u64` column whose values span 2^63 as its reference, as Rust does, instead of a reference that left those values at full width. ([#571](https://github.com/dfa1/vortex-java/pull/571)) - **Breaking:** `DType.Decimal` refuses a precision outside 1 to 76 or a scale above the precision, as Rust does, so the writer no longer writes decimal columns that Rust and vortex-java cannot read, and a variant's decimal is checked as it is read. ([#571](https://github.com/dfa1/vortex-java/pull/571)) - SQL `ORDER BY ... LIMIT` and `GROUP BY` with a few thousand result cells no longer fail with "Code grows beyond 64 KB"; past the limit Calcite scans instead. ([#569](https://github.com/dfa1/vortex-java/pull/569)) +- `VortexCalcite.connect` no longer fails with "No suitable driver" when the jar is loaded by its own classloader, as an IDE or application server does. ([#566](https://github.com/dfa1/vortex-java/issues/566)) - A file whose array FlatBuffer holds a corrupt offset fails the read with `VortexException` instead of `IndexOutOfBoundsException`. ([#493](https://github.com/dfa1/vortex-java/issues/493)) - A column whose dtype does not fit its bit-packed, delta or RLE encoding fails with `VortexException` instead of `ClassCastException`, and an empty RLE boolean chunk reads. ([#493](https://github.com/dfa1/vortex-java/issues/493)) - A struct dtype missing its field names or types, or a struct layout whose children do not match its fields, is rejected when the file opens, as Rust does, instead of failing the scan with `ArrayIndexOutOfBoundsException`. ([#493](https://github.com/dfa1/vortex-java/issues/493)) diff --git a/bom/pom.xml b/bom/pom.xml index 363808254..0540b19bb 100644 --- a/bom/pom.xml +++ b/bom/pom.xml @@ -75,6 +75,11 @@ vortex-calcite ${project.version} + + io.github.dfa1.vortex + vortex-jdbc-driver + ${project.version} + diff --git a/calcite/src/main/java/io/github/dfa1/vortex/calcite/VortexCalcite.java b/calcite/src/main/java/io/github/dfa1/vortex/calcite/VortexCalcite.java index ce564e8f0..8f9799dd3 100644 --- a/calcite/src/main/java/io/github/dfa1/vortex/calcite/VortexCalcite.java +++ b/calcite/src/main/java/io/github/dfa1/vortex/calcite/VortexCalcite.java @@ -1,12 +1,12 @@ package io.github.dfa1.vortex.calcite; import org.apache.calcite.jdbc.CalciteConnection; +import org.apache.calcite.jdbc.Driver; import org.apache.calcite.schema.SchemaPlus; import org.apache.calcite.sql.parser.babel.SqlBabelParserImpl; import java.nio.file.Path; import java.sql.Connection; -import java.sql.DriverManager; import java.sql.SQLException; import java.util.Map; import java.util.Objects; @@ -37,6 +37,9 @@ /// A column with one of those names must still be back-tick quoted: `select \`date\` from vtx.ohlc`. public final class VortexCalcite { + // One instance: building a driver loads a resource and sets up Avatica's factory by reflection. + private static final Driver CALCITE_DRIVER = new Driver(); + private VortexCalcite() { } @@ -44,8 +47,7 @@ private VortexCalcite() { /// /// Each map entry binds a SQL table name to the `.vortex` file backing it; the tables are /// reachable as `schemaName.tableName`. The returned connection is the caller's to close, which - /// releases the schema's readers. If schema registration fails the connection is closed before - /// the failure propagates, so no connection leaks. + /// releases the schema's readers. /// /// @param schemaName the schema name the tables are registered under /// @param tables map from SQL table name to the `.vortex` file path @@ -54,6 +56,30 @@ private VortexCalcite() { public static Connection connect(String schemaName, Map tables) throws SQLException { Objects.requireNonNull(schemaName, "schemaName"); Objects.requireNonNull(tables, "tables"); + return connect(VortexCalciteOptions.defaults().withSchemaName(schemaName).withTables(tables)); + } + + /// Opens a Calcite JDBC connection exposing every `*.vortex` file in `directory` as a table of + /// the default schema `vtx`, so `ohlc.vortex` is queried as `vtx.ohlc`. + /// + /// @param directory the directory to scan for `.vortex` files + /// @return an open Calcite JDBC connection with the Vortex schema registered + /// @throws SQLException if the JDBC connection cannot be opened or the schema cannot be registered + public static Connection connect(Path directory) throws SQLException { + return connect(VortexCalciteOptions.defaults().withDirectory(directory)); + } + + /// Opens a Calcite JDBC connection exposing the tables `options` describes. + /// + /// The returned connection is the caller's to close, which releases the schema's readers. If + /// schema registration fails the connection is closed before the failure propagates, so no + /// connection leaks. + /// + /// @param options the schema name and the files to expose + /// @return an open Calcite JDBC connection with the Vortex schema registered + /// @throws SQLException if the JDBC connection cannot be opened or the schema cannot be registered + public static Connection connect(VortexCalciteOptions options) throws SQLException { + Objects.requireNonNull(options, "options"); Properties info = new Properties(); // JAVA lex: case-sensitive identifiers, back-tick quoting, unquoted case preserved — so an // unquoted `ohlc` matches the registered table name exactly. @@ -61,10 +87,12 @@ public static Connection connect(String schemaName, Map tables) th // Babel parser: accept SQL reserved words (date, close, open, value, …) as identifiers, so // columnar files whose column names collide with keywords are queryable unquoted. info.setProperty("parserFactory", SqlBabelParserImpl.class.getName() + "#FACTORY"); - Connection connection = DriverManager.getConnection("jdbc:calcite:", info); + // Not DriverManager: it only sees drivers already registered, and a jar loaded by its own + // classloader (an IDE data source, an application server) has not had Calcite's registered. + Connection connection = CALCITE_DRIVER.connect("jdbc:calcite:", info); try { SchemaPlus root = connection.unwrap(CalciteConnection.class).getRootSchema(); - root.add(schemaName, new VortexSchema(tables)); + root.add(options.schemaName(), new VortexSchema(options)); return connection; } catch (SQLException | RuntimeException e) { try { diff --git a/calcite/src/main/java/io/github/dfa1/vortex/calcite/VortexCalciteOptions.java b/calcite/src/main/java/io/github/dfa1/vortex/calcite/VortexCalciteOptions.java new file mode 100644 index 000000000..0d96d61b4 --- /dev/null +++ b/calcite/src/main/java/io/github/dfa1/vortex/calcite/VortexCalciteOptions.java @@ -0,0 +1,156 @@ +package io.github.dfa1.vortex.calcite; + +import java.io.IOException; +import java.io.UncheckedIOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.Collections; +import java.util.LinkedHashMap; +import java.util.Map; +import java.util.Objects; +import java.util.Optional; +import java.util.stream.Stream; + +/// Settings for [VortexCalcite#connect(VortexCalciteOptions)]: which schema name to register and +/// which `.vortex` files become tables. +/// +/// Start from [#defaults()] and adjust with the `withXxx` methods, each of which returns a copy: +/// +/// ```java +/// VortexCalciteOptions options = VortexCalciteOptions.defaults() +/// .withDirectory(Path.of("/data/vortex")) // every *.vortex becomes a table +/// .withTable("trades", Path.of("/other/t.vortex")); +/// ``` +/// +/// A table is named after its file without the `.vortex` extension (`ohlc.vortex` is `ohlc`; hidden +/// files, whose name starts with a dot, are skipped); an +/// explicit [#withTable(String, Path)] entry wins over a discovered file of the same name. Columns +/// are never declared: they come from each file's embedded dtype. +public final class VortexCalciteOptions { + + private static final String DEFAULT_SCHEMA_NAME = "vtx"; + private static final String EXTENSION = ".vortex"; + + private final String schemaName; + private final Path directory; + private final Map tables; + + private VortexCalciteOptions(String schemaName, Path directory, Map tables) { + this.schemaName = schemaName; + this.directory = directory; + this.tables = tables; + } + + /// Returns the starting point: schema name `vtx`, no directory, no explicit tables. + /// + /// @return the default options + public static VortexCalciteOptions defaults() { + return new VortexCalciteOptions(DEFAULT_SCHEMA_NAME, null, Map.of()); + } + + /// Returns a copy registering the tables under `schemaName`. + /// + /// @param schemaName the schema name + /// @return the updated options + public VortexCalciteOptions withSchemaName(String schemaName) { + Objects.requireNonNull(schemaName, "schemaName"); + if (schemaName.isEmpty()) { + throw new IllegalArgumentException("schemaName is empty"); + } + return new VortexCalciteOptions(schemaName, directory, tables); + } + + /// Returns a copy that exposes every `*.vortex` file directly inside `directory` as a table. + /// + /// The directory is scanned when the schema is created, so a file added later appears only + /// on the next connection. + /// + /// @param directory the directory to scan + /// @return the updated options + public VortexCalciteOptions withDirectory(Path directory) { + Objects.requireNonNull(directory, "directory"); + return new VortexCalciteOptions(schemaName, directory, tables); + } + + /// Returns a copy that also binds the SQL table `name` to `file`. + /// + /// @param name the SQL table name + /// @param file the `.vortex` file backing it + /// @return the updated options + public VortexCalciteOptions withTable(String name, Path file) { + Objects.requireNonNull(name, "name"); + Objects.requireNonNull(file, "file"); + return withTables(Map.of(name, file)); + } + + /// Returns a copy that also binds every entry of `tables`, as [#withTable(String, Path)] would. + /// Entries keep the order they were added in. + /// + /// @param tables map from SQL table name to the `.vortex` file path + /// @return the updated options + public VortexCalciteOptions withTables(Map tables) { + Objects.requireNonNull(tables, "tables"); + Map copy = new LinkedHashMap<>(this.tables); + tables.forEach((name, file) -> { + Objects.requireNonNull(name, "name"); + Objects.requireNonNull(file, "file"); + if (name.isEmpty()) { + throw new IllegalArgumentException("table name is empty"); + } + copy.put(name, file); + }); + return new VortexCalciteOptions(schemaName, directory, Collections.unmodifiableMap(copy)); + } + + /// Returns the schema name the tables are registered under. + /// + /// @return the schema name + public String schemaName() { + return schemaName; + } + + /// Returns the directory scanned for `*.vortex` files, if one was set. + /// + /// @return the directory + public Optional directory() { + return Optional.ofNullable(directory); + } + + /// Returns the explicit table bindings, not including files found in the directory. + /// + /// @return an unmodifiable map from SQL table name to file + public Map tables() { + return tables; + } + + /// Resolves the full table set: files found in the directory, then the explicit bindings on top. + /// + /// @return map from SQL table name to the `.vortex` file path + /// @throws IllegalArgumentException if the directory is not a directory + /// @throws UncheckedIOException if the directory cannot be listed + Map resolveTables() { + Map resolved = new LinkedHashMap<>(); + if (directory != null) { + if (!Files.isDirectory(directory)) { + throw new IllegalArgumentException("not a directory: " + directory); + } + try (Stream entries = Files.list(directory)) { + entries.filter(Files::isRegularFile) + // a dot-prefixed name is not a table: macOS leaves ._x.vortex sidecars on exFAT/SMB + // volumes, and a writer may keep its unfinished file hidden. Such a file has no + // footer, and one table that cannot be read breaks introspection of all of them + .filter(p -> p.getFileName().toString().endsWith(EXTENSION) + && !p.getFileName().toString().startsWith(".")) + .sorted() + .forEach(p -> { + String name = p.getFileName().toString(); + resolved.put(name.substring(0, name.length() - EXTENSION.length()), p); + }); + } catch (IOException e) { + throw new UncheckedIOException("cannot list " + directory, e); + } + } + resolved.putAll(tables); + return resolved; + } +} diff --git a/calcite/src/main/java/io/github/dfa1/vortex/calcite/VortexSchema.java b/calcite/src/main/java/io/github/dfa1/vortex/calcite/VortexSchema.java index df8be495b..19bc336ae 100644 --- a/calcite/src/main/java/io/github/dfa1/vortex/calcite/VortexSchema.java +++ b/calcite/src/main/java/io/github/dfa1/vortex/calcite/VortexSchema.java @@ -22,7 +22,14 @@ public final class VortexSchema extends AbstractSchema { /// /// @param files map from table name to the `.vortex` file path public VortexSchema(Map files) { - this.tables = files.entrySet().stream() + this(VortexCalciteOptions.defaults().withTables(files)); + } + + /// Creates a schema from `options`: the files found in its directory plus its explicit tables. + /// + /// @param options which files become tables + public VortexSchema(VortexCalciteOptions options) { + this.tables = options.resolveTables().entrySet().stream() .collect(java.util.stream.Collectors.toUnmodifiableMap( Map.Entry::getKey, e -> new VortexTable(e.getValue()))); } diff --git a/calcite/src/main/java/io/github/dfa1/vortex/calcite/VortexSchemaFactory.java b/calcite/src/main/java/io/github/dfa1/vortex/calcite/VortexSchemaFactory.java new file mode 100644 index 000000000..236a96b34 --- /dev/null +++ b/calcite/src/main/java/io/github/dfa1/vortex/calcite/VortexSchemaFactory.java @@ -0,0 +1,102 @@ +package io.github.dfa1.vortex.calcite; + +import org.apache.calcite.schema.Schema; +import org.apache.calcite.schema.SchemaFactory; +import org.apache.calcite.schema.SchemaPlus; + +import java.io.File; +import java.nio.file.Path; +import java.util.LinkedHashSet; +import java.util.Map; +import java.util.Set; + +/// Builds a [VortexSchema] from a Calcite model, for `jdbc:calcite:model=...` URLs and tools that +/// only take a URL: +/// +/// ```json +/// { "version": "1.0", "defaultSchema": "vtx", +/// "schemas": [{ "name": "vtx", "type": "custom", +/// "factory": "io.github.dfa1.vortex.calcite.VortexSchemaFactory", +/// "operand": { "directory": "/data/vortex", +/// "files": { "trades": "/other/trades.vortex" } } }] } +/// ``` +/// +/// The schema name is the model's `name`. `operand` takes the same two settings as +/// [VortexCalciteOptions]: `directory` (every `*.vortex` in it becomes a table) and `files` (a +/// table name to path map that wins over a discovered file); at least one is required, and any other key is +/// rejected, so a typo does not silently expose no tables. A relative path is resolved against the +/// model file's directory. +/// +/// A model cannot set connection properties. To query reserved-word columns such as `close` +/// unquoted, add `lex=JAVA;parserFactory=org.apache.calcite.sql.parser.babel.SqlBabelParserImpl#FACTORY` +/// to the URL, or connect through [VortexCalcite], which sets both. +public final class VortexSchemaFactory implements SchemaFactory { + + private static final String DIRECTORY = "directory"; + // Not "tables": Calcite fills that key with the model's own (empty) table list. + private static final String FILES = "files"; + // Calcite adds these to every custom schema's operand; they are not ours to reject. + private static final Set CALCITE_KEYS = Set.of("modelUri", "baseDirectory", "tables"); + + /// Creates the factory Calcite instantiates by class name. + public VortexSchemaFactory() { + } + + @Override + public Schema create(SchemaPlus parent, String name, Map operand) { + return new VortexSchema(options(name, operand)); + } + + static VortexCalciteOptions options(String name, Map operand) { + Set unknown = new LinkedHashSet<>(operand.keySet()); + unknown.removeAll(Set.of(DIRECTORY, FILES)); + unknown.removeAll(CALCITE_KEYS); + if (!unknown.isEmpty()) { + throw new IllegalArgumentException( + "unknown VortexSchemaFactory operand " + unknown + "; expected " + DIRECTORY + ", " + FILES); + } + if (operand.get("tables") instanceof Map) { + // Calcite fills "tables" with the model's own list, so a map there is the user's, written by + // analogy with VortexCalciteOptions#tables; silently dropping it would expose no tables + throw new IllegalArgumentException("operand 'tables' is not supported; name the files in '" + FILES + "'"); + } + // relative paths in a model file are relative to the file, as in Calcite's own file adapters + Path base = baseDirectory(operand.get("baseDirectory")); + VortexCalciteOptions options = VortexCalciteOptions.defaults().withSchemaName(name); + boolean hasFiles = false; + if (operand.get(DIRECTORY) instanceof String directory) { + options = options.withDirectory(resolve(base, directory)); + hasFiles = true; + } else if (operand.containsKey(DIRECTORY)) { + throw new IllegalArgumentException("operand '" + DIRECTORY + "' must be a string"); + } + if (operand.get(FILES) instanceof Map files) { + for (Map.Entry entry : files.entrySet()) { + if (!(entry.getKey() instanceof String table) || !(entry.getValue() instanceof String file)) { + throw new IllegalArgumentException("operand '" + FILES + "' must map table names to path strings"); + } + options = options.withTable(table, resolve(base, file)); + hasFiles = true; + } + } else if (operand.containsKey(FILES)) { + throw new IllegalArgumentException("operand '" + FILES + "' must be an object"); + } + if (!hasFiles) { + throw new IllegalArgumentException("operand needs '" + DIRECTORY + "' or '" + FILES + "'"); + } + return options; + } + + private static Path baseDirectory(Object value) { + return switch (value) { + case null -> null; + case File file -> file.toPath(); + default -> Path.of(value.toString()); + }; + } + + private static Path resolve(Path base, String path) { + Path result = Path.of(path); + return base == null ? result : base.resolve(result); + } +} diff --git a/calcite/src/test/java/io/github/dfa1/vortex/calcite/VortexCalciteOptionsTest.java b/calcite/src/test/java/io/github/dfa1/vortex/calcite/VortexCalciteOptionsTest.java new file mode 100644 index 000000000..590a647d5 --- /dev/null +++ b/calcite/src/test/java/io/github/dfa1/vortex/calcite/VortexCalciteOptionsTest.java @@ -0,0 +1,197 @@ +package io.github.dfa1.vortex.calcite; + +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; + +import java.nio.file.Files; +import java.nio.file.Path; +import java.sql.Connection; +import java.sql.ResultSet; +import java.sql.Statement; +import java.util.Map; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatIllegalArgumentException; +import static org.assertj.core.api.Assertions.assertThatNullPointerException; + +/// Covers [VortexCalciteOptions] and directory discovery through [VortexCalcite#connect(Path)]: +/// a table is named after its file, columns come from the file's own dtype, and an explicit +/// binding wins over a discovered file. +class VortexCalciteOptionsTest { + + @TempDir + static Path tmp; + private static Path dir; + private static Path other; + + @BeforeAll + static void writeFiles() throws Exception { + dir = Files.createDirectory(tmp.resolve("data")); + // Different row counts tell the discovered file from the explicitly bound one. + OhlcGenerator.write(dir.resolve("ohlc.vortex"), 20_000, 10_000); + OhlcGenerator.write(dir.resolve("small.vortex"), 100, 100); + Files.writeString(dir.resolve("notes.txt"), "not a vortex file"); + other = tmp.resolve("other.vortex"); + OhlcGenerator.write(other, 7, 7); + } + + @Test + void defaults_useVtxSchemaAndNoTables() { + // Given / When + VortexCalciteOptions result = VortexCalciteOptions.defaults(); + + // Then + assertThat(result.schemaName()).isEqualTo("vtx"); + assertThat(result.directory()).isEmpty(); + assertThat(result.tables()).isEmpty(); + } + + @Test + void withMethods_returnCopiesAndLeaveTheOriginalUntouched() { + // Given + VortexCalciteOptions sut = VortexCalciteOptions.defaults(); + + // When + VortexCalciteOptions result = sut.withSchemaName("s").withDirectory(dir).withTable("t", other); + + // Then + assertThat(sut.schemaName()).isEqualTo("vtx"); + assertThat(sut.directory()).isEmpty(); + assertThat(sut.tables()).isEmpty(); + assertThat(result.schemaName()).isEqualTo("s"); + assertThat(result.directory()).contains(dir); + assertThat(result.tables()).containsExactly(Map.entry("t", other)); + } + + @Test + void resolveTables_namesTablesAfterFilesAndIgnoresOtherExtensions() { + // Given + VortexCalciteOptions sut = VortexCalciteOptions.defaults().withDirectory(dir); + + // When + Map result = sut.resolveTables(); + + // Then + assertThat(result).containsOnlyKeys("ohlc", "small"); + } + + @Test + void resolveTables_explicitTableWinsOverDiscoveredFile() { + // Given a directory holding ohlc.vortex and an explicit binding for the same name + VortexCalciteOptions sut = VortexCalciteOptions.defaults().withDirectory(dir).withTable("ohlc", other); + + // When + Map result = sut.resolveTables(); + + // Then + assertThat(result).containsEntry("ohlc", other).containsKey("small"); + } + + @Test + void resolveTables_rejectsAPathThatIsNotADirectory() { + // Given + VortexCalciteOptions sut = VortexCalciteOptions.defaults().withDirectory(other); + + // When / Then + assertThatIllegalArgumentException().isThrownBy(sut::resolveTables).withMessageContaining("not a directory"); + } + + @Test + void resolveTables_skipsHiddenFilesSoOneStrayFileCannotBreakIntrospection() throws Exception { + // Given a macOS AppleDouble sidecar next to a real table, as an exFAT or SMB volume gets + Path withSidecar = Files.createDirectory(tmp.resolve("sidecar")); + OhlcGenerator.write(withSidecar.resolve("ohlc.vortex"), 100, 100); + Files.writeString(withSidecar.resolve("._ohlc.vortex"), "resource fork, not a Vortex file"); + + // When + Map tables = VortexCalciteOptions.defaults().withDirectory(withSidecar).resolveTables(); + long columns = 0; + try (Connection conn = VortexCalcite.connect(withSidecar); + ResultSet rs = conn.getMetaData().getColumns(null, "vtx", "%", "%")) { + while (rs.next()) { + columns++; + } + } + + // Then the sidecar is no table, and listing every table's columns still works + assertThat(tables).containsOnlyKeys("ohlc"); + assertThat(columns).isPositive(); + } + + @Test + void withTables_keepsTheOrderTheEntriesWereAdded() { + // Given + Map ordered = new java.util.LinkedHashMap<>(); + ordered.put("z", other); + ordered.put("a", other); + ordered.put("m", other); + + // When + VortexCalciteOptions result = VortexCalciteOptions.defaults().withTables(ordered); + + // Then + assertThat(result.tables().keySet()).containsExactly("z", "a", "m"); + } + + @Test + void withMethods_rejectEmptyNames() { + // Given + VortexCalciteOptions sut = VortexCalciteOptions.defaults(); + + // When / Then + assertThatIllegalArgumentException().isThrownBy(() -> sut.withSchemaName("")); + assertThatIllegalArgumentException().isThrownBy(() -> sut.withTable("", other)); + } + + @Test + void withMethods_rejectNulls() { + // Given + VortexCalciteOptions sut = VortexCalciteOptions.defaults(); + + // When / Then + assertThatNullPointerException().isThrownBy(() -> sut.withSchemaName(null)); + assertThatNullPointerException().isThrownBy(() -> sut.withDirectory(null)); + assertThatNullPointerException().isThrownBy(() -> sut.withTable(null, other)); + assertThatNullPointerException().isThrownBy(() -> sut.withTable("t", null)); + } + + @Test + void connectDirectory_queriesEveryDiscoveredFileWithNoDeclaredColumns() throws Exception { + // Given a directory connection: no table map, no column list + try (Connection conn = VortexCalcite.connect(dir); + Statement st = conn.createStatement()) { + + // When both discovered tables are counted + long ohlc = count(st, "vtx.ohlc"); + long small = count(st, "vtx.small"); + + // Then each is named after its file and holds that file's rows + assertThat(ohlc).isEqualTo(20_000); + assertThat(small).isEqualTo(100); + } + } + + @Test + void connectOptions_usesTheConfiguredSchemaNameAndExplicitOverride() throws Exception { + // Given + VortexCalciteOptions options = VortexCalciteOptions.defaults() + .withSchemaName("data").withDirectory(dir).withTable("ohlc", other); + + // When + try (Connection conn = VortexCalcite.connect(options); + Statement st = conn.createStatement()) { + long result = count(st, "data.ohlc"); + + // Then the explicit 7-row file shadowed the discovered 20,000-row one + assertThat(result).isEqualTo(7); + } + } + + private static long count(Statement st, String table) throws Exception { + try (ResultSet rs = st.executeQuery("select count(*) from " + table)) { + rs.next(); + return rs.getLong(1); + } + } +} diff --git a/calcite/src/test/java/io/github/dfa1/vortex/calcite/VortexSchemaFactoryTest.java b/calcite/src/test/java/io/github/dfa1/vortex/calcite/VortexSchemaFactoryTest.java new file mode 100644 index 000000000..678cc8891 --- /dev/null +++ b/calcite/src/test/java/io/github/dfa1/vortex/calcite/VortexSchemaFactoryTest.java @@ -0,0 +1,127 @@ +package io.github.dfa1.vortex.calcite; + +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; + +import java.nio.file.Files; +import java.nio.file.Path; +import java.sql.Connection; +import java.sql.DriverManager; +import java.sql.ResultSet; +import java.sql.Statement; +import java.util.Map; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatIllegalArgumentException; + +/// Covers [VortexSchemaFactory]: the model `operand` maps onto [VortexCalciteOptions] with the same +/// rules as the Java API, and a plain `jdbc:calcite:model=inline:` URL, as an IDE would use, works. +class VortexSchemaFactoryTest { + + @TempDir + static Path tmp; + private static Path dir; + private static Path other; + + @BeforeAll + static void writeFiles() throws Exception { + dir = Files.createDirectory(tmp.resolve("data")); + OhlcGenerator.write(dir.resolve("ohlc.vortex"), 20_000, 10_000); + other = tmp.resolve("other.vortex"); + OhlcGenerator.write(other, 7, 7); + } + + @Test + void options_mapsDirectoryFilesAndTheModelSchemaName() { + // Given + Map operand = Map.of("directory", dir.toString(), "files", Map.of("extra", other.toString())); + + // When + VortexCalciteOptions result = VortexSchemaFactory.options("lake", operand); + + // Then + assertThat(result.schemaName()).isEqualTo("lake"); + assertThat(result.directory()).contains(dir); + assertThat(result.tables()).containsExactly(Map.entry("extra", other)); + } + + @Test + void options_rejectsAnUnknownKeyInsteadOfIgnoringATypo() { + // Given a misspelled key: silently ignoring it would expose no tables at all + Map operand = Map.of("directry", dir.toString()); + + // When / Then + assertThatIllegalArgumentException().isThrownBy(() -> VortexSchemaFactory.options("vtx", operand)) + .withMessageContaining("directry"); + } + + @Test + void options_rejectsWrongValueTypes() { + // Given / When / Then + assertThatIllegalArgumentException() + .isThrownBy(() -> VortexSchemaFactory.options("vtx", Map.of("directory", 1))); + assertThatIllegalArgumentException() + .isThrownBy(() -> VortexSchemaFactory.options("vtx", Map.of("files", "x"))); + assertThatIllegalArgumentException() + .isThrownBy(() -> VortexSchemaFactory.options("vtx", Map.of("files", Map.of("t", 1)))); + } + + @Test + void options_rejectsATablesMapInsteadOfSilentlyDroppingIt() { + // Given the key a user guesses from VortexCalciteOptions#tables; Calcite fills 'tables' itself + // with a list, so the factory only sees the user's by its type + Map operand = Map.of("tables", Map.of("trades", other.toString())); + + // When / Then + assertThatIllegalArgumentException().isThrownBy(() -> VortexSchemaFactory.options("vtx", operand)) + .withMessageContaining("files"); + } + + @Test + void options_rejectsAnOperandThatNamesNoFiles() { + // Given Calcite's own keys only, as for a schema with no operand at all + Map operand = Map.of("tables", java.util.List.of(), "modelUri", "inline:{}"); + + // When / Then + assertThatIllegalArgumentException().isThrownBy(() -> VortexSchemaFactory.options("vtx", operand)) + .withMessageContaining("directory"); + } + + @Test + void options_resolvesRelativePathsAgainstTheModelFile() { + // Given a model in tmp with a relative directory and file, run from some other working directory + Map operand = Map.of("directory", "data", "files", Map.of("t", "other.vortex"), + "baseDirectory", tmp.toFile()); + + // When + VortexCalciteOptions result = VortexSchemaFactory.options("vtx", operand); + + // Then + assertThat(result.directory()).contains(tmp.resolve("data")); + assertThat(result.tables()).containsExactly(Map.entry("t", tmp.resolve("other.vortex"))); + } + + @Test + void inlineModelUrl_queriesTheDirectory() throws Exception { + // Given a URL carrying the whole configuration, as pasted into an IDE data source. The path uses + // forward slashes: in a JSON model a Windows backslash starts an escape sequence ('\\U'). A model + // cannot set connection properties, so lex=JAVA must be in the URL: without it Calcite + // uppercases the unquoted table name and finds nothing. + String url = "jdbc:calcite:lex=JAVA;model=inline:{version:'1.0',defaultSchema:'vtx',schemas:[{name:'vtx'," + + "type:'custom',factory:'" + VortexSchemaFactory.class.getName() + "'," + + "operand:{directory:'" + dir.toString().replace('\\', '/') + "'}}]}"; + + // When + long result; + try (Connection conn = DriverManager.getConnection(url); + Statement st = conn.createStatement(); + ResultSet rs = st.executeQuery("select count(*) from ohlc")) { + rs.next(); + result = rs.getLong(1); + } + + // Then + assertThat(result).isEqualTo(20_000); + } +} diff --git a/docs/how-to.md b/docs/how-to.md index b25ca567f..74b8e7ea3 100644 --- a/docs/how-to.md +++ b/docs/how-to.md @@ -572,6 +572,15 @@ try (Connection conn = VortexCalcite.connect("vtx", tables)) { } ``` +Or point at a directory — every `*.vortex` file in it becomes a table named after the file, and the +columns come from each file's own schema: + +```java +try (Connection conn = VortexCalcite.connect(Path.of("/data/vortex"))) { + // select * from vtx.ohlc (ohlc.vortex) +} +``` + Whole-table `min`/`max`/`sum` aggregates are answered straight from zone-map statistics — no data segment is decoded — on files with the legacy `vortex.stats` zone map (written with `WriteOptions.defaults().withEdition(Editions.CORE_2025_10_0)`). Files with the default @@ -583,6 +592,73 @@ policy. --- +## Query Vortex files from an SQL tool (IntelliJ, DBeaver) + +`vortex-jdbc-driver` is a JDBC driver: one self-contained jar (Calcite and the Vortex modules +inside), so a tool that only takes a driver jar and a URL can browse and query `.vortex` files. It is +an embedded engine like DuckDB or SQLite: the query runs inside the tool's JVM, which must be JDK 25 +or later. It is read-only. + +### Set it up in IntelliJ + +1. **Get the jar.** Download `vortex-jdbc-driver--all.jar` from Maven Central, or build it: + `./mvnw package -pl jdbc-driver -am -DskipTests` leaves it in `jdbc-driver/target/`. Use the + `-all` jar, not the plain one. +2. **Add the driver.** Open the *Database* tool window, then *+* → *Driver*. Name it `Vortex`. Under + *Driver Files* click *+* → *Custom JARs* and pick the jar. Set *Class* to + `io.github.dfa1.vortex.jdbc.driver.VortexDriver` (it appears in the dropdown once the jar is + added). Apply. +3. **Add a data source.** *+* → *Data Source* → *Vortex*. Set the URL (see below), leave user and + password empty, and press *Test Connection*. +4. **Browse.** In the schema list tick `vtx`. Each `.vortex` file is a table, with its columns and + types. Double-click a table for the data grid, or open a query console on the data source and run SQL. + +DBeaver is the same: *Database* → *Driver Manager* → *New* with the same jar and class, then a +connection with the URL. + +### The URL + +``` +jdbc:vortex:/data/vortex +jdbc:vortex:/data/vortex?schema=lake&table.trades=/other/trades.vortex +``` + +Every `*.vortex` file in the directory becomes a table named after the file (`ohlc.vortex` is +`vtx.ohlc`); columns come from the file, so there is nothing to declare. `schema` renames `vtx`, and +`table.=` adds a file from elsewhere or overrides a discovered one. Any other key is an +error, so a typo cannot silently expose no tables. Values are URL-decoded. + +The directory is scanned when the connection opens, so a file added later needs a reconnect (and a +*Synchronize* in the Database tool window). + +### Writing queries + +```sql +select symbol, count(*), max(high) from vtx.ohlc group by symbol order by 2 desc; +select close, open from vtx.ohlc where `date` > 20000 limit 100; +``` + +- **Quote with back-ticks, not double quotes.** Identifiers are case-sensitive, and `"x"` is not an + identifier. `date`, `time`, `timestamp` and `interval` still need back-ticks even as column names; + other reserved words such as `close`, `open` and `value` work unquoted. +- **Filters and `LIMIT` are cheap.** A `WHERE` on a clustered column skips whole chunks using the + files' zone maps, and the data grid only reads the page it shows. +- **A sort or grouping that returns thousands of values is handed to Calcite's own engine** instead + of being answered from the file's statistics. The result is the same, only slower. A huge result in + the grid is still paged. + +### Troubleshooting + +| Symptom | Cause | +|---|---| +| `class file has wrong version` or `UnsupportedClassVersionError` | The JDBC process runs on a JDK older than 25. Set the project or IDE SDK to 25+. | +| `invalid Vortex URL ... unknown key` | A misspelled option; the message lists the keys that exist. | +| `Object 'OHLC' not found ... did you mean 'ohlc'?` | Names are case-sensitive; write `ohlc` as the file is named. | +| `parse failed: Encountered ". \""` | A double-quoted identifier; use back-ticks. | + +On JDK 25 the first query prints a `sun.misc.Unsafe` warning from protobuf, a Calcite dependency; +it is harmless. + ## Query a Vortex file with DuckDB [DuckDB](https://duckdb.org) reads Vortex files through its `vortex` extension, which is built from the Rust reference diff --git a/docs/reference.md b/docs/reference.md index cda5fe8a5..2027a1284 100644 --- a/docs/reference.md +++ b/docs/reference.md @@ -605,6 +605,25 @@ Calcite](https://calcite.apache.org/). `connect(String, Map)` fold | Method | Notes | |--------------------------------------------------------------------|------------------------------------------------------------------------| | `static connect(String schemaName, Map tables)` | Opens a Calcite JDBC connection with `tables` registered under `schemaName` (`schemaName.tableName` in SQL); connection is the caller's to close | +| `static connect(Path directory)` | Every `*.vortex` file in `directory` becomes a table of schema `vtx`, named after the file (`ohlc.vortex` is `vtx.ohlc`); scanned once at connect time | +| `static connect(VortexCalciteOptions options)` | The general form: schema name, a directory to scan, and explicit `withTable(name, path)` bindings that win over a discovered file of the same name | + +### `VortexSchemaFactory` (`io.github.dfa1.vortex.calcite.VortexSchemaFactory`) + +Builds the schema from a Calcite model, so a `jdbc:calcite:` URL (or an IDE data source) needs no +Java code. `operand` takes `directory` (every `*.vortex` becomes a table) and `files` (table name to +path, wins over a discovered file); any other key is rejected. The schema name is the model's +`name`. + +``` +jdbc:calcite:lex=JAVA;model=inline:{version:'1.0',defaultSchema:'vtx',schemas:[{name:'vtx',type:'custom',factory:'io.github.dfa1.vortex.calcite.VortexSchemaFactory',operand:{directory:'/data/vortex'}}]} +``` + +In a model, write a Windows path with forward slashes (`C:/data`) or doubled backslashes: a single backslash +starts a JSON escape sequence. A model cannot set connection properties, so put `lex=JAVA` in the URL (without it Calcite +uppercases unquoted names and `vtx.ohlc` is not found). For reserved-word columns such as `close` +unquoted, also add `parserFactory=org.apache.calcite.sql.parser.babel.SqlBabelParserImpl#FACTORY`; +[`VortexCalcite`] sets both for you. The connection uses Calcite's `JAVA` lexical policy (case-sensitive identifiers, unquoted case preserved) and the **Babel** SQL parser, so a column named after a reserved word — `close`, `open`, @@ -665,6 +684,32 @@ by hand instead of going through `connect`. --- +## JDBC driver + +### `VortexDriver` (`io.github.dfa1.vortex.jdbc.driver.VortexDriver`) + +Artifact `vortex-jdbc-driver`: a `java.sql.Driver` registered through +`META-INF/services/java.sql.Driver`, for SQL tools that take a URL. It parses the URL into a +[`VortexCalciteOptions`](#vortexcalcite-iogithubdfa1vortexcalcitevortexcalcite) and connects through +`VortexCalcite.connect(options)`, so it behaves exactly like the Java API. + +| URL | Meaning | +|------------------------------------------------------|---------------------------------------------------------------------------------| +| `jdbc:vortex:` | Every `*.vortex` in `` is a table of schema `vtx`, named after the file | +| `...?schema=` | Registers the tables under `` instead of `vtx` | +| `...?table.=` | Binds one file; wins over a discovered file of the same name. `` may be empty if at least one is given | + +Values are URL-decoded. An unknown key, a pair without `=`, or a URL exposing no files throws an +`SQLException` with state `08001`. Driver properties (`user`, `password`) are ignored. + +The `-all` classifier is the jar to hand to a tool: it merges `META-INF/services` (ours and +Calcite's `java.sql.Driver` entries), and strips signature files. Reading `vortex.zstd` files calls the zstd natives through FFM, so a host +that runs with `--illegal-native-access=deny` needs `--enable-native-access=ALL-UNNAMED`; by default the JDK only warns. +`ShadedJarIT` loads it alone in an isolated classloader and runs a query, so a lost service entry or a +stale signature fails `verify`. Needs JDK 25 or later. The jar leaves out the Calcite dependencies the driver never reaches +(the Avatica HTTP client, YAML, sketches, connection pooling), so Calcite model URLs +(`jdbc:calcite:model=...`) are not available through it; use `jdbc:vortex:`, or depend on `vortex-calcite`. + ## CLI The `-all` uber-jar is self-contained: pure-Java code plus the native `libzstd` for all diff --git a/integration/pom.xml b/integration/pom.xml index 5badc2461..c8f4c9a17 100644 --- a/integration/pom.xml +++ b/integration/pom.xml @@ -61,6 +61,12 @@ vortex-calcite test + + + io.github.dfa1.vortex + vortex-jdbc-driver + test + dev.hardwood hardwood-core diff --git a/jdbc-driver/pom.xml b/jdbc-driver/pom.xml new file mode 100644 index 000000000..e20a3ad2e --- /dev/null +++ b/jdbc-driver/pom.xml @@ -0,0 +1,173 @@ + + + 4.0.0 + + io.github.dfa1.vortex + vortex-java + 0.17.3-SNAPSHOT + + + vortex-jdbc-driver + + vortex-jdbc-driver + JDBC driver (jdbc:vortex:) to query Vortex files with SQL from any JDBC tool, shipped as a self-contained jar. + + + + --sun-misc-unsafe-memory-access=allow + + + + + + io.github.dfa1.vortex + vortex-calcite + + + + org.apache.httpcomponents.client5 + httpclient5 + + + org.apache.httpcomponents.core5 + httpcore5-h2 + + + com.fasterxml.jackson.dataformat + jackson-dataformat-yaml + + + org.yaml + snakeyaml + + + com.google.uzaygezen + uzaygezen-core + + + com.yahoo.datasketches + sketches-core + + + com.yahoo.datasketches + memory + + + org.apache.commons + commons-dbcp2 + + + org.apache.commons + commons-pool2 + + + jakarta.transaction + jakarta.transaction-api + + + net.hydromatic + aggdesigner-algorithm + + + + + + io.github.dfa1.zstd + zstd + + + io.github.dfa1.zstd + zstd-platform + runtime + + + + io.github.dfa1.vortex + vortex-core + test-jar + test + + + io.github.dfa1.vortex + vortex-writer + test + + + org.junit.jupiter + junit-jupiter + test + + + org.assertj + assertj-core + test + + + + + + + + org.apache.maven.plugins + maven-failsafe-plugin + + + **/*IT.java + + @{argLine} --sun-misc-unsafe-memory-access=allow + + + + + integration-test + verify + + + + + + org.apache.maven.plugins + maven-shade-plugin + + + package + + shade + + + true + all + false + + + vortex-jdbc-driver + + + + + + + + *:* + + module-info.class + META-INF/*.SF + META-INF/*.DSA + META-INF/*.RSA + + + + + + + + + + diff --git a/jdbc-driver/src/main/java/io/github/dfa1/vortex/jdbc/driver/VortexDriver.java b/jdbc-driver/src/main/java/io/github/dfa1/vortex/jdbc/driver/VortexDriver.java new file mode 100644 index 000000000..c2e0f7de0 --- /dev/null +++ b/jdbc-driver/src/main/java/io/github/dfa1/vortex/jdbc/driver/VortexDriver.java @@ -0,0 +1,152 @@ +package io.github.dfa1.vortex.jdbc.driver; + +import io.github.dfa1.vortex.calcite.VortexCalcite; +import io.github.dfa1.vortex.calcite.VortexCalciteOptions; + +import java.io.UncheckedIOException; +import java.net.URLDecoder; +import java.nio.charset.StandardCharsets; +import java.nio.file.Path; +import java.sql.Connection; +import java.sql.Driver; +import java.sql.DriverManager; +import java.sql.DriverPropertyInfo; +import java.sql.SQLException; +import java.sql.SQLFeatureNotSupportedException; +import java.util.Properties; +import java.util.logging.Logger; + +/// JDBC driver that queries Vortex files with SQL, for IDEs and tools that only accept a URL: +/// +/// ``` +/// jdbc:vortex:/data/vortex +/// jdbc:vortex:/data/vortex?schema=lake&table.trades=/other/trades.vortex +/// ``` +/// +/// The path is a directory whose `*.vortex` files become tables named after the file (`ohlc.vortex` +/// is `vtx.ohlc`); columns come from each file, never declared. The query string takes `schema` +/// (default `vtx`) and `table.=` (an explicit file, which wins over a discovered one). +/// Any other key is rejected, so a typo cannot silently expose no tables. Values are percent-decoded (a `+` is a plus). +/// +/// The URL maps one-to-one onto [VortexCalciteOptions]; the connection is +/// [VortexCalcite#connect(VortexCalciteOptions)], so identifiers are case-sensitive and reserved +/// words such as `close` work unquoted. Driver properties are ignored (IDEs always pass `user` and +/// `password`). +public final class VortexDriver implements Driver { + + static final String PREFIX = "jdbc:vortex:"; + private static final String SCHEMA = "schema"; + private static final String TABLE_PREFIX = "table."; + + static { + try { + DriverManager.registerDriver(new VortexDriver()); + } catch (SQLException e) { + throw new ExceptionInInitializerError(e); + } + } + + /// Creates the driver; `DriverManager` does this through `META-INF/services/java.sql.Driver`. + public VortexDriver() { + } + + @Override + public Connection connect(String url, Properties info) throws SQLException { + if (!acceptsURL(url)) { + return null; + } + try { + return VortexCalcite.connect(options(url)); + } catch (IllegalArgumentException | UncheckedIOException e) { + // a bad escape, an invalid path, a directory that is not one or cannot be listed + throw new SQLException(e.getMessage(), "08001", e); + } + } + + @Override + public boolean acceptsURL(String url) { + return url != null && url.startsWith(PREFIX); + } + + /// Parses `url` into the options it describes. + /// + /// @param url a `jdbc:vortex:` URL + /// @return the options + /// @throws SQLException if the URL has an unknown key, a malformed pair or no files to expose + static VortexCalciteOptions options(String url) throws SQLException { + String rest = url.substring(PREFIX.length()); + int query = rest.indexOf('?'); + String path = query < 0 ? rest : rest.substring(0, query); + VortexCalciteOptions options = VortexCalciteOptions.defaults(); + boolean hasFiles = !path.isEmpty(); + if (hasFiles) { + options = options.withDirectory(Path.of(decode(path))); + } + if (query >= 0) { + for (String pair : rest.substring(query + 1).split("&")) { + if (pair.isEmpty()) { + continue; + } + int eq = pair.indexOf('='); + if (eq <= 0) { + throw invalid(url, "expected key=value, got '" + pair + "'"); + } + String key = decode(pair.substring(0, eq)); + String value = decode(pair.substring(eq + 1)); + if (key.equals(SCHEMA)) { + if (value.isEmpty()) { + throw invalid(url, "schema is empty"); + } + options = options.withSchemaName(value); + } else if (key.startsWith(TABLE_PREFIX) && key.length() > TABLE_PREFIX.length()) { + if (value.isEmpty()) { + throw invalid(url, key + " has no file"); + } + options = options.withTable(key.substring(TABLE_PREFIX.length()), Path.of(value)); + hasFiles = true; + } else { + throw invalid(url, "unknown key '" + key + "'; expected schema or table."); + } + } + } + if (!hasFiles) { + throw invalid(url, "give a directory (jdbc:vortex:/path) or at least one table.="); + } + return options; + } + + /// Percent-decodes `value`. A `+` stays a plus: `URLDecoder` follows form encoding and would turn it + /// into a space, so `/data/c++/x` would name a directory that does not exist. + private static String decode(String value) { + return URLDecoder.decode(value.replace("+", "%2B"), StandardCharsets.UTF_8); + } + + private static SQLException invalid(String url, String reason) { + return new SQLException("invalid Vortex URL '" + url + "': " + reason, "08001"); + } + + @Override + public DriverPropertyInfo[] getPropertyInfo(String url, Properties info) { + return new DriverPropertyInfo[0]; + } + + @Override + public int getMajorVersion() { + return 0; + } + + @Override + public int getMinorVersion() { + return 0; + } + + @Override + public boolean jdbcCompliant() { + return false; + } + + @Override + public Logger getParentLogger() throws SQLFeatureNotSupportedException { + throw new SQLFeatureNotSupportedException("the Vortex driver does not use java.util.logging"); + } +} diff --git a/jdbc-driver/src/main/resources/META-INF/services/java.sql.Driver b/jdbc-driver/src/main/resources/META-INF/services/java.sql.Driver new file mode 100644 index 000000000..bef5aa932 --- /dev/null +++ b/jdbc-driver/src/main/resources/META-INF/services/java.sql.Driver @@ -0,0 +1 @@ +io.github.dfa1.vortex.jdbc.driver.VortexDriver diff --git a/jdbc-driver/src/test/java/io/github/dfa1/vortex/jdbc/driver/Fixtures.java b/jdbc-driver/src/test/java/io/github/dfa1/vortex/jdbc/driver/Fixtures.java new file mode 100644 index 000000000..753570af0 --- /dev/null +++ b/jdbc-driver/src/test/java/io/github/dfa1/vortex/jdbc/driver/Fixtures.java @@ -0,0 +1,39 @@ +package io.github.dfa1.vortex.jdbc.driver; + +import io.github.dfa1.vortex.core.model.ColumnName; +import io.github.dfa1.vortex.core.model.Editions; +import io.github.dfa1.vortex.core.testing.OhlcData; +import io.github.dfa1.vortex.writer.VortexWriter; +import io.github.dfa1.vortex.writer.WriteOptions; + +import java.io.IOException; +import java.nio.channels.FileChannel; +import java.nio.file.Path; +import java.nio.file.StandardOpenOption; +import java.util.LinkedHashMap; +import java.util.Map; + +/// Writes a small single-chunk OHLC Vortex file for the driver tests. +final class Fixtures { + + private Fixtures() { + } + + static void write(Path file, int rows) throws IOException { + WriteOptions options = WriteOptions.cascading(0).withEdition(Editions.CORE_2025_10_0); + try (var channel = FileChannel.open(file, StandardOpenOption.CREATE, StandardOpenOption.WRITE); + var writer = VortexWriter.create(channel, OhlcData.SCHEMA, options)) { + for (OhlcData.Batch batch : OhlcData.generate(rows, rows)) { + Map columns = new LinkedHashMap<>(); + columns.put(ColumnName.of("date"), batch.date()); + columns.put(ColumnName.of("symbol"), batch.symbol()); + columns.put(ColumnName.of("open"), batch.open()); + columns.put(ColumnName.of("high"), batch.high()); + columns.put(ColumnName.of("low"), batch.low()); + columns.put(ColumnName.of("close"), batch.close()); + columns.put(ColumnName.of("volume"), batch.volume()); + writer.writeChunk(columns); + } + } + } +} diff --git a/jdbc-driver/src/test/java/io/github/dfa1/vortex/jdbc/driver/ShadedJarIT.java b/jdbc-driver/src/test/java/io/github/dfa1/vortex/jdbc/driver/ShadedJarIT.java new file mode 100644 index 000000000..43955aa0d --- /dev/null +++ b/jdbc-driver/src/test/java/io/github/dfa1/vortex/jdbc/driver/ShadedJarIT.java @@ -0,0 +1,82 @@ +package io.github.dfa1.vortex.jdbc.driver; + +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; + +import java.io.IOException; +import java.net.URL; +import java.net.URLClassLoader; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; +import java.sql.Connection; +import java.sql.Driver; +import java.sql.ResultSet; +import java.sql.Statement; +import java.util.Properties; +import java.util.jar.JarFile; +import java.util.stream.Stream; + +import static org.assertj.core.api.Assertions.assertThat; + +/// Exercises the shaded `-all` jar the way an IDE does: alone, with none of this module's own +/// classpath. A plain unit test cannot catch what shading loses (a clobbered `java.sql.Driver` +/// service file, leftover signature files that make the JVM reject the jar), so this runs after +/// `package`, under failsafe. The jar is loaded through a classloader whose parent is the JDK +/// platform loader, so no test dependency (Calcite, the Vortex modules) can leak in. +class ShadedJarIT { + + @TempDir + Path tmp; + + @Test + void shadedJar_mergesDriverEntriesAndCarriesNoSignatures() throws Exception { + // Given + Path jar = shadedJar(); + + // When + String services; + long signatures; + try (JarFile jarFile = new JarFile(jar.toFile())) { + services = new String(jarFile.getInputStream(jarFile.getEntry("META-INF/services/java.sql.Driver")) + .readAllBytes(), StandardCharsets.UTF_8); + signatures = jarFile.stream().filter(e -> e.getName().matches("META-INF/.*\\.(SF|RSA|DSA)")).count(); + } + + // Then ServicesResourceTransformer kept ours and Calcite's; a stale signature would make the JVM + // reject the whole jar with a SecurityException + assertThat(services).contains(VortexDriver.class.getName(), "org.apache.calcite.jdbc.Driver"); + assertThat(signatures).isZero(); + } + + @Test + void shadedJar_runsSqlWithNothingElseOnTheClasspath() throws Exception { + // Given a Vortex file and only the shaded jar on an isolated classloader + Path dir = Files.createDirectory(tmp.resolve("data")); + Fixtures.write(dir.resolve("ohlc.vortex"), 500); + URL[] urls = {shadedJar().toUri().toURL()}; + try (URLClassLoader isolated = new URLClassLoader(urls, ClassLoader.getPlatformClassLoader())) { + Driver driver = (Driver) isolated.loadClass(VortexDriver.class.getName()) + .getDeclaredConstructor().newInstance(); + + // When a query runs through it + long result; + try (Connection conn = driver.connect("jdbc:vortex:" + dir, new Properties()); + Statement st = conn.createStatement(); + ResultSet rs = st.executeQuery("select count(*) from vtx.ohlc")) { + rs.next(); + result = rs.getLong(1); + } + + // Then every dependency the query needed came from the jar + assertThat(result).isEqualTo(500); + } + } + + private static Path shadedJar() throws IOException { + try (Stream files = Files.list(Path.of("target"))) { + return files.filter(p -> p.getFileName().toString().endsWith("-all.jar")).findFirst() + .orElseThrow(() -> new AssertionError("no -all.jar in target/: run `verify`, not `test`")); + } + } +} diff --git a/jdbc-driver/src/test/java/io/github/dfa1/vortex/jdbc/driver/VortexDriverTest.java b/jdbc-driver/src/test/java/io/github/dfa1/vortex/jdbc/driver/VortexDriverTest.java new file mode 100644 index 000000000..84202d36f --- /dev/null +++ b/jdbc-driver/src/test/java/io/github/dfa1/vortex/jdbc/driver/VortexDriverTest.java @@ -0,0 +1,220 @@ +package io.github.dfa1.vortex.jdbc.driver; + +import io.github.dfa1.vortex.calcite.VortexCalciteOptions; + +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.ValueSource; + +import java.nio.file.Files; +import java.nio.file.Path; +import java.sql.Connection; +import java.sql.Driver; +import java.sql.DriverManager; +import java.sql.ResultSet; +import java.sql.SQLException; +import java.sql.Statement; +import java.util.Map; +import java.util.Properties; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; + +/// Covers [VortexDriver]: URL parsing onto [VortexCalciteOptions], rejection of malformed URLs, and +/// that `DriverManager` finds the driver through its service entry and runs real SQL. +class VortexDriverTest { + + @TempDir + static Path tmp; + private static Path dir; + private static Path other; + + @BeforeAll + static void writeFiles() throws Exception { + dir = Files.createDirectory(tmp.resolve("data")); + Fixtures.write(dir.resolve("ohlc.vortex"), 2_000); + other = tmp.resolve("other.vortex"); + Fixtures.write(other, 7); + } + + @Test + void options_directoryOnly() throws Exception { + // Given + String url = "jdbc:vortex:" + dir; + + // When + VortexCalciteOptions result = VortexDriver.options(url); + + // Then + assertThat(result.directory()).contains(dir); + assertThat(result.schemaName()).isEqualTo("vtx"); + assertThat(result.tables()).isEmpty(); + } + + @Test + void options_schemaAndExplicitTables() throws Exception { + // Given a URL with an encoded space, which an IDE produces for a path with one + String url = "jdbc:vortex:" + dir + "?schema=lake&table.trades=" + other.toString().replace(" ", "%20"); + + // When + VortexCalciteOptions result = VortexDriver.options(url); + + // Then + assertThat(result.schemaName()).isEqualTo("lake"); + assertThat(result.tables()).containsExactly(Map.entry("trades", other)); + } + + @Test + void options_tablesWithoutDirectoryAreEnough() throws Exception { + // Given + String url = "jdbc:vortex:?table.t=" + other; + + // When + VortexCalciteOptions result = VortexDriver.options(url); + + // Then + assertThat(result.directory()).isEmpty(); + assertThat(result.tables()).containsKey("t"); + } + + @ParameterizedTest + @ValueSource(strings = { + "jdbc:vortex:", // nothing to expose + "jdbc:vortex:?schema=lake", // a schema with no files + "jdbc:vortex:/d?directry=/x", // typo: silently ignoring it would expose no tables + "jdbc:vortex:/d?schema", // no '=' + "jdbc:vortex:/d?=v", // empty key + "jdbc:vortex:/d?table.=/x", // empty table name + "jdbc:vortex:/d?schema=", // would register the tables under a name SQL cannot write + "jdbc:vortex:/d?table.t=", // would bind t to the working directory + }) + void options_rejectsMalformedUrls(String url) { + // Given / When / Then + assertThatThrownBy(() -> VortexDriver.options(url)) + .isInstanceOf(SQLException.class) + .hasMessageContaining("invalid Vortex URL") + .extracting(e -> ((SQLException) e).getSQLState()).isEqualTo("08001"); + } + + @Test + void options_aPlusInAPathStaysAPlus() throws Exception { + // Given a directory whose name has a '+', pasted raw as the docs show + Path plus = Files.createDirectory(tmp.resolve("c++ data")); + + // When + VortexCalciteOptions result = VortexDriver.options("jdbc:vortex:" + plus.toString().replace(" ", "%20")); + + // Then URLDecoder alone would have turned the '+' into a space and named a directory that is not there + assertThat(result.directory()).contains(plus); + } + + @Test + void connect_aBadEscapeIsAnSqlExceptionNotAnUncheckedOne() { + // Given + VortexDriver sut = new VortexDriver(); + + // When / Then + assertThatThrownBy(() -> sut.connect("jdbc:vortex:/data/100%", new Properties())) + .isInstanceOf(SQLException.class) + .extracting(e -> ((SQLException) e).getSQLState()).isEqualTo("08001"); + } + + @Test + void acceptsUrl_onlyOwnScheme() { + // Given + VortexDriver sut = new VortexDriver(); + + // When / Then + assertThat(sut.acceptsURL("jdbc:vortex:/x")).isTrue(); + assertThat(sut.acceptsURL("jdbc:postgresql://h/db")).isFalse(); + assertThat(sut.acceptsURL(null)).isFalse(); + } + + @Test + void connect_returnsNullForAForeignUrl() throws Exception { + // Given + VortexDriver sut = new VortexDriver(); + + // When + Connection result = sut.connect("jdbc:postgresql://h/db", new Properties()); + + // Then the JDBC contract: another driver gets its turn + assertThat(result).isNull(); + } + + @Test + void connect_wrapsAMissingDirectoryAsSqlException() { + // Given + VortexDriver sut = new VortexDriver(); + + // When / Then + assertThatThrownBy(() -> sut.connect("jdbc:vortex:" + tmp.resolve("missing"), new Properties())) + .isInstanceOf(SQLException.class) + .hasMessageContaining("not a directory"); + } + + @Test + void driverManager_findsTheDriverAndRunsSql() throws Exception { + // Given the driver registered only through META-INF/services, as in an IDE + String url = "jdbc:vortex:" + dir + "?table.small=" + other; + + // When + Driver driver = DriverManager.getDriver(url); + long ohlc; + long small; + try (Connection conn = DriverManager.getConnection(url); + Statement st = conn.createStatement()) { + ohlc = count(st, "vtx.ohlc"); + small = count(st, "vtx.small"); + } + + // Then + assertThat(driver).isInstanceOf(VortexDriver.class); + assertThat(ohlc).isEqualTo(2_000); + assertThat(small).isEqualTo(7); + } + + @Test + void driverManager_ideStyleCredentialsAreIgnored() throws Exception { + // Given IDEs always send user and password properties + Properties info = new Properties(); + info.setProperty("user", ""); + info.setProperty("password", ""); + + // When + try (Connection conn = DriverManager.getConnection("jdbc:vortex:" + dir, info); + Statement st = conn.createStatement()) { + long result = count(st, "vtx.ohlc"); + + // Then + assertThat(result).isEqualTo(2_000); + } + } + + @Test + void reservedWordColumnsWorkUnquoted() throws Exception { + // Given the connection goes through VortexCalcite, which sets JAVA lex and the Babel parser + try (Connection conn = DriverManager.getConnection("jdbc:vortex:" + dir); + Statement st = conn.createStatement(); + ResultSet rs = st.executeQuery("select close, open from vtx.ohlc")) { + + // When + long result = 0; + while (rs.next()) { + result++; + } + + // Then + assertThat(result).isEqualTo(2_000); + } + } + + private static long count(Statement st, String table) throws SQLException { + try (ResultSet rs = st.executeQuery("select count(*) from " + table)) { + rs.next(); + return rs.getLong(1); + } + } +} diff --git a/pom.xml b/pom.xml index af66b231a..3c69602d0 100644 --- a/pom.xml +++ b/pom.xml @@ -51,6 +51,7 @@ cli inspector calcite + jdbc-driver demo integration fuzz @@ -204,6 +205,11 @@ vortex-calcite ${project.version} + + io.github.dfa1.vortex + vortex-jdbc-driver + ${project.version} + io.github.dfa1.vortex vortex-server