feat(jdbc-driver): jdbc:vortex: driver to query Vortex files from any JDBC tool

CHANGELOG.md

@@ -8,6 +8,9 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 ## [Unreleased]
 
 ### Added
+- `vortex-jdbc-driver` is a `jdbc:vortex:<dir>` JDBC driver in one self-contained jar, so IntelliJ, DBeaver and other SQL tools can query a directory of `.vortex` files. ([#566](https://github.com/dfa1/vortex-java/issues/566))
+- `VortexSchemaFactory` builds the Calcite schema from a `jdbc:calcite:model=...` URL, so SQL tools can query a directory of `.vortex` files without Java code. ([#566](https://github.com/dfa1/vortex-java/issues/566))
+- `VortexCalcite.connect(Path)` and `VortexCalciteOptions` expose every `.vortex` file in a directory as a SQL table. ([#566](https://github.com/dfa1/vortex-java/issues/566))
 - `DecimalArray.getUnscaledLong(long)` reads a decimal's unscaled value without building a `BigDecimal`. ([#563](https://github.com/dfa1/vortex-java/pull/563))
 - `Compute.groupedAggregate(Chunk, ColumnName, RowFilter, ColumnName)` folds an aggregate per value of a dictionary-encoded string, boolean or integer column. ([#555](https://github.com/dfa1/vortex-java/pull/555))
 - `Chunk.slice(offset, length)` gives a zero-copy view of a row range. ([#550](https://github.com/dfa1/vortex-java/pull/550))
@@ -48,6 +51,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 - Frame-of-reference encoding takes the unsigned minimum of a `u64` column whose values span 2^63 as its reference, as Rust does, instead of a reference that left those values at full width. ([#571](https://github.com/dfa1/vortex-java/pull/571))
 - **Breaking:** `DType.Decimal` refuses a precision outside 1 to 76 or a scale above the precision, as Rust does, so the writer no longer writes decimal columns that Rust and vortex-java cannot read, and a variant's decimal is checked as it is read. ([#571](https://github.com/dfa1/vortex-java/pull/571))
 - SQL `ORDER BY ... LIMIT` and `GROUP BY` with a few thousand result cells no longer fail with "Code grows beyond 64 KB"; past the limit Calcite scans instead. ([#569](https://github.com/dfa1/vortex-java/pull/569))
+- `VortexCalcite.connect` no longer fails with "No suitable driver" when the jar is loaded by its own classloader, as an IDE or application server does. ([#566](https://github.com/dfa1/vortex-java/issues/566))
 - A file whose array FlatBuffer holds a corrupt offset fails the read with `VortexException` instead of `IndexOutOfBoundsException`. ([#493](https://github.com/dfa1/vortex-java/issues/493))
 - A column whose dtype does not fit its bit-packed, delta or RLE encoding fails with `VortexException` instead of `ClassCastException`, and an empty RLE boolean chunk reads. ([#493](https://github.com/dfa1/vortex-java/issues/493))
 - A struct dtype missing its field names or types, or a struct layout whose children do not match its fields, is rejected when the file opens, as Rust does, instead of failing the scan with `ArrayIndexOutOfBoundsException`. ([#493](https://github.com/dfa1/vortex-java/issues/493))

bom/pom.xml

@@ -75,6 +75,11 @@
                 <artifactId>vortex-calcite</artifactId>
                 <version>${project.version}</version>
             </dependency>
+            <dependency>
+                <groupId>io.github.dfa1.vortex</groupId>
+                <artifactId>vortex-jdbc-driver</artifactId>
+                <version>${project.version}</version>
+            </dependency>
         </dependencies>
     </dependencyManagement>
 

calcite/src/main/java/io/github/dfa1/vortex/calcite/VortexCalcite.java

@@ -1,12 +1,12 @@
 package io.github.dfa1.vortex.calcite;
 
 import org.apache.calcite.jdbc.CalciteConnection;
+import org.apache.calcite.jdbc.Driver;
 import org.apache.calcite.schema.SchemaPlus;
 import org.apache.calcite.sql.parser.babel.SqlBabelParserImpl;
 
 import java.nio.file.Path;
 import java.sql.Connection;
-import java.sql.DriverManager;
 import java.sql.SQLException;
 import java.util.Map;
 import java.util.Objects;
@@ -37,15 +37,17 @@
 /// A column with one of those names must still be back-tick quoted: `select \`date\` from vtx.ohlc`.
 public final class VortexCalcite {
 
+    // One instance: building a driver loads a resource and sets up Avatica's factory by reflection.
+    private static final Driver CALCITE_DRIVER = new Driver();
+
     private VortexCalcite() {
     }
 
     /// Opens a Calcite JDBC connection exposing `tables` under a schema named `schemaName`.
     ///
     /// Each map entry binds a SQL table name to the `.vortex` file backing it; the tables are
     /// reachable as `schemaName.tableName`. The returned connection is the caller's to close, which
-    /// releases the schema's readers. If schema registration fails the connection is closed before
-    /// the failure propagates, so no connection leaks.
+    /// releases the schema's readers.
     ///
     /// @param schemaName the schema name the tables are registered under
     /// @param tables     map from SQL table name to the `.vortex` file path
@@ -54,17 +56,43 @@ private VortexCalcite() {
     public static Connection connect(String schemaName, Map<String, Path> tables) throws SQLException {
         Objects.requireNonNull(schemaName, "schemaName");
         Objects.requireNonNull(tables, "tables");
+        return connect(VortexCalciteOptions.defaults().withSchemaName(schemaName).withTables(tables));
+    }
+
+    /// Opens a Calcite JDBC connection exposing every `*.vortex` file in `directory` as a table of
+    /// the default schema `vtx`, so `ohlc.vortex` is queried as `vtx.ohlc`.
+    ///
+    /// @param directory the directory to scan for `.vortex` files
+    /// @return an open Calcite JDBC connection with the Vortex schema registered
+    /// @throws SQLException if the JDBC connection cannot be opened or the schema cannot be registered
+    public static Connection connect(Path directory) throws SQLException {
+        return connect(VortexCalciteOptions.defaults().withDirectory(directory));
+    }
+
+    /// Opens a Calcite JDBC connection exposing the tables `options` describes.
+    ///
+    /// The returned connection is the caller's to close, which releases the schema's readers. If
+    /// schema registration fails the connection is closed before the failure propagates, so no
+    /// connection leaks.
+    ///
+    /// @param options the schema name and the files to expose
+    /// @return an open Calcite JDBC connection with the Vortex schema registered
+    /// @throws SQLException if the JDBC connection cannot be opened or the schema cannot be registered
+    public static Connection connect(VortexCalciteOptions options) throws SQLException {
+        Objects.requireNonNull(options, "options");
         Properties info = new Properties();
         // JAVA lex: case-sensitive identifiers, back-tick quoting, unquoted case preserved — so an
         // unquoted `ohlc` matches the registered table name exactly.
         info.setProperty("lex", "JAVA");
         // Babel parser: accept SQL reserved words (date, close, open, value, …) as identifiers, so
         // columnar files whose column names collide with keywords are queryable unquoted.
         info.setProperty("parserFactory", SqlBabelParserImpl.class.getName() + "#FACTORY");
-        Connection connection = DriverManager.getConnection("jdbc:calcite:", info);
+        // Not DriverManager: it only sees drivers already registered, and a jar loaded by its own
+        // classloader (an IDE data source, an application server) has not had Calcite's registered.
+        Connection connection = CALCITE_DRIVER.connect("jdbc:calcite:", info);
         try {
             SchemaPlus root = connection.unwrap(CalciteConnection.class).getRootSchema();
-            root.add(schemaName, new VortexSchema(tables));
+            root.add(options.schemaName(), new VortexSchema(options));
             return connection;
         } catch (SQLException | RuntimeException e) {
             try {

calcite/src/main/java/io/github/dfa1/vortex/calcite/VortexCalciteOptions.java

@@ -0,0 +1,156 @@
+package io.github.dfa1.vortex.calcite;
+
+import java.io.IOException;
+import java.io.UncheckedIOException;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.util.Collections;
+import java.util.LinkedHashMap;
+import java.util.Map;
+import java.util.Objects;
+import java.util.Optional;
+import java.util.stream.Stream;
+
+/// Settings for [VortexCalcite#connect(VortexCalciteOptions)]: which schema name to register and
+/// which `.vortex` files become tables.
+///
+/// Start from [#defaults()] and adjust with the `withXxx` methods, each of which returns a copy:
+///
+/// ```java
+/// VortexCalciteOptions options = VortexCalciteOptions.defaults()
+///         .withDirectory(Path.of("/data/vortex"))      // every *.vortex becomes a table
+///         .withTable("trades", Path.of("/other/t.vortex"));
+/// ```
+///
+/// A table is named after its file without the `.vortex` extension (`ohlc.vortex` is `ohlc`; hidden
+/// files, whose name starts with a dot, are skipped); an
+/// explicit [#withTable(String, Path)] entry wins over a discovered file of the same name. Columns
+/// are never declared: they come from each file's embedded dtype.
+public final class VortexCalciteOptions {
+
+    private static final String DEFAULT_SCHEMA_NAME = "vtx";
+    private static final String EXTENSION = ".vortex";
+
+    private final String schemaName;
+    private final Path directory;
+    private final Map<String, Path> tables;
+
+    private VortexCalciteOptions(String schemaName, Path directory, Map<String, Path> tables) {
+        this.schemaName = schemaName;
+        this.directory = directory;
+        this.tables = tables;
+    }
+
+    /// Returns the starting point: schema name `vtx`, no directory, no explicit tables.
+    ///
+    /// @return the default options
+    public static VortexCalciteOptions defaults() {
+        return new VortexCalciteOptions(DEFAULT_SCHEMA_NAME, null, Map.of());
+    }
+
+    /// Returns a copy registering the tables under `schemaName`.
+    ///
+    /// @param schemaName the schema name
+    /// @return the updated options
+    public VortexCalciteOptions withSchemaName(String schemaName) {
+        Objects.requireNonNull(schemaName, "schemaName");
+        if (schemaName.isEmpty()) {
+            throw new IllegalArgumentException("schemaName is empty");
+        }
+        return new VortexCalciteOptions(schemaName, directory, tables);
+    }
+
+    /// Returns a copy that exposes every `*.vortex` file directly inside `directory` as a table.
+    ///
+    /// The directory is scanned when the schema is created, so a file added later appears only
+    /// on the next connection.
+    ///
+    /// @param directory the directory to scan
+    /// @return the updated options
+    public VortexCalciteOptions withDirectory(Path directory) {
+        Objects.requireNonNull(directory, "directory");
+        return new VortexCalciteOptions(schemaName, directory, tables);
+    }
+
+    /// Returns a copy that also binds the SQL table `name` to `file`.
+    ///
+    /// @param name the SQL table name
+    /// @param file the `.vortex` file backing it
+    /// @return the updated options
+    public VortexCalciteOptions withTable(String name, Path file) {
+        Objects.requireNonNull(name, "name");
+        Objects.requireNonNull(file, "file");
+        return withTables(Map.of(name, file));
+    }
+
+    /// Returns a copy that also binds every entry of `tables`, as [#withTable(String, Path)] would.
+    /// Entries keep the order they were added in.
+    ///
+    /// @param tables map from SQL table name to the `.vortex` file path
+    /// @return the updated options
+    public VortexCalciteOptions withTables(Map<String, Path> tables) {
+        Objects.requireNonNull(tables, "tables");
+        Map<String, Path> copy = new LinkedHashMap<>(this.tables);
+        tables.forEach((name, file) -> {
+            Objects.requireNonNull(name, "name");
+            Objects.requireNonNull(file, "file");
+            if (name.isEmpty()) {
+                throw new IllegalArgumentException("table name is empty");
+            }
+            copy.put(name, file);
+        });
+        return new VortexCalciteOptions(schemaName, directory, Collections.unmodifiableMap(copy));
+    }
+
+    /// Returns the schema name the tables are registered under.
+    ///
+    /// @return the schema name
+    public String schemaName() {
+        return schemaName;
+    }
+
+    /// Returns the directory scanned for `*.vortex` files, if one was set.
+    ///
+    /// @return the directory
+    public Optional<Path> directory() {
+        return Optional.ofNullable(directory);
+    }
+
+    /// Returns the explicit table bindings, not including files found in the directory.
+    ///
+    /// @return an unmodifiable map from SQL table name to file
+    public Map<String, Path> tables() {
+        return tables;
+    }
+
+    /// Resolves the full table set: files found in the directory, then the explicit bindings on top.
+    ///
+    /// @return map from SQL table name to the `.vortex` file path
+    /// @throws IllegalArgumentException if the directory is not a directory
+    /// @throws UncheckedIOException     if the directory cannot be listed
+    Map<String, Path> resolveTables() {
+        Map<String, Path> resolved = new LinkedHashMap<>();
+        if (directory != null) {
+            if (!Files.isDirectory(directory)) {
+                throw new IllegalArgumentException("not a directory: " + directory);
+            }
+            try (Stream<Path> entries = Files.list(directory)) {
+                entries.filter(Files::isRegularFile)
+                        // a dot-prefixed name is not a table: macOS leaves ._x.vortex sidecars on exFAT/SMB
+                        // volumes, and a writer may keep its unfinished file hidden. Such a file has no
+                        // footer, and one table that cannot be read breaks introspection of all of them
+                        .filter(p -> p.getFileName().toString().endsWith(EXTENSION)
+                                && !p.getFileName().toString().startsWith("."))
+                        .sorted()
+                        .forEach(p -> {
+                            String name = p.getFileName().toString();
+                            resolved.put(name.substring(0, name.length() - EXTENSION.length()), p);
+                        });
+            } catch (IOException e) {
+                throw new UncheckedIOException("cannot list " + directory, e);
+            }
+        }
+        resolved.putAll(tables);
+        return resolved;
+    }
+}

calcite/src/main/java/io/github/dfa1/vortex/calcite/VortexSchema.java

@@ -22,7 +22,14 @@ public final class VortexSchema extends AbstractSchema {
     ///
     /// @param files map from table name to the `.vortex` file path
     public VortexSchema(Map<String, Path> files) {
-        this.tables = files.entrySet().stream()
+        this(VortexCalciteOptions.defaults().withTables(files));
+    }
+
+    /// Creates a schema from `options`: the files found in its directory plus its explicit tables.
+    ///
+    /// @param options which files become tables
+    public VortexSchema(VortexCalciteOptions options) {
+        this.tables = options.resolveTables().entrySet().stream()
                 .collect(java.util.stream.Collectors.toUnmodifiableMap(
                         Map.Entry::getKey, e -> new VortexTable(e.getValue())));
     }

calcite/src/main/java/io/github/dfa1/vortex/calcite/VortexSchemaFactory.java

@@ -0,0 +1,102 @@
+package io.github.dfa1.vortex.calcite;
+
+import org.apache.calcite.schema.Schema;
+import org.apache.calcite.schema.SchemaFactory;
+import org.apache.calcite.schema.SchemaPlus;
+
+import java.io.File;
+import java.nio.file.Path;
+import java.util.LinkedHashSet;
+import java.util.Map;
+import java.util.Set;
+
+/// Builds a [VortexSchema] from a Calcite model, for `jdbc:calcite:model=...` URLs and tools that
+/// only take a URL:
+///
+/// ```json
+/// { "version": "1.0", "defaultSchema": "vtx",
+///   "schemas": [{ "name": "vtx", "type": "custom",
+///     "factory": "io.github.dfa1.vortex.calcite.VortexSchemaFactory",
+///     "operand": { "directory": "/data/vortex",
+///                  "files": { "trades": "/other/trades.vortex" } } }] }
+/// ```
+///
+/// The schema name is the model's `name`. `operand` takes the same two settings as
+/// [VortexCalciteOptions]: `directory` (every `*.vortex` in it becomes a table) and `files` (a
+/// table name to path map that wins over a discovered file); at least one is required, and any other key is
+/// rejected, so a typo does not silently expose no tables. A relative path is resolved against the
+/// model file's directory.
+///
+/// A model cannot set connection properties. To query reserved-word columns such as `close`
+/// unquoted, add `lex=JAVA;parserFactory=org.apache.calcite.sql.parser.babel.SqlBabelParserImpl#FACTORY`
+/// to the URL, or connect through [VortexCalcite], which sets both.
+public final class VortexSchemaFactory implements SchemaFactory {
+
+    private static final String DIRECTORY = "directory";
+    // Not "tables": Calcite fills that key with the model's own (empty) table list.
+    private static final String FILES = "files";
+    // Calcite adds these to every custom schema's operand; they are not ours to reject.
+    private static final Set<String> CALCITE_KEYS = Set.of("modelUri", "baseDirectory", "tables");
+
+    /// Creates the factory Calcite instantiates by class name.
+    public VortexSchemaFactory() {
+    }
+
+    @Override
+    public Schema create(SchemaPlus parent, String name, Map<String, Object> operand) {
+        return new VortexSchema(options(name, operand));
+    }
+
+    static VortexCalciteOptions options(String name, Map<String, Object> operand) {
+        Set<String> unknown = new LinkedHashSet<>(operand.keySet());
+        unknown.removeAll(Set.of(DIRECTORY, FILES));
+        unknown.removeAll(CALCITE_KEYS);
+        if (!unknown.isEmpty()) {
+            throw new IllegalArgumentException(
+                    "unknown VortexSchemaFactory operand " + unknown + "; expected " + DIRECTORY + ", " + FILES);
+        }
+        if (operand.get("tables") instanceof Map<?, ?>) {
+            // Calcite fills "tables" with the model's own list, so a map there is the user's, written by
+            // analogy with VortexCalciteOptions#tables; silently dropping it would expose no tables
+            throw new IllegalArgumentException("operand 'tables' is not supported; name the files in '" + FILES + "'");
+        }
+        // relative paths in a model file are relative to the file, as in Calcite's own file adapters
+        Path base = baseDirectory(operand.get("baseDirectory"));
+        VortexCalciteOptions options = VortexCalciteOptions.defaults().withSchemaName(name);
+        boolean hasFiles = false;
+        if (operand.get(DIRECTORY) instanceof String directory) {
+            options = options.withDirectory(resolve(base, directory));
+            hasFiles = true;
+        } else if (operand.containsKey(DIRECTORY)) {
+            throw new IllegalArgumentException("operand '" + DIRECTORY + "' must be a string");
+        }
+        if (operand.get(FILES) instanceof Map<?, ?> files) {
+            for (Map.Entry<?, ?> entry : files.entrySet()) {
+                if (!(entry.getKey() instanceof String table) || !(entry.getValue() instanceof String file)) {
+                    throw new IllegalArgumentException("operand '" + FILES + "' must map table names to path strings");
+                }
+                options = options.withTable(table, resolve(base, file));
+                hasFiles = true;
+            }
+        } else if (operand.containsKey(FILES)) {
+            throw new IllegalArgumentException("operand '" + FILES + "' must be an object");
+        }
+        if (!hasFiles) {
+            throw new IllegalArgumentException("operand needs '" + DIRECTORY + "' or '" + FILES + "'");
+        }
+        return options;
+    }
+
+    private static Path baseDirectory(Object value) {
+        return switch (value) {
+            case null -> null;
+            case File file -> file.toPath();
+            default -> Path.of(value.toString());
+        };
+    }
+
+    private static Path resolve(Path base, String path) {
+        Path result = Path.of(path);
+        return base == null ? result : base.resolve(result);
+    }
+}