From 16cb51d52e9f0bdd1297d945bd433a48d619b7b8 Mon Sep 17 00:00:00 2001 From: MPCoreDeveloper Date: Sun, 25 Oct 2026 10:22:03 +0100 Subject: [PATCH] fix(vector): ORDER BY vec_distance_* reaches the index instead of dying in a dead branch MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `CREATE VECTOR INDEX` and the `ORDER BY vec_distance_*(col, vector) LIMIT k` planner branch were both permanently inactive, and restoring the deleted method alone would have revived neither. `SqlParser.VectorQueryOptimizer` was an `internal static` that no code in SharpCoreDB ever assigned and no other assembly could assign (the parser is internal), while SharpCoreDB.VectorSearch registers the optimizer with the DI container. So `VectorQueryOptimizer?.BuildIndex(...)` in the DDL path and `VectorQueryOptimizer is not null` in the SELECT path were no-ops in both directions: nothing built an index and nothing looked for one. Wiring: Database.GetSharedSqlParser now resolves IVectorQueryOptimizer from the service provider — the same pattern GRAPH_RAG already used — and attaches it via SetVectorQueryOptimizer; the property is per parser instance instead of static. The second SqlParser construction site (the compiled-query fallback) now goes through GetSharedSqlParser, so falling back cannot produce a parser without the extensions the first site attaches. Restored and repaired: TryExecuteVectorOptimized was replaced by a stub returning null in 997103be, after working in e220461e. The restored code is not a snapshot of it. The old version copied every column and appended a "distance" key whatever the SELECT list said, and did so for DESC and WHERE queries too, so an optimized query and an unoptimized one returned different shapes for the same SQL. Rows are now projected onto the SELECT list (SELECT *, plain columns, aliases, and the distance expression substituted from the key the optimizer injects), ORDER BY may name either the alias or the function, and the two shapes an ascending top-k index cannot answer — DESC and a WHERE clause — fall back to the scan instead of returning a plausible wrong answer. Found while testing: an unquoted vector literal (`[1,0,0,0]`, which is exactly what ParameterBinder produces for a float[] parameter) lost its commas to the argument splitter, leaving "[1" as the query vector. Brackets and braces now count as nesting, so a bound float[] parameter works end to end. Tests: tests/SharpCoreDB.VectorSearch.Tests/SqlVectorQueryTests.cs, seven end-to-end cases through AddVectorSupport() + Database — the only level at which the wiring is observable, which is why the existing VectorQueryOptimizer unit tests stayed green throughout the outage. 255/255 in that project, 2137/2137 in SharpCoreDB.Tests. Known limitation, unchanged: an index is a snapshot of the table at CREATE VECTOR INDEX; rows inserted afterwards are not in it until it is rebuilt. --- .../Database/Core/Database.Core.cs | 1 + .../Database/Execution/Database.Execution.cs | 61 ++- src/SharpCoreDB/Services/SqlParser.Core.cs | 31 +- src/SharpCoreDB/Services/SqlParser.DDL.cs | 4 +- src/SharpCoreDB/Services/SqlParser.DML.cs | 397 +++++++++++++++++- .../SqlVectorQueryTests.cs | 217 ++++++++++ 6 files changed, 683 insertions(+), 28 deletions(-) create mode 100644 tests/SharpCoreDB.VectorSearch.Tests/SqlVectorQueryTests.cs diff --git a/src/SharpCoreDB/Database/Core/Database.Core.cs b/src/SharpCoreDB/Database/Core/Database.Core.cs index 89f99847..52012fed 100644 --- a/src/SharpCoreDB/Database/Core/Database.Core.cs +++ b/src/SharpCoreDB/Database/Core/Database.Core.cs @@ -57,6 +57,7 @@ public partial class Database : IDatabase, IDisposable, IAsyncDisposable private QueryPlanCache? planCache; // ✅ Lazy-initialized query plan cache private SqlParser? _sharedSqlParser; // ✅ Reusable SqlParser for compiled queries private IGraphRagProvider? _cachedGraphRagProvider; // v2: cached DI resolution (avoids per-call GetService) + private IVectorQueryOptimizer? _cachedVectorQueryOptimizer; // resolved once, same reason /// /// v2 (Native AOT readiness): metadata JSON options. resolves the diff --git a/src/SharpCoreDB/Database/Execution/Database.Execution.cs b/src/SharpCoreDB/Database/Execution/Database.Execution.cs index 18639a5c..4b52c430 100644 --- a/src/SharpCoreDB/Database/Execution/Database.Execution.cs +++ b/src/SharpCoreDB/Database/Execution/Database.Execution.cs @@ -31,21 +31,47 @@ private SqlParser GetSharedSqlParser() { _sharedSqlParser = new SqlParser(tables, _dbPath, storage, isReadOnly, queryCache, config); _sharedSqlParser.Database = this; + AttachOptionalProviders(_sharedSqlParser); + } - // v2: resolve the optional GRAPH_RAG provider once and cache it for the parser - // lifetime. Previously this DI lookup ran on every ExecuteSQL/ExecuteQuery call. - if (_cachedGraphRagProvider is null) - { - _cachedGraphRagProvider = _serviceProvider.GetService(typeof(IGraphRagProvider)) as IGraphRagProvider; - } + return _sharedSqlParser; + } - if (_cachedGraphRagProvider is not null) - { - _sharedSqlParser.SetGraphRagProvider(_cachedGraphRagProvider); - } + /// + /// Attaches the optional SQL extensions that live in separate assemblies and are therefore + /// resolved from the service provider instead of being referenced directly: the GRAPH_RAG + /// provider and the vector query optimizer. + /// + /// The vector optimizer used to be attached nowhere at all. SqlParser.VectorQueryOptimizer + /// was an internal static property that no code inside this assembly ever set and no other + /// assembly could set (the type is internal), so both of its consumers were permanent no-ops: + /// CREATE VECTOR INDEX built no index, and the ORDER BY vec_distance_*(...) LIMIT k + /// planner branch was never entered. Resolving here — the same + /// way the GRAPH_RAG provider is resolved — is what makes AddVectorSupport() reach the + /// planner. Both lookups are cached on the database, so a parser rebuild costs nothing when the + /// extension is registered and resolves to null only once when it is not. + /// + /// + /// The parser instance to configure. + private void AttachOptionalProviders(SqlParser parser) + { + if (_cachedGraphRagProvider is null) + { + _cachedGraphRagProvider = _serviceProvider.GetService(typeof(IGraphRagProvider)) as IGraphRagProvider; } - return _sharedSqlParser; + if (_cachedGraphRagProvider is not null) + { + parser.SetGraphRagProvider(_cachedGraphRagProvider); + } + + _cachedVectorQueryOptimizer ??= + _serviceProvider.GetService(typeof(IVectorQueryOptimizer)) as IVectorQueryOptimizer; + + if (_cachedVectorQueryOptimizer is not null) + { + parser.SetVectorQueryOptimizer(_cachedVectorQueryOptimizer); + } } @@ -469,14 +495,11 @@ public List> ExecuteCompiledQuery(DataStructures.Prep return executor.Execute(stmt.CompiledPlan, parameters); } - // ✅ Fallback: Lazy-initialize shared SqlParser (thread-safe via Interlocked) - if (_sharedSqlParser is null) - { - _sharedSqlParser = new SqlParser(tables, _dbPath, storage, isReadOnly, queryCache, config); - _sharedSqlParser.Database = this; - } - - return _sharedSqlParser.ExecuteQuery(stmt.Plan, parameters); + // ✅ Fallback: the shared parser created by GetSharedSqlParser() (which also attaches the + // optional GRAPH_RAG provider and vector optimizer). This block used to build its own + // SqlParser, so a compiled query falling back here got a parser without either extension — + // the second of the two creation sites is exactly how an extension point loses its wiring. + return GetSharedSqlParser().ExecuteQuery(stmt.Plan, parameters); } /// diff --git a/src/SharpCoreDB/Services/SqlParser.Core.cs b/src/SharpCoreDB/Services/SqlParser.Core.cs index 4ea9d69b..abe76ee9 100644 --- a/src/SharpCoreDB/Services/SqlParser.Core.cs +++ b/src/SharpCoreDB/Services/SqlParser.Core.cs @@ -118,12 +118,33 @@ internal SqlParser(Dictionary tables, string dbPath, bool isRead private List> _pendingQueryResults = []; /// - /// Optional vector query optimizer registered by the VectorSearch module. - /// When set, the query planner detects ORDER BY vec_distance_*() + LIMIT patterns - /// and routes them to a vector index instead of a full table scan. - /// Thread-safe: set once during Database initialization, read from any thread. + /// Optional vector query optimizer supplied by the VectorSearch module. + /// + /// When set, the planner detects ORDER BY vec_distance_*(col, vector) LIMIT k and + /// CREATE VECTOR INDEX routes them to a vector index instead of a full table scan. + /// + /// + /// This is a per-parser instance member, not a static one. It used to be + /// internal static, which made the extension point unreachable in practice: an + /// internal static on an internal class cannot be assigned by another assembly + /// (SharpCoreDB.VectorSearch registers with the DI + /// container, it cannot see this type), and no code inside this assembly ever assigned it + /// either. Every call site was therefore a null-conditional no-op, in both directions: + /// CREATE VECTOR INDEX built nothing and ORDER BY vec_distance_* never reached + /// the optimizer. resolves the registered optimizer from its service + /// provider and attaches it here (see Database.GetSharedSqlParser). + /// /// - internal static IVectorQueryOptimizer? VectorQueryOptimizer { get; set; } + internal IVectorQueryOptimizer? VectorQueryOptimizer { get; private set; } + + /// + /// Attaches the optional vector query optimizer for this parser instance. + /// Called by once, when the shared parser is created, with the + /// registered by AddVectorSupport() (if any). + /// + /// The optimizer to use, or for none. + internal void SetVectorQueryOptimizer(IVectorQueryOptimizer? optimizer) + => this.VectorQueryOptimizer = optimizer; /// public void Execute(string sql, IWAL? wal = null) diff --git a/src/SharpCoreDB/Services/SqlParser.DDL.cs b/src/SharpCoreDB/Services/SqlParser.DDL.cs index 788e8735..f7c880ae 100644 --- a/src/SharpCoreDB/Services/SqlParser.DDL.cs +++ b/src/SharpCoreDB/Services/SqlParser.DDL.cs @@ -1157,7 +1157,7 @@ private void ExecuteCreateVectorIndex(string sql, string[] parts, IWAL? wal) // Phase 5.4: Build live in-memory index if optimizer is registered // Supports DISKANN via VectorIndexManager / DiskAnnIndex - SqlParser.VectorQueryOptimizer?.BuildIndex(table, tableName, columnName, indexType); + this.VectorQueryOptimizer?.BuildIndex(table, tableName, columnName, indexType); wal?.Log(sql); } @@ -1191,7 +1191,7 @@ private void ExecuteDropVectorIndex(string sql, string[] parts, IWAL? wal) table.RemoveMetadata($"vector_index:{colName}:sql"); // Phase 5.4: Drop live in-memory index if optimizer is registered - SqlParser.VectorQueryOptimizer?.DropIndex(tableName, colName); + this.VectorQueryOptimizer?.DropIndex(tableName, colName); found = true; break; diff --git a/src/SharpCoreDB/Services/SqlParser.DML.cs b/src/SharpCoreDB/Services/SqlParser.DML.cs index 9a8af7a4..3a432609 100644 --- a/src/SharpCoreDB/Services/SqlParser.DML.cs +++ b/src/SharpCoreDB/Services/SqlParser.DML.cs @@ -1074,10 +1074,14 @@ private List> ExecuteSelectQuery(string sql, string[] TrackColumnUsage(tableName, whereStr); - // Phase 5.4: Detect ORDER BY vec_distance_*(col, query) LIMIT k → route to vector index + // Phase 5.4: Detect ORDER BY vec_distance_*(col, query) LIMIT k → route to vector index. + // The scan below stays the answer whenever the index cannot reproduce it exactly; see + // TryExecuteVectorOptimized for the two cases it refuses. if (limit.HasValue && limit.Value > 0 && VectorQueryOptimizer is not null) { - var vectorResult = TryExecuteVectorOptimized(sql, selectClause, tableName, orderBy, limit.Value, noEncrypt); + var vectorResult = TryExecuteVectorOptimized( + sql, selectClause, tableName, whereStr, orderBy, asc, limit.Value, noEncrypt); + if (vectorResult is not null) return vectorResult; } @@ -1108,6 +1112,395 @@ private List> ExecuteSelectQuery(string sql, string[] } + /// + /// The distance functions a vector index can answer. Kept in sync with + /// SharpCoreDB.VectorSearch.VectorQueryOptimizer; a name the optimizer does not know is + /// rejected by anyway. + /// + private static readonly string[] VectorDistanceFunctions = + [ + "VEC_DISTANCE_COSINE", + "VEC_DISTANCE_L2", + "VEC_DISTANCE_DOT", + ]; + + /// + /// Answers SELECT ... ORDER BY vec_distance_*(col, vector) LIMIT k from the vector index + /// instead of scanning and sorting the table. + /// + /// Returns — meaning "take the scan path" — unless the index can produce + /// exactly the rows, in exactly the shape, the query asks for. The implementation this replaces + /// returned every column plus a distance key no matter what the SELECT list said, and did + /// so for DESC and WHERE queries too, so an optimized query and an unoptimized one + /// returned different results for the same SQL. The rows are now projected like any other SELECT, + /// and the two shapes an ascending top-k index cannot answer fall through to the scan. + /// + /// + /// The full statement (the ORDER BY region is parsed from here). + /// The SELECT list, used for the distance alias and the projection. + /// The table being selected from. + /// The WHERE clause, if any, so a filtered query can be refused. + /// The single token the legacy parser read after ORDER BY. + /// False when ORDER BY ... DESC was requested. + /// The LIMIT value (k). + /// Whether to read ciphertext. + /// Projected rows, or null when the query must be answered by a scan. + private List>? TryExecuteVectorOptimized( + string sql, + string selectClause, + string tableName, + string? whereStr, + string? orderBy, + bool ascending, + int limit, + bool noEncrypt) + { + if (VectorQueryOptimizer is null || string.IsNullOrEmpty(orderBy) || limit <= 0) + { + return null; + } + + // An index scan returns the k nearest rows. That is not the answer to the other two shapes + // SQL allows here, and returning it anyway is how an "optimization" becomes a wrong result: + // * ORDER BY ... DESC asks for the k farthest, which reversing the nearest k does not give; + // * a WHERE clause would have to be applied after the index already chose its k, and + // post-filtering a limited set drops matching rows instead of fetching more of them. + if (!ascending || !string.IsNullOrWhiteSpace(whereStr)) + { + return null; + } + + var orderStart = sql.IndexOf(SqlConstants.ORDER, StringComparison.OrdinalIgnoreCase); + var orderCall = orderStart >= 0 ? FindVectorDistanceCall(sql, orderStart) : null; + var selectCall = FindVectorDistanceCall(selectClause, 0); + + // ORDER BY either re-states the distance function or names the alias the SELECT gave it. + var distanceAlias = selectCall is { } projection ? ReadAliasAfter(selectClause, projection.End) : null; + var orderByNamesAlias = distanceAlias is not null + && orderBy.Trim().Equals(distanceAlias, StringComparison.OrdinalIgnoreCase); + + // When the ORDER BY re-states the call, that call is the one being ordered on; the SELECT + // need not mention the function at all (SELECT id ... ORDER BY vec_distance_*(...)). + (string Function, string Column, string Argument, int End) call; + if (orderCall is { } orderedCall) + { + call = orderedCall; + } + else if (orderByNamesAlias && selectCall is { } projectedCall) + { + call = projectedCall; + } + else + { + return null; + } + + if (!IsSimpleIdentifier(call.Column) + || !VectorQueryOptimizer.CanOptimize(tableName, call.Column, call.Function, limit)) + { + return null; + } + + var queryVector = Unquote(call.Argument); + if (queryVector.Length == 0) + { + return null; + } + + var rows = VectorQueryOptimizer.ExecuteOptimized( + this.tables[tableName], + tableName, + call.Column, + call.Function, + queryVector, + limit, + noEncrypt); + + return ProjectVectorRows(rows, selectClause, distanceAlias, call.Function); + } + + /// + /// Finds the first vec_distance_*(column, vector) call at or after + /// and returns its function name, its column (bare name), its second argument as written, and the + /// index of its closing parenthesis. + /// + /// The text to search. + /// The index to start searching from. + /// The parsed call, or null when there is none. + private static (string Function, string Column, string Argument, int End)? FindVectorDistanceCall(string text, int start) + { + var searchFrom = Math.Min(start, text.Length); + var bestIndex = -1; + string? function = null; + + foreach (var candidate in VectorDistanceFunctions) + { + var index = text.IndexOf(candidate, searchFrom, StringComparison.OrdinalIgnoreCase); + if (index >= 0 && (bestIndex < 0 || index < bestIndex)) + { + bestIndex = index; + function = candidate; + } + } + + if (function is null) + { + return null; + } + + var open = text.IndexOf('(', bestIndex); + if (open < 0) + { + return null; + } + + var close = FindClosingParenthesis(text, open); + if (close < 0) + { + return null; + } + + var arguments = SplitVectorArguments(text[(open + 1)..close]); + if (arguments.Count < 2) + { + return null; + } + + return (function, ExtractBareColumn(arguments[0]), arguments[1], close); + } + + /// + /// Returns the index of the parenthesis matching the one at , tracking + /// nesting and skipping quoted text. + /// + /// The text holding the parentheses. + /// The index of the opening parenthesis. + /// The index of the closing parenthesis, or -1 when unbalanced. + private static int FindClosingParenthesis(string text, int open) + { + var depth = 0; + var inQuote = false; + + for (var i = open; i < text.Length; i++) + { + var c = text[i]; + + if (c == '\'') + { + inQuote = !inQuote; + continue; + } + + if (inQuote) + { + continue; + } + + if (c == '(') + { + depth++; + } + else if (c == ')' && --depth == 0) + { + return i; + } + } + + return -1; + } + + /// + /// Splits the argument list of a function call on its top-level commas, so a vector literal or a + /// nested call inside one argument stays in one piece. Brackets and braces count as nesting too: + /// a bound float[] parameter arrives as an unquoted [1,0,0,0], whose commas are + /// part of the value, and splitting there would leave [1 as the query vector. + /// + /// The text between the parentheses. + /// The trimmed arguments. + private static List SplitVectorArguments(string arguments) + { + var items = new List(); + var current = new System.Text.StringBuilder(); + var nesting = 0; + var inQuote = false; + + foreach (var c in arguments) + { + if (c == '\'') + { + inQuote = !inQuote; + } + + if (!inQuote) + { + if (c is '(' or '[' or '{') + { + nesting++; + } + else if (c is ')' or ']' or '}') + { + if (nesting > 0) + { + nesting--; + } + } + else if (c == ',' && nesting == 0) + { + items.Add(current.ToString().Trim()); + current.Clear(); + continue; + } + } + + current.Append(c); + } + + if (current.Length > 0) + { + items.Add(current.ToString().Trim()); + } + + return items; + } + + /// + /// Reads an alias that follows a parsed call: ... ) AS distance or the bare form + /// ... ) distance. Returns null when the next token is punctuation, an operator, or the + /// end of the clause. + /// + /// The clause the call was parsed from. + /// The index of the call's closing parenthesis. + /// The alias, or null. + private static string? ReadAliasAfter(string text, int callEnd) + { + if (callEnd + 1 >= text.Length) + { + return null; + } + + var rest = text[(callEnd + 1)..].TrimStart(); + + if (rest.StartsWith("AS", StringComparison.OrdinalIgnoreCase) + && rest.Length > 2 + && char.IsWhiteSpace(rest[2])) + { + rest = rest[2..].TrimStart(); + } + + var end = 0; + while (end < rest.Length && (char.IsLetterOrDigit(rest[end]) || rest[end] == '_')) + { + end++; + } + + return end == 0 ? null : rest[..end]; + } + + /// + /// Projects rows returned by the vector index onto the SELECT list, substituting the + /// distance the optimizer injected for the distance expression itself. The scan path + /// around it does not project, so this is what keeps an optimized query's column set identical + /// to the same query without an index. + /// + /// Rows from . + /// The SELECT list. + /// The alias given to the distance expression, if any. + /// The distance function name used by the query. + /// The projected rows. + private static List> ProjectVectorRows( + List> rows, + string selectClause, + string? distanceAlias, + string distanceFunction) + { + var items = SplitVectorArguments(selectClause); + var distanceKey = string.IsNullOrEmpty(distanceAlias) ? "distance" : distanceAlias; + var selectAll = items.Count == 1 && items[0].Trim() == "*"; + + var result = new List>(rows.Count); + foreach (var row in rows) + { + if (selectAll) + { + // `SELECT *` asked for the row, not for the ordering expression. + result.Add(new Dictionary(row, StringComparer.OrdinalIgnoreCase)); + continue; + } + + var projected = new Dictionary(items.Count, StringComparer.OrdinalIgnoreCase); + foreach (var item in items) + { + var (expression, alias) = SplitSelectItem(item); + + if (expression.Contains(distanceFunction, StringComparison.OrdinalIgnoreCase)) + { + projected[alias ?? distanceKey] = row.TryGetValue("distance", out var distance) + ? distance + : DBNull.Value; + continue; + } + + projected[alias ?? ExtractBareColumn(expression)] = FindValue(row, expression) ?? DBNull.Value; + } + + result.Add(projected); + } + + return result; + } + + /// + /// Splits one SELECT item into its expression and its alias, using the same rule the join + /// projection uses (expr AS alias or expr alias). + /// + /// One comma-separated SELECT item. + /// The expression and the alias, if any. + private static (string Expression, string? Alias) SplitSelectItem(string item) + { + var trimmed = item.Trim(); + var match = System.Text.RegularExpressions.Regex.Match( + trimmed, + @"^(.+?)\s+(?:AS\s+)?(\w+)$", + System.Text.RegularExpressions.RegexOptions.IgnoreCase, + TimeSpan.FromSeconds(1)); + + return match.Success + ? (match.Groups[1].Value.Trim(), match.Groups[2].Value.Trim()) + : (trimmed, null); + } + + /// + /// Returns true when the value is a plain column reference (letters, digits, underscore), so a + /// numeric literal or an expression can never be handed to the index registry as a column name. + /// + /// The candidate column name. + /// True when the value is a simple identifier. + private static bool IsSimpleIdentifier(string value) + { + if (value.Length == 0 || !(char.IsLetter(value[0]) || value[0] == '_')) + { + return false; + } + + foreach (var c in value) + { + if (!(char.IsLetterOrDigit(c) || c == '_')) + { + return false; + } + } + + return true; + } + + /// + /// Strips the surrounding quotes from a SQL string literal, leaving a vector literal such as + /// [0.25,0.5,0.75,1] as the text the optimizer's vector reader expects. + /// + /// The literal as written in the statement. + /// The unquoted value. + private static string Unquote(string value) => value.Trim().Trim('\'', '"'); + /// /// Executes COUNT(*) aggregate query with modern patterns. /// ✅ MODERNIZED: Uses pattern matching and null-coalescing. diff --git a/tests/SharpCoreDB.VectorSearch.Tests/SqlVectorQueryTests.cs b/tests/SharpCoreDB.VectorSearch.Tests/SqlVectorQueryTests.cs new file mode 100644 index 00000000..a3adc62a --- /dev/null +++ b/tests/SharpCoreDB.VectorSearch.Tests/SqlVectorQueryTests.cs @@ -0,0 +1,217 @@ +// +// Copyright (c) 2026 MPCoreDeveloper and GitHub Copilot. All rights reserved. +// Licensed under the MIT License. See LICENSE file in the project root for full license information. +// +namespace SharpCoreDB.VectorSearch.Tests; + +using System.Globalization; +using Microsoft.Extensions.DependencyInjection; +using SharpCoreDB.Interfaces; + +/// +/// End-to-end coverage for the planner branch that answers +/// ORDER BY vec_distance_*(column, vector) LIMIT k from a vector index (Phase 5.4). +/// +/// These tests exist because the branch was dead in two independent ways at once: the planner call +/// site was guarded by SqlParser.VectorQueryOptimizer is not null, and that member was an +/// internal static that nothing ever assigned (so CREATE VECTOR INDEX built nothing +/// either), and the method behind the guard had been replaced by a stub returning null. Unit tests +/// over VectorQueryOptimizer passed throughout, because the optimizer itself was fine — it +/// was never called. Only a query through Database can tell the difference, which is why +/// these go through AddVectorSupport() and SQL rather than constructing the optimizer. +/// +/// +public sealed class SqlVectorQueryTests : IDisposable +{ + private const string QueryVectorLiteral = "'[1,0,0,0]'"; + + private readonly string _path = Path.Combine(Path.GetTempPath(), $"SCDB_VectorSql_{Guid.NewGuid():N}"); + private readonly ServiceProvider _provider; + private IDatabase? _db; + + /// Registers vector support exactly as a host application would. + public SqlVectorQueryTests() + { + _provider = new ServiceCollection() + .AddSharpCoreDB() + .AddVectorSupport() + .BuildServiceProvider(); + } + + /// + public void Dispose() + { + (_db as IDisposable)?.Dispose(); + _provider.Dispose(); + + try + { + if (Directory.Exists(_path)) + Directory.Delete(_path, true); + } + catch (IOException) + { + // A leaked temp directory must not fail the suite. + } + } + + [Fact] + public void OrderByDistanceAlias_WithIndex_ReturnsNearestRowsProjectedOnTheSelectList() + { + var db = SeededDatabase(); + db.ExecuteSQL("CREATE VECTOR INDEX idx_docs ON docs(embedding) USING FLAT"); + + var rows = Query( + db, + $"SELECT id, body, vec_distance_cosine(embedding, {QueryVectorLiteral}) AS distance " + + "FROM docs ORDER BY distance LIMIT 2"); + + Assert.Equal(new[] { 1L, 3L }, rows.Select(r => Convert.ToInt64(r["id"], CultureInfo.InvariantCulture)).ToArray()); + + // Cosine distance to [1,0,0,0]: id 1 is identical (0.0), id 3 has similarity 0.8 (0.2). + Assert.Equal(0.0f, Convert.ToSingle(rows[0]["distance"], CultureInfo.InvariantCulture), 3); + Assert.Equal(0.2f, Convert.ToSingle(rows[1]["distance"], CultureInfo.InvariantCulture), 3); + + // The projection is the SELECT list, not the whole row: the restored implementation copied + // every column and appended "distance" whatever the query asked for. + Assert.Equal(3, rows[0].Count); + Assert.True(rows[0].ContainsKey("id") && rows[0].ContainsKey("body") && rows[0].ContainsKey("distance")); + } + + [Fact] + public void OrderByDistanceFunction_WithoutDistanceInTheSelectList_IsStillServedFromTheIndex() + { + var db = SeededDatabase(); + db.ExecuteSQL("CREATE VECTOR INDEX idx_docs ON docs(embedding) USING FLAT"); + + var rows = Query( + db, + $"SELECT id FROM docs ORDER BY vec_distance_cosine(embedding, {QueryVectorLiteral}) LIMIT 1"); + + Assert.Single(rows); + Assert.Equal(1L, Convert.ToInt64(rows[0]["id"], CultureInfo.InvariantCulture)); + + // ORDER BY named the function, not an alias of it; the distance was not selected. + Assert.Single(rows[0]); + } + + [Fact] + public void OrderByDistanceAlias_WithBoundVectorParameter_IsServedFromTheIndex() + { + var db = SeededDatabase(); + db.ExecuteSQL("CREATE VECTOR INDEX idx_docs ON docs(embedding) USING FLAT"); + + // A float[] parameter is bound to the bracketed vector literal by ParameterBinder, so the + // planner sees the same text an inline literal would produce. + var rows = db.ExecuteQuery( + "SELECT id, vec_distance_cosine(embedding, @q) AS distance FROM docs ORDER BY distance LIMIT 1", + new Dictionary { ["q"] = new float[] { 1, 0, 0, 0 } }); + + Assert.Single(rows); + Assert.Equal(1L, Convert.ToInt64(rows[0]["id"], CultureInfo.InvariantCulture)); + Assert.True(rows[0].ContainsKey("distance")); + } + + [Fact] + public void OrderByDistanceDescending_IsNotServedFromTheIndex() + { + var db = SeededDatabase(); + db.ExecuteSQL("CREATE VECTOR INDEX idx_docs ON docs(embedding) USING FLAT"); + + var rows = Query( + db, + $"SELECT id, body, vec_distance_cosine(embedding, {QueryVectorLiteral}) AS distance " + + "FROM docs ORDER BY distance DESC LIMIT 2"); + + Assert.Equal(2, rows.Count); + + // A top-k index holds the k nearest rows; the k farthest cannot be derived from it, so the + // query must keep its scan behaviour rather than receive a nearest-k answer labelled as DESC. + // The scan path neither projects nor computes the distance, which is what this asserts. + Assert.DoesNotContain("distance", rows[0].Keys); + Assert.Contains("embedding", rows[0].Keys); + } + + [Fact] + public void OrderByDistanceWithWhereClause_IsNotServedFromTheIndex() + { + var db = SeededDatabase(); + db.ExecuteSQL("CREATE VECTOR INDEX idx_docs ON docs(embedding) USING FLAT"); + + // Applying the predicate after the index chose its k would drop matching rows instead of + // fetching further down the ranking, so a filtered top-k stays on the scan. + var rows = Query( + db, + $"SELECT id, body, vec_distance_cosine(embedding, {QueryVectorLiteral}) AS distance " + + "FROM docs WHERE id > 1 ORDER BY distance LIMIT 2"); + + Assert.Equal(2, rows.Count); + Assert.DoesNotContain("distance", rows[0].Keys); + Assert.Contains("embedding", rows[0].Keys); + } + + [Fact] + public void DropVectorIndex_ReturnsTheQueryToTheScanPath() + { + var db = SeededDatabase(); + db.ExecuteSQL("CREATE VECTOR INDEX idx_docs ON docs(embedding) USING FLAT"); + + var optimized = Query( + db, + $"SELECT id, vec_distance_cosine(embedding, {QueryVectorLiteral}) AS distance " + + "FROM docs ORDER BY distance LIMIT 2"); + Assert.True(optimized[0].ContainsKey("distance")); + + db.ExecuteSQL("DROP VECTOR INDEX idx_docs ON docs"); + + var scanned = Query( + db, + $"SELECT id, vec_distance_cosine(embedding, {QueryVectorLiteral}) AS distance " + + "FROM docs ORDER BY distance LIMIT 2"); + Assert.DoesNotContain("distance", scanned[0].Keys); + } + + [Fact] + public void OrderByDistanceWithoutAnyIndex_FallsBackToTheScan() + { + var db = SeededDatabase(); + + var rows = Query( + db, + $"SELECT id, vec_distance_cosine(embedding, {QueryVectorLiteral}) AS distance " + + "FROM docs ORDER BY distance LIMIT 2"); + + Assert.Equal(2, rows.Count); + Assert.DoesNotContain("distance", rows[0].Keys); + } + + /// + /// Creates a database through the DI factory with four documents whose distances to + /// [1,0,0,0] are distinct enough to identify, and where two rows tie on distance so the + /// index's tie-break (id, ascending) is exercised too. + /// + /// The open database; the caller disposes it. + private IDatabase SeededDatabase() + { + if (_db is not null) + { + return _db; + } + + var factory = _provider.GetRequiredService(); + var db = factory.Create(_path, "pw", isReadOnly: false, config: new DatabaseConfig()); + + db.ExecuteSQL("CREATE TABLE docs (id INTEGER PRIMARY KEY, body TEXT, embedding VECTOR(4))"); + db.ExecuteSQL("INSERT INTO docs (id, body, embedding) VALUES (1, 'nearest', [1,0,0,0])"); + db.ExecuteSQL("INSERT INTO docs (id, body, embedding) VALUES (2, 'orthogonal', [0,1,0,0])"); + db.ExecuteSQL("INSERT INTO docs (id, body, embedding) VALUES (3, 'close', [0.8,0.6,0,0])"); + db.ExecuteSQL("INSERT INTO docs (id, body, embedding) VALUES (4, 'far', [0,0,1,0])"); + + _db = db; + return db; + } + + private static List> Query(IDatabase db, string sql) + => db.ExecuteQuery(sql, new Dictionary()); +} +