diff --git a/src/SharpCoreDB/Database/Core/Database.Core.cs b/src/SharpCoreDB/Database/Core/Database.Core.cs index 89f99847..52012fed 100644 --- a/src/SharpCoreDB/Database/Core/Database.Core.cs +++ b/src/SharpCoreDB/Database/Core/Database.Core.cs @@ -57,6 +57,7 @@ public partial class Database : IDatabase, IDisposable, IAsyncDisposable private QueryPlanCache? planCache; // ✅ Lazy-initialized query plan cache private SqlParser? _sharedSqlParser; // ✅ Reusable SqlParser for compiled queries private IGraphRagProvider? _cachedGraphRagProvider; // v2: cached DI resolution (avoids per-call GetService) + private IVectorQueryOptimizer? _cachedVectorQueryOptimizer; // resolved once, same reason /// /// v2 (Native AOT readiness): metadata JSON options. resolves the diff --git a/src/SharpCoreDB/Database/Execution/Database.Execution.cs b/src/SharpCoreDB/Database/Execution/Database.Execution.cs index 18639a5c..4b52c430 100644 --- a/src/SharpCoreDB/Database/Execution/Database.Execution.cs +++ b/src/SharpCoreDB/Database/Execution/Database.Execution.cs @@ -31,21 +31,47 @@ private SqlParser GetSharedSqlParser() { _sharedSqlParser = new SqlParser(tables, _dbPath, storage, isReadOnly, queryCache, config); _sharedSqlParser.Database = this; + AttachOptionalProviders(_sharedSqlParser); + } - // v2: resolve the optional GRAPH_RAG provider once and cache it for the parser - // lifetime. Previously this DI lookup ran on every ExecuteSQL/ExecuteQuery call. - if (_cachedGraphRagProvider is null) - { - _cachedGraphRagProvider = _serviceProvider.GetService(typeof(IGraphRagProvider)) as IGraphRagProvider; - } + return _sharedSqlParser; + } - if (_cachedGraphRagProvider is not null) - { - _sharedSqlParser.SetGraphRagProvider(_cachedGraphRagProvider); - } + /// + /// Attaches the optional SQL extensions that live in separate assemblies and are therefore + /// resolved from the service provider instead of being referenced directly: the GRAPH_RAG + /// provider and the vector query optimizer. + /// + /// The vector optimizer used to be attached nowhere at all. SqlParser.VectorQueryOptimizer + /// was an internal static property that no code inside this assembly ever set and no other + /// assembly could set (the type is internal), so both of its consumers were permanent no-ops: + /// CREATE VECTOR INDEX built no index, and the ORDER BY vec_distance_*(...) LIMIT k + /// planner branch was never entered. Resolving here — the same + /// way the GRAPH_RAG provider is resolved — is what makes AddVectorSupport() reach the + /// planner. Both lookups are cached on the database, so a parser rebuild costs nothing when the + /// extension is registered and resolves to null only once when it is not. + /// + /// + /// The parser instance to configure. + private void AttachOptionalProviders(SqlParser parser) + { + if (_cachedGraphRagProvider is null) + { + _cachedGraphRagProvider = _serviceProvider.GetService(typeof(IGraphRagProvider)) as IGraphRagProvider; } - return _sharedSqlParser; + if (_cachedGraphRagProvider is not null) + { + parser.SetGraphRagProvider(_cachedGraphRagProvider); + } + + _cachedVectorQueryOptimizer ??= + _serviceProvider.GetService(typeof(IVectorQueryOptimizer)) as IVectorQueryOptimizer; + + if (_cachedVectorQueryOptimizer is not null) + { + parser.SetVectorQueryOptimizer(_cachedVectorQueryOptimizer); + } } @@ -469,14 +495,11 @@ public List> ExecuteCompiledQuery(DataStructures.Prep return executor.Execute(stmt.CompiledPlan, parameters); } - // ✅ Fallback: Lazy-initialize shared SqlParser (thread-safe via Interlocked) - if (_sharedSqlParser is null) - { - _sharedSqlParser = new SqlParser(tables, _dbPath, storage, isReadOnly, queryCache, config); - _sharedSqlParser.Database = this; - } - - return _sharedSqlParser.ExecuteQuery(stmt.Plan, parameters); + // ✅ Fallback: the shared parser created by GetSharedSqlParser() (which also attaches the + // optional GRAPH_RAG provider and vector optimizer). This block used to build its own + // SqlParser, so a compiled query falling back here got a parser without either extension — + // the second of the two creation sites is exactly how an extension point loses its wiring. + return GetSharedSqlParser().ExecuteQuery(stmt.Plan, parameters); } /// diff --git a/src/SharpCoreDB/Services/SqlParser.Core.cs b/src/SharpCoreDB/Services/SqlParser.Core.cs index 4ea9d69b..abe76ee9 100644 --- a/src/SharpCoreDB/Services/SqlParser.Core.cs +++ b/src/SharpCoreDB/Services/SqlParser.Core.cs @@ -118,12 +118,33 @@ internal SqlParser(Dictionary tables, string dbPath, bool isRead private List> _pendingQueryResults = []; /// - /// Optional vector query optimizer registered by the VectorSearch module. - /// When set, the query planner detects ORDER BY vec_distance_*() + LIMIT patterns - /// and routes them to a vector index instead of a full table scan. - /// Thread-safe: set once during Database initialization, read from any thread. + /// Optional vector query optimizer supplied by the VectorSearch module. + /// + /// When set, the planner detects ORDER BY vec_distance_*(col, vector) LIMIT k and + /// CREATE VECTOR INDEX routes them to a vector index instead of a full table scan. + /// + /// + /// This is a per-parser instance member, not a static one. It used to be + /// internal static, which made the extension point unreachable in practice: an + /// internal static on an internal class cannot be assigned by another assembly + /// (SharpCoreDB.VectorSearch registers with the DI + /// container, it cannot see this type), and no code inside this assembly ever assigned it + /// either. Every call site was therefore a null-conditional no-op, in both directions: + /// CREATE VECTOR INDEX built nothing and ORDER BY vec_distance_* never reached + /// the optimizer. resolves the registered optimizer from its service + /// provider and attaches it here (see Database.GetSharedSqlParser). + /// /// - internal static IVectorQueryOptimizer? VectorQueryOptimizer { get; set; } + internal IVectorQueryOptimizer? VectorQueryOptimizer { get; private set; } + + /// + /// Attaches the optional vector query optimizer for this parser instance. + /// Called by once, when the shared parser is created, with the + /// registered by AddVectorSupport() (if any). + /// + /// The optimizer to use, or for none. + internal void SetVectorQueryOptimizer(IVectorQueryOptimizer? optimizer) + => this.VectorQueryOptimizer = optimizer; /// public void Execute(string sql, IWAL? wal = null) diff --git a/src/SharpCoreDB/Services/SqlParser.DDL.cs b/src/SharpCoreDB/Services/SqlParser.DDL.cs index 788e8735..f7c880ae 100644 --- a/src/SharpCoreDB/Services/SqlParser.DDL.cs +++ b/src/SharpCoreDB/Services/SqlParser.DDL.cs @@ -1157,7 +1157,7 @@ private void ExecuteCreateVectorIndex(string sql, string[] parts, IWAL? wal) // Phase 5.4: Build live in-memory index if optimizer is registered // Supports DISKANN via VectorIndexManager / DiskAnnIndex - SqlParser.VectorQueryOptimizer?.BuildIndex(table, tableName, columnName, indexType); + this.VectorQueryOptimizer?.BuildIndex(table, tableName, columnName, indexType); wal?.Log(sql); } @@ -1191,7 +1191,7 @@ private void ExecuteDropVectorIndex(string sql, string[] parts, IWAL? wal) table.RemoveMetadata($"vector_index:{colName}:sql"); // Phase 5.4: Drop live in-memory index if optimizer is registered - SqlParser.VectorQueryOptimizer?.DropIndex(tableName, colName); + this.VectorQueryOptimizer?.DropIndex(tableName, colName); found = true; break; diff --git a/src/SharpCoreDB/Services/SqlParser.DML.cs b/src/SharpCoreDB/Services/SqlParser.DML.cs index 9a8af7a4..3a432609 100644 --- a/src/SharpCoreDB/Services/SqlParser.DML.cs +++ b/src/SharpCoreDB/Services/SqlParser.DML.cs @@ -1074,10 +1074,14 @@ private List> ExecuteSelectQuery(string sql, string[] TrackColumnUsage(tableName, whereStr); - // Phase 5.4: Detect ORDER BY vec_distance_*(col, query) LIMIT k → route to vector index + // Phase 5.4: Detect ORDER BY vec_distance_*(col, query) LIMIT k → route to vector index. + // The scan below stays the answer whenever the index cannot reproduce it exactly; see + // TryExecuteVectorOptimized for the two cases it refuses. if (limit.HasValue && limit.Value > 0 && VectorQueryOptimizer is not null) { - var vectorResult = TryExecuteVectorOptimized(sql, selectClause, tableName, orderBy, limit.Value, noEncrypt); + var vectorResult = TryExecuteVectorOptimized( + sql, selectClause, tableName, whereStr, orderBy, asc, limit.Value, noEncrypt); + if (vectorResult is not null) return vectorResult; } @@ -1108,6 +1112,395 @@ private List> ExecuteSelectQuery(string sql, string[] } + /// + /// The distance functions a vector index can answer. Kept in sync with + /// SharpCoreDB.VectorSearch.VectorQueryOptimizer; a name the optimizer does not know is + /// rejected by anyway. + /// + private static readonly string[] VectorDistanceFunctions = + [ + "VEC_DISTANCE_COSINE", + "VEC_DISTANCE_L2", + "VEC_DISTANCE_DOT", + ]; + + /// + /// Answers SELECT ... ORDER BY vec_distance_*(col, vector) LIMIT k from the vector index + /// instead of scanning and sorting the table. + /// + /// Returns — meaning "take the scan path" — unless the index can produce + /// exactly the rows, in exactly the shape, the query asks for. The implementation this replaces + /// returned every column plus a distance key no matter what the SELECT list said, and did + /// so for DESC and WHERE queries too, so an optimized query and an unoptimized one + /// returned different results for the same SQL. The rows are now projected like any other SELECT, + /// and the two shapes an ascending top-k index cannot answer fall through to the scan. + /// + /// + /// The full statement (the ORDER BY region is parsed from here). + /// The SELECT list, used for the distance alias and the projection. + /// The table being selected from. + /// The WHERE clause, if any, so a filtered query can be refused. + /// The single token the legacy parser read after ORDER BY. + /// False when ORDER BY ... DESC was requested. + /// The LIMIT value (k). + /// Whether to read ciphertext. + /// Projected rows, or null when the query must be answered by a scan. + private List>? TryExecuteVectorOptimized( + string sql, + string selectClause, + string tableName, + string? whereStr, + string? orderBy, + bool ascending, + int limit, + bool noEncrypt) + { + if (VectorQueryOptimizer is null || string.IsNullOrEmpty(orderBy) || limit <= 0) + { + return null; + } + + // An index scan returns the k nearest rows. That is not the answer to the other two shapes + // SQL allows here, and returning it anyway is how an "optimization" becomes a wrong result: + // * ORDER BY ... DESC asks for the k farthest, which reversing the nearest k does not give; + // * a WHERE clause would have to be applied after the index already chose its k, and + // post-filtering a limited set drops matching rows instead of fetching more of them. + if (!ascending || !string.IsNullOrWhiteSpace(whereStr)) + { + return null; + } + + var orderStart = sql.IndexOf(SqlConstants.ORDER, StringComparison.OrdinalIgnoreCase); + var orderCall = orderStart >= 0 ? FindVectorDistanceCall(sql, orderStart) : null; + var selectCall = FindVectorDistanceCall(selectClause, 0); + + // ORDER BY either re-states the distance function or names the alias the SELECT gave it. + var distanceAlias = selectCall is { } projection ? ReadAliasAfter(selectClause, projection.End) : null; + var orderByNamesAlias = distanceAlias is not null + && orderBy.Trim().Equals(distanceAlias, StringComparison.OrdinalIgnoreCase); + + // When the ORDER BY re-states the call, that call is the one being ordered on; the SELECT + // need not mention the function at all (SELECT id ... ORDER BY vec_distance_*(...)). + (string Function, string Column, string Argument, int End) call; + if (orderCall is { } orderedCall) + { + call = orderedCall; + } + else if (orderByNamesAlias && selectCall is { } projectedCall) + { + call = projectedCall; + } + else + { + return null; + } + + if (!IsSimpleIdentifier(call.Column) + || !VectorQueryOptimizer.CanOptimize(tableName, call.Column, call.Function, limit)) + { + return null; + } + + var queryVector = Unquote(call.Argument); + if (queryVector.Length == 0) + { + return null; + } + + var rows = VectorQueryOptimizer.ExecuteOptimized( + this.tables[tableName], + tableName, + call.Column, + call.Function, + queryVector, + limit, + noEncrypt); + + return ProjectVectorRows(rows, selectClause, distanceAlias, call.Function); + } + + /// + /// Finds the first vec_distance_*(column, vector) call at or after + /// and returns its function name, its column (bare name), its second argument as written, and the + /// index of its closing parenthesis. + /// + /// The text to search. + /// The index to start searching from. + /// The parsed call, or null when there is none. + private static (string Function, string Column, string Argument, int End)? FindVectorDistanceCall(string text, int start) + { + var searchFrom = Math.Min(start, text.Length); + var bestIndex = -1; + string? function = null; + + foreach (var candidate in VectorDistanceFunctions) + { + var index = text.IndexOf(candidate, searchFrom, StringComparison.OrdinalIgnoreCase); + if (index >= 0 && (bestIndex < 0 || index < bestIndex)) + { + bestIndex = index; + function = candidate; + } + } + + if (function is null) + { + return null; + } + + var open = text.IndexOf('(', bestIndex); + if (open < 0) + { + return null; + } + + var close = FindClosingParenthesis(text, open); + if (close < 0) + { + return null; + } + + var arguments = SplitVectorArguments(text[(open + 1)..close]); + if (arguments.Count < 2) + { + return null; + } + + return (function, ExtractBareColumn(arguments[0]), arguments[1], close); + } + + /// + /// Returns the index of the parenthesis matching the one at , tracking + /// nesting and skipping quoted text. + /// + /// The text holding the parentheses. + /// The index of the opening parenthesis. + /// The index of the closing parenthesis, or -1 when unbalanced. + private static int FindClosingParenthesis(string text, int open) + { + var depth = 0; + var inQuote = false; + + for (var i = open; i < text.Length; i++) + { + var c = text[i]; + + if (c == '\'') + { + inQuote = !inQuote; + continue; + } + + if (inQuote) + { + continue; + } + + if (c == '(') + { + depth++; + } + else if (c == ')' && --depth == 0) + { + return i; + } + } + + return -1; + } + + /// + /// Splits the argument list of a function call on its top-level commas, so a vector literal or a + /// nested call inside one argument stays in one piece. Brackets and braces count as nesting too: + /// a bound float[] parameter arrives as an unquoted [1,0,0,0], whose commas are + /// part of the value, and splitting there would leave [1 as the query vector. + /// + /// The text between the parentheses. + /// The trimmed arguments. + private static List SplitVectorArguments(string arguments) + { + var items = new List(); + var current = new System.Text.StringBuilder(); + var nesting = 0; + var inQuote = false; + + foreach (var c in arguments) + { + if (c == '\'') + { + inQuote = !inQuote; + } + + if (!inQuote) + { + if (c is '(' or '[' or '{') + { + nesting++; + } + else if (c is ')' or ']' or '}') + { + if (nesting > 0) + { + nesting--; + } + } + else if (c == ',' && nesting == 0) + { + items.Add(current.ToString().Trim()); + current.Clear(); + continue; + } + } + + current.Append(c); + } + + if (current.Length > 0) + { + items.Add(current.ToString().Trim()); + } + + return items; + } + + /// + /// Reads an alias that follows a parsed call: ... ) AS distance or the bare form + /// ... ) distance. Returns null when the next token is punctuation, an operator, or the + /// end of the clause. + /// + /// The clause the call was parsed from. + /// The index of the call's closing parenthesis. + /// The alias, or null. + private static string? ReadAliasAfter(string text, int callEnd) + { + if (callEnd + 1 >= text.Length) + { + return null; + } + + var rest = text[(callEnd + 1)..].TrimStart(); + + if (rest.StartsWith("AS", StringComparison.OrdinalIgnoreCase) + && rest.Length > 2 + && char.IsWhiteSpace(rest[2])) + { + rest = rest[2..].TrimStart(); + } + + var end = 0; + while (end < rest.Length && (char.IsLetterOrDigit(rest[end]) || rest[end] == '_')) + { + end++; + } + + return end == 0 ? null : rest[..end]; + } + + /// + /// Projects rows returned by the vector index onto the SELECT list, substituting the + /// distance the optimizer injected for the distance expression itself. The scan path + /// around it does not project, so this is what keeps an optimized query's column set identical + /// to the same query without an index. + /// + /// Rows from . + /// The SELECT list. + /// The alias given to the distance expression, if any. + /// The distance function name used by the query. + /// The projected rows. + private static List> ProjectVectorRows( + List> rows, + string selectClause, + string? distanceAlias, + string distanceFunction) + { + var items = SplitVectorArguments(selectClause); + var distanceKey = string.IsNullOrEmpty(distanceAlias) ? "distance" : distanceAlias; + var selectAll = items.Count == 1 && items[0].Trim() == "*"; + + var result = new List>(rows.Count); + foreach (var row in rows) + { + if (selectAll) + { + // `SELECT *` asked for the row, not for the ordering expression. + result.Add(new Dictionary(row, StringComparer.OrdinalIgnoreCase)); + continue; + } + + var projected = new Dictionary(items.Count, StringComparer.OrdinalIgnoreCase); + foreach (var item in items) + { + var (expression, alias) = SplitSelectItem(item); + + if (expression.Contains(distanceFunction, StringComparison.OrdinalIgnoreCase)) + { + projected[alias ?? distanceKey] = row.TryGetValue("distance", out var distance) + ? distance + : DBNull.Value; + continue; + } + + projected[alias ?? ExtractBareColumn(expression)] = FindValue(row, expression) ?? DBNull.Value; + } + + result.Add(projected); + } + + return result; + } + + /// + /// Splits one SELECT item into its expression and its alias, using the same rule the join + /// projection uses (expr AS alias or expr alias). + /// + /// One comma-separated SELECT item. + /// The expression and the alias, if any. + private static (string Expression, string? Alias) SplitSelectItem(string item) + { + var trimmed = item.Trim(); + var match = System.Text.RegularExpressions.Regex.Match( + trimmed, + @"^(.+?)\s+(?:AS\s+)?(\w+)$", + System.Text.RegularExpressions.RegexOptions.IgnoreCase, + TimeSpan.FromSeconds(1)); + + return match.Success + ? (match.Groups[1].Value.Trim(), match.Groups[2].Value.Trim()) + : (trimmed, null); + } + + /// + /// Returns true when the value is a plain column reference (letters, digits, underscore), so a + /// numeric literal or an expression can never be handed to the index registry as a column name. + /// + /// The candidate column name. + /// True when the value is a simple identifier. + private static bool IsSimpleIdentifier(string value) + { + if (value.Length == 0 || !(char.IsLetter(value[0]) || value[0] == '_')) + { + return false; + } + + foreach (var c in value) + { + if (!(char.IsLetterOrDigit(c) || c == '_')) + { + return false; + } + } + + return true; + } + + /// + /// Strips the surrounding quotes from a SQL string literal, leaving a vector literal such as + /// [0.25,0.5,0.75,1] as the text the optimizer's vector reader expects. + /// + /// The literal as written in the statement. + /// The unquoted value. + private static string Unquote(string value) => value.Trim().Trim('\'', '"'); + /// /// Executes COUNT(*) aggregate query with modern patterns. /// ✅ MODERNIZED: Uses pattern matching and null-coalescing. diff --git a/tests/SharpCoreDB.VectorSearch.Tests/SqlVectorQueryTests.cs b/tests/SharpCoreDB.VectorSearch.Tests/SqlVectorQueryTests.cs new file mode 100644 index 00000000..a3adc62a --- /dev/null +++ b/tests/SharpCoreDB.VectorSearch.Tests/SqlVectorQueryTests.cs @@ -0,0 +1,217 @@ +// +// Copyright (c) 2026 MPCoreDeveloper and GitHub Copilot. All rights reserved. +// Licensed under the MIT License. See LICENSE file in the project root for full license information. +// +namespace SharpCoreDB.VectorSearch.Tests; + +using System.Globalization; +using Microsoft.Extensions.DependencyInjection; +using SharpCoreDB.Interfaces; + +/// +/// End-to-end coverage for the planner branch that answers +/// ORDER BY vec_distance_*(column, vector) LIMIT k from a vector index (Phase 5.4). +/// +/// These tests exist because the branch was dead in two independent ways at once: the planner call +/// site was guarded by SqlParser.VectorQueryOptimizer is not null, and that member was an +/// internal static that nothing ever assigned (so CREATE VECTOR INDEX built nothing +/// either), and the method behind the guard had been replaced by a stub returning null. Unit tests +/// over VectorQueryOptimizer passed throughout, because the optimizer itself was fine — it +/// was never called. Only a query through Database can tell the difference, which is why +/// these go through AddVectorSupport() and SQL rather than constructing the optimizer. +/// +/// +public sealed class SqlVectorQueryTests : IDisposable +{ + private const string QueryVectorLiteral = "'[1,0,0,0]'"; + + private readonly string _path = Path.Combine(Path.GetTempPath(), $"SCDB_VectorSql_{Guid.NewGuid():N}"); + private readonly ServiceProvider _provider; + private IDatabase? _db; + + /// Registers vector support exactly as a host application would. + public SqlVectorQueryTests() + { + _provider = new ServiceCollection() + .AddSharpCoreDB() + .AddVectorSupport() + .BuildServiceProvider(); + } + + /// + public void Dispose() + { + (_db as IDisposable)?.Dispose(); + _provider.Dispose(); + + try + { + if (Directory.Exists(_path)) + Directory.Delete(_path, true); + } + catch (IOException) + { + // A leaked temp directory must not fail the suite. + } + } + + [Fact] + public void OrderByDistanceAlias_WithIndex_ReturnsNearestRowsProjectedOnTheSelectList() + { + var db = SeededDatabase(); + db.ExecuteSQL("CREATE VECTOR INDEX idx_docs ON docs(embedding) USING FLAT"); + + var rows = Query( + db, + $"SELECT id, body, vec_distance_cosine(embedding, {QueryVectorLiteral}) AS distance " + + "FROM docs ORDER BY distance LIMIT 2"); + + Assert.Equal(new[] { 1L, 3L }, rows.Select(r => Convert.ToInt64(r["id"], CultureInfo.InvariantCulture)).ToArray()); + + // Cosine distance to [1,0,0,0]: id 1 is identical (0.0), id 3 has similarity 0.8 (0.2). + Assert.Equal(0.0f, Convert.ToSingle(rows[0]["distance"], CultureInfo.InvariantCulture), 3); + Assert.Equal(0.2f, Convert.ToSingle(rows[1]["distance"], CultureInfo.InvariantCulture), 3); + + // The projection is the SELECT list, not the whole row: the restored implementation copied + // every column and appended "distance" whatever the query asked for. + Assert.Equal(3, rows[0].Count); + Assert.True(rows[0].ContainsKey("id") && rows[0].ContainsKey("body") && rows[0].ContainsKey("distance")); + } + + [Fact] + public void OrderByDistanceFunction_WithoutDistanceInTheSelectList_IsStillServedFromTheIndex() + { + var db = SeededDatabase(); + db.ExecuteSQL("CREATE VECTOR INDEX idx_docs ON docs(embedding) USING FLAT"); + + var rows = Query( + db, + $"SELECT id FROM docs ORDER BY vec_distance_cosine(embedding, {QueryVectorLiteral}) LIMIT 1"); + + Assert.Single(rows); + Assert.Equal(1L, Convert.ToInt64(rows[0]["id"], CultureInfo.InvariantCulture)); + + // ORDER BY named the function, not an alias of it; the distance was not selected. + Assert.Single(rows[0]); + } + + [Fact] + public void OrderByDistanceAlias_WithBoundVectorParameter_IsServedFromTheIndex() + { + var db = SeededDatabase(); + db.ExecuteSQL("CREATE VECTOR INDEX idx_docs ON docs(embedding) USING FLAT"); + + // A float[] parameter is bound to the bracketed vector literal by ParameterBinder, so the + // planner sees the same text an inline literal would produce. + var rows = db.ExecuteQuery( + "SELECT id, vec_distance_cosine(embedding, @q) AS distance FROM docs ORDER BY distance LIMIT 1", + new Dictionary { ["q"] = new float[] { 1, 0, 0, 0 } }); + + Assert.Single(rows); + Assert.Equal(1L, Convert.ToInt64(rows[0]["id"], CultureInfo.InvariantCulture)); + Assert.True(rows[0].ContainsKey("distance")); + } + + [Fact] + public void OrderByDistanceDescending_IsNotServedFromTheIndex() + { + var db = SeededDatabase(); + db.ExecuteSQL("CREATE VECTOR INDEX idx_docs ON docs(embedding) USING FLAT"); + + var rows = Query( + db, + $"SELECT id, body, vec_distance_cosine(embedding, {QueryVectorLiteral}) AS distance " + + "FROM docs ORDER BY distance DESC LIMIT 2"); + + Assert.Equal(2, rows.Count); + + // A top-k index holds the k nearest rows; the k farthest cannot be derived from it, so the + // query must keep its scan behaviour rather than receive a nearest-k answer labelled as DESC. + // The scan path neither projects nor computes the distance, which is what this asserts. + Assert.DoesNotContain("distance", rows[0].Keys); + Assert.Contains("embedding", rows[0].Keys); + } + + [Fact] + public void OrderByDistanceWithWhereClause_IsNotServedFromTheIndex() + { + var db = SeededDatabase(); + db.ExecuteSQL("CREATE VECTOR INDEX idx_docs ON docs(embedding) USING FLAT"); + + // Applying the predicate after the index chose its k would drop matching rows instead of + // fetching further down the ranking, so a filtered top-k stays on the scan. + var rows = Query( + db, + $"SELECT id, body, vec_distance_cosine(embedding, {QueryVectorLiteral}) AS distance " + + "FROM docs WHERE id > 1 ORDER BY distance LIMIT 2"); + + Assert.Equal(2, rows.Count); + Assert.DoesNotContain("distance", rows[0].Keys); + Assert.Contains("embedding", rows[0].Keys); + } + + [Fact] + public void DropVectorIndex_ReturnsTheQueryToTheScanPath() + { + var db = SeededDatabase(); + db.ExecuteSQL("CREATE VECTOR INDEX idx_docs ON docs(embedding) USING FLAT"); + + var optimized = Query( + db, + $"SELECT id, vec_distance_cosine(embedding, {QueryVectorLiteral}) AS distance " + + "FROM docs ORDER BY distance LIMIT 2"); + Assert.True(optimized[0].ContainsKey("distance")); + + db.ExecuteSQL("DROP VECTOR INDEX idx_docs ON docs"); + + var scanned = Query( + db, + $"SELECT id, vec_distance_cosine(embedding, {QueryVectorLiteral}) AS distance " + + "FROM docs ORDER BY distance LIMIT 2"); + Assert.DoesNotContain("distance", scanned[0].Keys); + } + + [Fact] + public void OrderByDistanceWithoutAnyIndex_FallsBackToTheScan() + { + var db = SeededDatabase(); + + var rows = Query( + db, + $"SELECT id, vec_distance_cosine(embedding, {QueryVectorLiteral}) AS distance " + + "FROM docs ORDER BY distance LIMIT 2"); + + Assert.Equal(2, rows.Count); + Assert.DoesNotContain("distance", rows[0].Keys); + } + + /// + /// Creates a database through the DI factory with four documents whose distances to + /// [1,0,0,0] are distinct enough to identify, and where two rows tie on distance so the + /// index's tie-break (id, ascending) is exercised too. + /// + /// The open database; the caller disposes it. + private IDatabase SeededDatabase() + { + if (_db is not null) + { + return _db; + } + + var factory = _provider.GetRequiredService(); + var db = factory.Create(_path, "pw", isReadOnly: false, config: new DatabaseConfig()); + + db.ExecuteSQL("CREATE TABLE docs (id INTEGER PRIMARY KEY, body TEXT, embedding VECTOR(4))"); + db.ExecuteSQL("INSERT INTO docs (id, body, embedding) VALUES (1, 'nearest', [1,0,0,0])"); + db.ExecuteSQL("INSERT INTO docs (id, body, embedding) VALUES (2, 'orthogonal', [0,1,0,0])"); + db.ExecuteSQL("INSERT INTO docs (id, body, embedding) VALUES (3, 'close', [0.8,0.6,0,0])"); + db.ExecuteSQL("INSERT INTO docs (id, body, embedding) VALUES (4, 'far', [0,0,1,0])"); + + _db = db; + return db; + } + + private static List> Query(IDatabase db, string sql) + => db.ExecuteQuery(sql, new Dictionary()); +} +