Native find_nearest: include_vector=False returns hits without the indexed vector (explicit opt-in)

docs/modules/ROOT/pages/how-to/vectors.adoc

@@ -157,7 +157,7 @@ print([row["name"] for row in rows])
 
 === find_nearest
 
-`VectorIndex.find_nearest(query_vector, k=10, ef_search=None, allowed_rids=None)` searches without a statement. It takes a list or a NumPy array and returns a list of `(record, distance)` tuples, nearest first. Use it when you already hold the index object, or when you have a list of record ids to search within.
+`VectorIndex.find_nearest(query_vector, k=10, ef_search=None, allowed_rids=None, *, include_vector=True)` searches without a statement. It takes a list or a NumPy array and returns a list of `(record, distance)` tuples, nearest first. Use it when you already hold the index object, or when you have a list of record ids to search within.
 
 [source,python]
 ----
@@ -171,6 +171,22 @@ python-ish
 python
 ----
 
+=== Hits without the vector
+
+Every hit of `find_nearest` carries its indexed vector, so a search at 1536 dimensions ships and decodes 1536 numbers per hit whether you read them or not. If you need only the ids, the distances or other properties of the hits, pass `include_vector=False`. On the native build each hit is then built without the vector. It loads the vector, with the rest of the record, in one lookup by record id the first time you read it (`get` of the vector, `has_property`, `to_dict`, `get_property_names`, `modify` or `save`), so a loop that reads the vector of every hit pays one lookup per hit and does more work than the default. The values are the ones `db.lookup_by_rid` returns, and a hit deleted since the search raises `ArcadeDBError` on that first read. The arcadedb-jvm package accepts the argument and returns whole records either way.
+
+[source,python]
+----
+for record, distance in index.find_nearest(query, k=2, include_vector=False):
+    print(record.get("name"))
+----
+
+[source,text]
+----
+python-ish
+python
+----
+
 === Filtered search
 
 `allowed_rids` restricts `find_nearest` to the records whose ids you list, as strings. The search considers only those records, so it still returns `k` of them.

src/arcadedb/_native/records.py

@@ -50,9 +50,17 @@ def __init__(self, database, type_name, rid=None, props=None, mutable=False):
         self._props: Dict[str, Any] = dict(props or {})
         self._mutable = mutable or not rid
         self._property_names_cache = None
+        # names of properties a search hit was built without (see ``_wrap_partial``); None for every other record
+        self._lazy = None
 
     # -- reading
 
+    def _fill(self) -> None:
+        """Load the properties a search hit was built without, in one lookup by RID (engine order); nothing to do for any other record."""
+        if self._lazy:
+            self._props = dict(self._db.lookup_by_rid(self._rid)._props)
+            self._lazy = None
+
     def _check_open(self) -> None:
         database = self._database
         if database is not None and database._closed and self._rid:
@@ -63,21 +71,27 @@ def _check_open(self) -> None:
 
     def get(self, name: str, convert_types: bool = True) -> Any:
         self._check_open()
+        if self._lazy and name in self._lazy:
+            self._fill()
         return self._props.get(name)
 
     def get_raw(self, name: str) -> Any:
         return self.get(name)
 
     def has_property(self, name: str) -> bool:
         self._check_open()
+        if self._lazy and name in self._lazy:
+            self._fill()
         return name in self._props
 
     def get_property_names(self) -> List[str]:
         self._check_open()
+        self._fill()
         return list(self._props)
 
     def to_dict(self, convert_types: bool = True) -> Dict[str, Any]:
         self._check_open()
+        self._fill()
         return dict(self._props)
 
     def get_rid(self) -> str:
@@ -106,6 +120,7 @@ def set(self, name: str, value: Any):
         return self
 
     def save(self):
+        self._fill()
         db = self._db
         if self._rid is None:
             from .extras import veto_count
@@ -138,6 +153,7 @@ def delete(self):
 
     def modify(self):
         self._check_open()
+        self._fill()
         return type(self)(self._db, self._type, self._rid, self._props, mutable=True)
 
     def __repr__(self) -> str:
@@ -218,3 +234,15 @@ def _wrap_dict(database, record: dict, kind: str = "d"):
     type_name = props.pop("@type", None)
     kind = props.pop("@cat", kind)
     return _CLASS_OF.get(kind, Document)(database, type_name, rid, props)
+
+
+def _wrap_partial(database, rid, type_name, kind, props, omitted):
+    """The wrapper for a search hit built without the properties named in ``omitted`` (the indexed vector: not shipped with every hit).
+
+    Reading one of them, or anything that needs the whole record (``to_dict``, ``get_property_names``, ``modify``, ``save``), loads the
+    record once by RID; ``get`` and ``has_property`` of any other property are answered from what was shipped.
+    """
+    cls = {"v": Vertex, "e": Edge}.get(kind, Document)
+    record = cls(database, type_name, rid, props)
+    record._lazy = frozenset(omitted)
+    return record

src/arcadedb/_native/schema_objects.py

@@ -9,6 +9,7 @@
 from ..exceptions import ArcadeDBError
 from ..vector import VectorIndex as _BaseVectorIndex
 from . import needs_jvm
+from .codec import _np
 
 _PROPERTY = re.compile(r"name: (.+?), type: ([A-Z_]+)")
 
@@ -169,6 +170,41 @@ def _rid_text(value) -> str:
     return get_rid() if get_rid is not None else str(value)
 
 
+def _query_vector_param(query_vector):
+    """The query vector as the one parameter the library takes it in.
+
+    A numeric 1-d vector (an ndarray, or a list/tuple of numbers: numpy walks a list in C) becomes a float32 array, which
+    ``encode_params`` sends through the binary float32 side buffer (tag ``B``): no ``tolist()``, no float32-exactness check of a
+    list, and no JSON number text for a float64 query whose values float32 would round. The engine reads the vector as float32
+    whatever it is given, so the rounding is the same. Anything else (text or None entries, a generator, numpy missing) stays a
+    list and reaches the engine as it was given; numpy would turn a None into nan.
+    """
+    if _np is not None:
+        try:
+            arr = _np.asarray(query_vector)
+        except (
+            Exception
+        ):  # nosec B110 - ragged or unconvertible input: the list path below reports it
+            arr = None
+        if arr is not None and arr.ndim == 1 and arr.dtype.kind in "fiu":
+            return arr.astype(_np.float32, copy=False)
+    return (
+        query_vector.tolist() if hasattr(query_vector, "tolist") else list(query_vector)
+    )
+
+
+def _quote_name(name: str) -> str:
+    return "`" + name.replace("`", "``") + "`"
+
+
+# find_nearest(include_vector=False): the engine's row of a hit is the record's properties plus the columns @rid, @type and distance
+# (ScoredRecordRow; before 26.11.1, ArcadeData/arcadedb#10898, also a nested `record` column holding every property a second time).
+# `*` copies the row, `!name` drops the indexed vector and the `record` column. A property of the record called distance or record is
+# hidden by those columns in the row, so such a hit holds them back too (_LAZY_COLUMNS) and loads them with the vector.
+_LAZY_COLUMNS = ("record", "distance")
+_HITS_FROM = "FROM (SELECT expand(vectorNeighbors(?, ?, ?)))"
+
+
 class NativeVectorIndex(_BaseVectorIndex):
     """``VectorIndex`` over ``vectorNeighbors``: ``find_nearest`` returns ``[(record, distance)]`` like the JPype wrapper."""
 
@@ -186,17 +222,24 @@ def _check_live(self):
         return
 
     def _run_search(
-        self, *, query_vector, k, allowed_rids, approximate, ef_search=None
+        self,
+        *,
+        query_vector,
+        k,
+        allowed_rids,
+        approximate,
+        ef_search=None,
+        include_vector=True,
     ):
         if approximate:
             return self._run_approximate(query_vector, k, allowed_rids)
+        if not include_vector:
+            return self._run_search_without_vector(
+                query_vector, k, allowed_rids, ef_search
+            )
         from .records import _wrap_dict
 
-        vector = (
-            query_vector.tolist()
-            if hasattr(query_vector, "tolist")
-            else list(query_vector)
-        )
+        vector = _query_vector_param(query_vector)
         # the statement's own options: efSearch, and filter (the RIDs the search may return), as the JPype path passes them
         options = {}
         if ef_search is not None:
@@ -237,6 +280,54 @@ def _run_search(
             out.append((record, float(row.get("distance"))))
         return self._sort_results(out)[:k]
 
+    def _run_search_without_vector(self, query_vector, k, allowed_rids, ef_search):
+        """``find_nearest(include_vector=False)``: the hits come back without the indexed vector, which loads on first use.
+
+        ``expand(vectorNeighbors(..))`` alone ships every hit's vector, and the engine reads each record to build the row whatever
+        the columns asked for. Here the statement drops the vector (and the nested ``record`` column of an engine before 26.11.1),
+        and ``_Mixin._fill`` loads the record, once, by RID, the first time the vector or the whole record is needed.
+        """
+        from .records import _wrap_partial
+
+        vector = _query_vector_param(query_vector)
+        options = {}
+        if ef_search is not None:
+            options["efSearch"] = int(ef_search)
+        if allowed_rids:
+            options["filter"] = [_rid_text(r) for r in allowed_rids]
+        prop = self._shim._prop
+        dropped = ", ".join(
+            "!" + _quote_name(name) for name in dict.fromkeys((prop, "record"))
+        )
+        sql = f"SELECT *, {dropped} {_HITS_FROM}"  # nosec B608 - dropped is backtick-quoted schema names, the values go through ? parameters
+        if options:
+            sql = sql.replace("vectorNeighbors(?, ?, ?)", "vectorNeighbors(?, ?, ?, ?)")
+            rows = self._database.query(
+                "sql", sql, self._shim._name, vector, int(k), options
+            )
+        else:
+            rows = self._database.query("sql", sql, self._shim._name, vector, int(k))
+        omitted = (prop, *_LAZY_COLUMNS)
+        out = []
+        for row in rows:
+            props = row.to_dict()
+            rid = props.pop("@rid", None)
+            type_name = props.pop("@type", None)
+            distance = props.pop("distance")
+            if rid is None:
+                record = None
+            else:
+                record = _wrap_partial(
+                    self._database,
+                    _rid_text(rid),
+                    type_name,
+                    self._database._kind_of(type_name),
+                    props,
+                    omitted,
+                )
+            out.append((record, float(distance)))
+        return self._sort_results(out)[:k]
+
     def _run_approximate(self, query_vector, k, allowed_rids):
         """The PQ-scored search of the engine (no statement runs it): the engine returns (RID, distance), the records are loaded here."""
         from . import lib

src/arcadedb/vector.py

@@ -566,6 +566,7 @@ def _run_search(
         allowed_rids,
         approximate,
         ef_search=None,
+        include_vector=True,  # accepted for API parity, the JVM build loads whole records
     ):
         java_vector = to_java_float_array(query_vector)
         allowed_rids_set = self._build_allowed_rids_set(allowed_rids)
@@ -584,6 +585,8 @@ def find_nearest(
         k=10,
         ef_search=None,
         allowed_rids=None,
+        *,
+        include_vector=True,
     ):
         """
         Find k nearest neighbors to the query vector.
@@ -604,16 +607,31 @@ def find_nearest(
                 uses the Java index default/adaptive behavior.
             allowed_rids: Optional list of RID strings (e.g. ["#1:0", "#2:5"]) to
                 restrict search
+            include_vector: Keyword only, default ``True``: every hit carries its
+                indexed vector, as always. Pass ``False`` when you need only the
+                ids or the distances (or other properties) of the hits. On the
+                native build each hit is then built without the indexed vector, and
+                loads it, with the rest of the record, in one lookup by RID the
+                first time you read it (``get`` of the vector, ``has_property``,
+                ``to_dict``, ``get_property_names``, ``modify``, ``save``); the
+                values are those of ``lookup_by_rid``, and a hit deleted since the
+                search raises ``ArcadeDBError`` on that first read. The JVM build
+                accepts the argument and ignores it: it loads whole records either
+                way, so the call costs the same.
 
         Returns:
             List of tuples: [(record, score), ...]
             - record: The matching ArcadeDB record (Vertex, Document, or Edge)
             - score: The similarity score/distance
         """
         self._check_live()
-        return self._find_nearest_live(query_vector, k, ef_search, allowed_rids)
+        return self._find_nearest_live(
+            query_vector, k, ef_search, allowed_rids, include_vector
+        )
 
-    def _find_nearest_live(self, query_vector, k, ef_search, allowed_rids):
+    def _find_nearest_live(
+        self, query_vector, k, ef_search, allowed_rids, include_vector=True
+    ):
         # find_nearest after _check_live(): find_nearest_by_key checks once and
         # comes here, so a call never pays the isValid() crossing twice.
         effective_ef_search = self._normalize_ef_search(ef_search)
@@ -625,6 +643,7 @@ def _find_nearest_live(self, query_vector, k, ef_search, allowed_rids):
                 allowed_rids=allowed_rids,
                 approximate=False,
                 ef_search=effective_ef_search,
+                include_vector=include_vector,
             )
 
         except ArcadeDBError:

tests/conftest.py

@@ -38,6 +38,10 @@
 if BACKEND != "native":
     # Java -> Python upcalls (Python functions in SQL) exist on the native build only
     collect_ignore.append("test_native_udf.py")
+    # how find_nearest hands the query vector to the native library (the codec's side buffer)
+    collect_ignore.append("test_native_find_nearest_params.py")
+    # find_nearest(include_vector=False) on the native wheel fills a hit's vector on first use (counts lookups of the native wrapper)
+    collect_ignore.append("test_native_vector_hit_lazy.py")
     # the columnar insert's stored records are compared between the JSON and the columnar entry points of the native library
     collect_ignore.append("test_native_columnar_insert.py")
     # the native build's ingest defaults (insert_many and graph_batch commit size), observed on the native library