Native to_json_list: skip the walk for untagged results; RIDs and map keys JSON-safe

docs/modules/ROOT/pages/how-to/databases-and-queries.adoc

@@ -241,7 +241,7 @@ A query vector for `vectorNeighbors` can be a numpy array or a list. The convers
 
 === Record ids in rows
 
-A record id inside a row value (`SELECT @rid AS r`, a `LINK` property, a list of links such as `out('Knows').@rid`) comes back as an RID object, not as text. `str(rid)` is the text (`"#10:5"`), so is an f-string, and `get_rid()` always returns text or `None`. Two RIDs of the same record are equal and hash alike, so they work in sets and as dict keys, and RIDs sort by bucket, then position. `getBucketId()`, `getPosition()`, `toString()` and `isValid()` work on the object. It goes back in as a query parameter (`WHERE @rid = ?`, or a list passed by name) and to `db.lookup_by_rid()`, as itself or as its text.
+A record id inside a row value (`SELECT @rid AS r`, a `LINK` property, a list of links such as `out('Knows').@rid`) comes back as an RID object, not as text. `str(rid)` is the text (`"#10:5"`), so is an f-string, and `get_rid()` always returns text or `None`. Two RIDs of the same record are equal and hash alike, so they work in sets and as dict keys, and RIDs sort by bucket, then position. `getBucketId()`, `getPosition()`, `toString()` and `isValid()` work on the object. It goes back in as a query parameter (`WHERE @rid = ?`, or a list passed by name) and to `db.lookup_by_rid()`, as itself or as its text. `to_json_list()` is the exception: a record id in its rows is the text (`"#10:5"`), on both builds, and so is a map key that is not a string, because a JSON object has no other kind of key.
 
 On the native build the record wrappers send their record id to the engine as a parameter, not in the statement text. The engine keeps a parsed statement in a cache keyed by its text (`arcadedb.sqlStatementCache`, 300 entries by default) and leaves a record id written in a `FROM` target as part of that text, so each record would be a cache entry of its own. `get_out_edges()`, `get_in_edges()`, `get_both_edges()`, `new_edge()`, `Edge.get_in()`, `Edge.get_out()`, `save()` and `delete()` send the id as a parameter, so every record shares one text. `db.lookup_by_rid()` keeps the id in the statement text: measured, the parameter path was 2 % slower for a lookup, so it stays as it is.
 

src/arcadedb/_native/codec.py

@@ -383,7 +383,36 @@ def hook(obj):
     return json.loads(text, object_hook=hook)
 
 
-# --- JSON-native form of a row (ResultSet.to_json_list): DATE/DATETIME as epoch millis, DECIMAL as float.
+def loads_tagged(text: str, floats=None):
+    """:func:`loads` that also says whether a tagged value was restored: ``(value, restored)``.
+
+    ``restored`` is False for a reply without the escaped tag, and for one whose only tags are ``["B", offset, count]``
+    (they decode to plain float lists). Then every value is a ``json.loads`` type and :func:`json_native` returns it
+    unchanged, which is what lets ``to_json_list`` skip its walk. The decoded value is the one :func:`loads` returns.
+    """
+    if _TAG_ESCAPED not in text:
+        return json.loads(text), False
+    restored = False
+
+    def hook(obj):
+        nonlocal restored
+        tagged = obj.get(TAG)
+        if tagged is None or len(obj) != 1 or type(tagged) is not list:
+            return obj
+        size = len(tagged)
+        if size == 3 and floats is not None and tagged[0] == "B":
+            return floats[tagged[1] : tagged[1] + tagged[2]].tolist()
+        if size == 2 and tagged[0] in _SIMPLE:
+            restored = True
+            return _SIMPLE[tagged[0]](tagged[1])
+        return obj
+
+    value = json.loads(text, object_hook=hook)
+    return value, restored
+
+
+# --- JSON-native form of a row (ResultSet.to_json_list): DATE/DATETIME as epoch millis, DECIMAL as float, an RID as its text
+# (``"#10:5"``), sets as lists, and a map key that is not a string as the string the JVM build's JSON object writes.
 
 
 def _epoch_millis(delta):
@@ -395,7 +424,22 @@ def _epoch_millis(delta):
     return (delta.days * 86400 + delta.seconds) * 1000 + delta.microseconds // 1000
 
 
+def _json_key(key):
+    """The text of a map key that is not a string, as the JVM build's JSON object has it (``String.valueOf``)."""
+    if type(key) is str:
+        return key
+    if isinstance(key, RID):
+        return key.text
+    if key is None:
+        return "null"
+    if isinstance(key, bool):
+        return "true" if key else "false"
+    return str(key)
+
+
 def json_native(value):
+    if isinstance(value, RID):
+        return value.text
     if isinstance(value, datetime):
         return _epoch_millis(
             value - _EPOCH_AWARE if value.tzinfo is not None else value - _EPOCH_NAIVE
@@ -411,5 +455,8 @@ def json_native(value):
     if isinstance(value, list):
         return [json_native(v) for v in value]
     if isinstance(value, dict):
-        return {k: json_native(v) for k, v in value.items()}
+        return {
+            (k if type(k) is str else _json_key(k)): json_native(v)
+            for k, v in value.items()
+        }
     return value

src/arcadedb/_native/database.py

@@ -16,7 +16,7 @@
 from . import interrupts, lib, needs_jvm
 from .bulk import insert_rows_columnar
 from .codec import _prep as _prep_row
-from .codec import encode_params, json_native, loads, row_default
+from .codec import encode_params, json_native, loads, loads_tagged, row_default
 
 _LAZY_COMMAND_ROWS = os.environ.get("ARCADEDB_NATIVE_LAZY_CMD", "1") != "0"
 _BINARY_REPLY = os.environ.get("ARCADEDB_NATIVE_BIN_REPLY", "1") != "0"
@@ -164,8 +164,11 @@ def __repr__(self) -> str:
 class NativeResultSet:
     """Cursor over a statement's rows, fetched in batches through ``arcadedb_n_next``."""
 
-    def __init__(self, database, reply: dict):
+    def __init__(self, database, reply: dict, tagged: bool = True):
         self._database = database  # strong reference: a result keeps its database alive
+        # whether a reply of this result carried a restored tagged value (codec.loads_tagged), so a row may hold a value that is not
+        # JSON-native; to_json_list walks the rows only then. True is the safe answer for a caller that does not know
+        self._tagged = tagged
         self._rs = reply["rs"]
         self._buf = deque(zip(reply["rows"], reply["meta"]))
         self._more = not reply["drained"]
@@ -182,6 +185,7 @@ def _lazy(cls, database, rs_id: int) -> "NativeResultSet":
         self._database = database
         self._rs = rs_id
         self._buf = deque()
+        self._tagged = False
         self._more = rs_id > 0
         self._closed = False
         self._exhausted = False
@@ -218,13 +222,17 @@ def _fill(self) -> bool:
             return False
         try:
             if lib.NEXT_BIN and _BINARY_REPLY:
-                reply = loads(
+                reply, tagged = loads_tagged(
                     *lib.call_bin("arcadedb_n_next_bin", self._rs, _NEXT_BATCH)
                 )
             else:
-                reply = loads(lib.call("arcadedb_n_next", self._rs, _NEXT_BATCH))
+                reply, tagged = loads_tagged(
+                    lib.call("arcadedb_n_next", self._rs, _NEXT_BATCH)
+                )
         except ArcadeDBError:
             raise
+        if tagged:
+            self._tagged = True
         if not reply.get("ok"):
             self._more = False
             self._closed = True
@@ -291,9 +299,17 @@ def iter_dicts(self, convert_types: bool = True) -> Iterator[Dict[str, Any]]:
             yield row.to_dict()
 
     def to_json_list(self, batch_size: int = 10_000) -> List[Dict[str, Any]]:
+        """JSON-native rows as on the JVM build (:meth:`arcadedb.results.ResultSet.to_json_list`). The walk that converts DATE, DATETIME,
+        DECIMAL, sets and record ids runs only when a reply of this result carried such a value; otherwise the rows are returned as
+        parsed.
+        """
         if int(batch_size) < 1:
             raise ValueError("batch_size must be at least 1")
-        return [json_native(row) for row in self._drain_dicts()]
+        rows = self._drain_dicts()
+        if not self._tagged:
+            # no reply of this result restored a tagged value: every value is a json.loads type, which json_native returns as it is
+            return rows
+        return [json_native(row) for row in rows]
 
     def iter_json_batches(self, batch_size: int = 10_000, **kwargs):
         if int(batch_size) < 1:
@@ -561,10 +577,10 @@ def _exec(
                 lib.b(encode_params(args)),
                 _FIRST_BATCH,
             )
-        reply = loads(raw, floats)
+        reply, tagged = loads_tagged(raw, floats)
         if not reply.get("ok"):
             raise _error(prefix, reply)
-        return NativeResultSet(self, reply)
+        return NativeResultSet(self, reply, tagged)
 
     def query(self, language: str, command: str, *args) -> NativeResultSet:
         """Run a read statement. Parameters bind as on the JPype path (``?`` positional, ``:name`` from one dict)."""

src/arcadedb/results.py

@@ -568,6 +568,14 @@ def to_json_list(self, batch_size: int = 10_000) -> List[Dict[str, Any]]:
         floats. Use ``to_list()`` when you want ``date``, ``datetime`` and
         ``Decimal`` values: it costs about the same.
 
+        A record id (a ``LINK`` value, ``@rid``) arrives as its text
+        (``"#10:5"``), at the top level and inside a list or a map, and a map key
+        that is not a string arrives as text (``"1"``, ``"true"``, ``"null"``),
+        on both builds. On the native build the conversion over the rows runs
+        only for a result in which a reply carried a value that JSON cannot
+        carry (DECIMAL, DATE, DATETIME, a set, NaN, a record id, ...); any other
+        result is returned as parsed.
+
         NaN, Infinity and -Infinity arrive as Python floats (``float('nan')``,
         ``float('inf')``), at the top level and inside a list or a map, as on
         the native build. JSON has no literal for them, so the bridge writes

tests/conftest.py

@@ -57,6 +57,8 @@
     collect_ignore.append("test_native_async_callbacks.py")
     # the short call path of a point read (codec.encode_params, lib.exec_query, first()), observed on the native library
     collect_ignore.append("test_native_point_read.py")
+    # to_json_list walks the rows with codec.json_native only when a reply restored a tagged value (#219), observed on the native library
+    collect_ignore.append("test_native_json_list.py")
     # the vector provider the native glue picks by default (ARCADEDB_VECTOR_PROVIDER, #153), observed on the native library
     collect_ignore.append("test_native_vector_provider.py")
     # crossing counts of the transaction entry points (lib.call is the native library's ctypes door)