From 88aafa51b24e57b6f35a0be38477398c9b5c3c85 Mon Sep 17 00:00:00 2001 From: Taewoon Kim Date: Wed, 4 Nov 2026 01:04:47 +0900 Subject: [PATCH 1/2] Native to_json_list: skip the json_native walk when no reply was tagged; RIDs and map keys JSON-safe (closes #219) A reply without the escaped tag decodes to json.loads types, which json_native returns unchanged, so the walk over every value was overhead (51 to 95 percent of the to_json_list wall on paper-shaped results up to 100 rows in the untimed probe). loads_tagged says whether a tagged value was restored; the result set keeps the flag across every reply it decodes (sticky) and to_json_list walks only when it is set. A reply whose only tags are float-tail "B" tags decodes to plain float lists and does not set it. json_native returned the RID object for a link, which json.dumps refuses, where the JVM build writes the text. It now returns the text and writes map keys that are not strings as the JVM build's JSON object does (RID text, "1", "true", "null"). Co-Authored-By: Claude Opus 5.5 --- src/arcadedb/_native/codec.py | 51 ++++- src/arcadedb/_native/database.py | 26 ++- tests/conftest.py | 2 + tests/test_native_json_list.py | 370 +++++++++++++++++++++++++++++++ tests/test_to_json_list_links.py | 67 ++++++ 5 files changed, 507 insertions(+), 9 deletions(-) create mode 100644 tests/test_native_json_list.py create mode 100644 tests/test_to_json_list_links.py diff --git a/src/arcadedb/_native/codec.py b/src/arcadedb/_native/codec.py index 929f548..c2aaeee 100644 --- a/src/arcadedb/_native/codec.py +++ b/src/arcadedb/_native/codec.py @@ -383,7 +383,36 @@ def hook(obj): return json.loads(text, object_hook=hook) -# --- JSON-native form of a row (ResultSet.to_json_list): DATE/DATETIME as epoch millis, DECIMAL as float. +def loads_tagged(text: str, floats=None): + """:func:`loads` that also says whether a tagged value was restored: ``(value, restored)``. + + ``restored`` is False for a reply without the escaped tag, and for one whose only tags are ``["B", offset, count]`` + (they decode to plain float lists). Then every value is a ``json.loads`` type and :func:`json_native` returns it + unchanged, which is what lets ``to_json_list`` skip its walk. The decoded value is the one :func:`loads` returns. + """ + if _TAG_ESCAPED not in text: + return json.loads(text), False + restored = False + + def hook(obj): + nonlocal restored + tagged = obj.get(TAG) + if tagged is None or len(obj) != 1 or type(tagged) is not list: + return obj + size = len(tagged) + if size == 3 and floats is not None and tagged[0] == "B": + return floats[tagged[1] : tagged[1] + tagged[2]].tolist() + if size == 2 and tagged[0] in _SIMPLE: + restored = True + return _SIMPLE[tagged[0]](tagged[1]) + return obj + + value = json.loads(text, object_hook=hook) + return value, restored + + +# --- JSON-native form of a row (ResultSet.to_json_list): DATE/DATETIME as epoch millis, DECIMAL as float, an RID as its text +# (``"#10:5"``), sets as lists, and a map key that is not a string as the string the JVM build's JSON object writes. def _epoch_millis(delta): @@ -395,7 +424,22 @@ def _epoch_millis(delta): return (delta.days * 86400 + delta.seconds) * 1000 + delta.microseconds // 1000 +def _json_key(key): + """The text of a map key that is not a string, as the JVM build's JSON object has it (``String.valueOf``).""" + if type(key) is str: + return key + if isinstance(key, RID): + return key.text + if key is None: + return "null" + if isinstance(key, bool): + return "true" if key else "false" + return str(key) + + def json_native(value): + if isinstance(value, RID): + return value.text if isinstance(value, datetime): return _epoch_millis( value - _EPOCH_AWARE if value.tzinfo is not None else value - _EPOCH_NAIVE @@ -411,5 +455,8 @@ def json_native(value): if isinstance(value, list): return [json_native(v) for v in value] if isinstance(value, dict): - return {k: json_native(v) for k, v in value.items()} + return { + (k if type(k) is str else _json_key(k)): json_native(v) + for k, v in value.items() + } return value diff --git a/src/arcadedb/_native/database.py b/src/arcadedb/_native/database.py index 8fb321f..57216b7 100644 --- a/src/arcadedb/_native/database.py +++ b/src/arcadedb/_native/database.py @@ -16,7 +16,7 @@ from . import interrupts, lib, needs_jvm from .bulk import insert_rows_columnar from .codec import _prep as _prep_row -from .codec import encode_params, json_native, loads, row_default +from .codec import encode_params, json_native, loads, loads_tagged, row_default _LAZY_COMMAND_ROWS = os.environ.get("ARCADEDB_NATIVE_LAZY_CMD", "1") != "0" _BINARY_REPLY = os.environ.get("ARCADEDB_NATIVE_BIN_REPLY", "1") != "0" @@ -164,8 +164,11 @@ def __repr__(self) -> str: class NativeResultSet: """Cursor over a statement's rows, fetched in batches through ``arcadedb_n_next``.""" - def __init__(self, database, reply: dict): + def __init__(self, database, reply: dict, tagged: bool = True): self._database = database # strong reference: a result keeps its database alive + # whether a reply of this result carried a restored tagged value (codec.loads_tagged), so a row may hold a value that is not + # JSON-native; to_json_list walks the rows only then. True is the safe answer for a caller that does not know + self._tagged = tagged self._rs = reply["rs"] self._buf = deque(zip(reply["rows"], reply["meta"])) self._more = not reply["drained"] @@ -182,6 +185,7 @@ def _lazy(cls, database, rs_id: int) -> "NativeResultSet": self._database = database self._rs = rs_id self._buf = deque() + self._tagged = False self._more = rs_id > 0 self._closed = False self._exhausted = False @@ -218,13 +222,17 @@ def _fill(self) -> bool: return False try: if lib.NEXT_BIN and _BINARY_REPLY: - reply = loads( + reply, tagged = loads_tagged( *lib.call_bin("arcadedb_n_next_bin", self._rs, _NEXT_BATCH) ) else: - reply = loads(lib.call("arcadedb_n_next", self._rs, _NEXT_BATCH)) + reply, tagged = loads_tagged( + lib.call("arcadedb_n_next", self._rs, _NEXT_BATCH) + ) except ArcadeDBError: raise + if tagged: + self._tagged = True if not reply.get("ok"): self._more = False self._closed = True @@ -293,7 +301,11 @@ def iter_dicts(self, convert_types: bool = True) -> Iterator[Dict[str, Any]]: def to_json_list(self, batch_size: int = 10_000) -> List[Dict[str, Any]]: if int(batch_size) < 1: raise ValueError("batch_size must be at least 1") - return [json_native(row) for row in self._drain_dicts()] + rows = self._drain_dicts() + if not self._tagged: + # no reply of this result restored a tagged value: every value is a json.loads type, which json_native returns as it is + return rows + return [json_native(row) for row in rows] def iter_json_batches(self, batch_size: int = 10_000, **kwargs): if int(batch_size) < 1: @@ -561,10 +573,10 @@ def _exec( lib.b(encode_params(args)), _FIRST_BATCH, ) - reply = loads(raw, floats) + reply, tagged = loads_tagged(raw, floats) if not reply.get("ok"): raise _error(prefix, reply) - return NativeResultSet(self, reply) + return NativeResultSet(self, reply, tagged) def query(self, language: str, command: str, *args) -> NativeResultSet: """Run a read statement. Parameters bind as on the JPype path (``?`` positional, ``:name`` from one dict).""" diff --git a/tests/conftest.py b/tests/conftest.py index 01ab332..4c54976 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -57,6 +57,8 @@ collect_ignore.append("test_native_async_callbacks.py") # the short call path of a point read (codec.encode_params, lib.exec_query, first()), observed on the native library collect_ignore.append("test_native_point_read.py") + # to_json_list walks the rows with codec.json_native only when a reply restored a tagged value (#219), observed on the native library + collect_ignore.append("test_native_json_list.py") # the vector provider the native glue picks by default (ARCADEDB_VECTOR_PROVIDER, #153), observed on the native library collect_ignore.append("test_native_vector_provider.py") # crossing counts of the transaction entry points (lib.call is the native library's ctypes door) diff --git a/tests/test_native_json_list.py b/tests/test_native_json_list.py new file mode 100644 index 0000000..bb4df26 --- /dev/null +++ b/tests/test_native_json_list.py @@ -0,0 +1,370 @@ +"""``to_json_list`` on the native build walks the rows with ``codec.json_native`` only when a reply of the result restored a tagged value. + +A reply without the escaped tag decodes to ``json.loads`` types, which ``json_native`` returns unchanged, so the walk over every value of +every row was pure overhead (#219). A restored tag (DATE, DATETIME, DECIMAL, a set, an RID, a map with non-string keys) still takes it: +the rows are then the JSON-native rows they always were. The flag is per result and sticky: a result whose first reply is untagged and a +later one is tagged (or the other way round) walks. A reply whose only tags are ``["B", offset, count]`` (the float tail of vectors) +decodes to plain float lists and does not. conftest.py collects this file on the native backend only (the JVM build has no such walk); +the values the two builds return for a link are compared in test_to_json_list_links.py. +""" + +import calendar +import json +import struct +from datetime import date, datetime +from decimal import Decimal + +import numpy as np +import pytest + +from arcadedb._native import codec, database +from arcadedb._native.codec import RID + +DIM = ( + 16 # 8 or more floats per vector: they travel in the float32 tail of a binary reply +) + + +def f32(x): + return struct.unpack("f", struct.pack("f", x))[0] + + +def _ms(year, month, day): + return calendar.timegm((year, month, day, 0, 0, 0)) * 1000 + + +@pytest.fixture +def spy(monkeypatch): + """Counts the rows ``json_native`` is called for (the walk) and records, per decoded reply, ``(restored, has float tail)``.""" + seen = {"walk": 0, "replies": []} + real_walk = database.json_native + + def walk(value): + seen["walk"] += 1 + return real_walk(value) + + monkeypatch.setattr(database, "json_native", walk) + real_loads = getattr( + database, "loads_tagged", None + ) # absent on a build that has no flag + if real_loads is not None: + + def loads_tagged(text, floats=None): + value, restored = real_loads(text, floats) + seen["replies"].append( + (restored, floats is not None and len(floats) > 0 and "\\u0001" in text) + ) + return value, restored + + monkeypatch.setattr(database, "loads_tagged", loads_tagged) + return seen + + +def _docs(db, rows, **properties): + db.command("sql", "CREATE DOCUMENT TYPE Doc") + db.command("sql", "CREATE PROPERTY Doc.id INTEGER") + for name, type_name in properties.items(): + db.command("sql", f"CREATE PROPERTY Doc.{name} {type_name}") + db.insert_many("Doc", rows) + return db + + +class TestUntaggedResultSkipsTheWalk: + def test_a_short_untagged_result_is_not_walked(self, temp_db, spy): + db = _docs( + temp_db, + [ + {"id": i, "name": f"n{i}", "score": i / 4, "ok": i % 2 == 0} + for i in range(5) + ], + name="STRING", + score="DOUBLE", + ok="BOOLEAN", + ) + spy["walk"] = 0 + got = db.query( + "sql", "SELECT id, name, score, ok FROM Doc ORDER BY id" + ).to_json_list() + assert spy["walk"] == 0 + assert spy["replies"][-1] == (False, False) + assert got == [ + {"id": i, "name": f"n{i}", "score": i / 4, "ok": i % 2 == 0} + for i in range(5) + ] + assert ( + got + == db.query( + "sql", "SELECT id, name, score, ok FROM Doc ORDER BY id" + ).to_list() + ) + + def test_every_batch_of_a_long_untagged_result_is_untagged_and_not_walked( + self, temp_db, spy + ): + db = _docs( + temp_db, [{"id": i, "name": f"n{i}"} for i in range(3000)], name="STRING" + ) + spy["walk"] = 0 + spy["replies"].clear() + got = db.query("sql", "SELECT id, name FROM Doc ORDER BY id").to_json_list() + assert ( + len(spy["replies"]) >= 3 + ) # 256, then 2048 + 696 through arcadedb_n_next_bin + assert all(restored is False for restored, _ in spy["replies"]) + assert spy["walk"] == 0 + assert [r["id"] for r in got] == list(range(3000)) + + def test_an_empty_result_is_not_walked(self, temp_db, spy): + db = _docs(temp_db, []) + spy["walk"] = 0 + assert db.query("sql", "SELECT FROM Doc").to_json_list() == [] + assert spy["walk"] == 0 + + def test_the_rows_are_the_decoded_rows(self, temp_db): + db = _docs(temp_db, [{"id": 1}]) + result = db.query("sql", "SELECT id FROM Doc") + assert result.to_json_list() == [{"id": 1}] + assert result._tagged is False + + +class TestTaggedResultStillWalks: + ROW = { + "id": 1, + "d": date(2026, 1, 2), + "dt": datetime(2026, 1, 2, 3, 4, 5, 678000), + "amount": Decimal("1.50"), + } + + def _db(self, temp_db): + return _docs(temp_db, [self.ROW], d="DATE", dt="DATETIME", amount="DECIMAL") + + def test_a_tagged_reply_is_walked_and_converted(self, temp_db, spy): + db = self._db(temp_db) + spy["walk"] = 0 + got = db.query("sql", "SELECT id, d, dt, amount FROM Doc").to_json_list() + assert spy["walk"] == 1 + assert spy["replies"][-1][0] is True + row = got[0] + assert row["id"] == 1 + assert type(row["d"]) is int and row["d"] == _ms(2026, 1, 2) + assert ( + type(row["dt"]) is int + and row["dt"] == _ms(2026, 1, 2) + (3 * 3600 + 4 * 60 + 5) * 1000 + 678 + ) + assert type(row["amount"]) is float and row["amount"] == 1.5 + json.dumps(got) + + def test_to_list_keeps_the_python_types(self, temp_db): + db = self._db(temp_db) + row = db.query("sql", "SELECT id, d, dt, amount FROM Doc").to_list()[0] + assert row["d"] == date(2026, 1, 2) + assert isinstance(row["dt"], datetime) + assert row["amount"] == Decimal("1.50") and isinstance(row["amount"], Decimal) + + def test_the_first_reply_untagged_and_a_later_one_tagged_walks(self, temp_db, spy): + rows = [{"id": i} for i in range(300)] + for row in rows[270:]: + row["d"] = date(2026, 1, 2) + db = _docs(temp_db, rows, d="DATE") + spy["walk"] = 0 + spy["replies"].clear() + got = db.query("sql", "SELECT id, d FROM Doc ORDER BY id").to_json_list() + flags = [restored for restored, _ in spy["replies"]] + assert ( + flags[0] is False and True in flags[1:] + ) # the premise: untagged first, tagged later + assert spy["walk"] == 300 + assert got[0].get("d") is None + assert got[299]["d"] == _ms(2026, 1, 2) and type(got[299]["d"]) is int + json.dumps(got) + + def test_the_first_reply_tagged_and_a_later_one_untagged_walks(self, temp_db, spy): + rows = [{"id": i} for i in range(300)] + for row in rows[:10]: + row["d"] = date(2026, 1, 2) + db = _docs(temp_db, rows, d="DATE") + spy["walk"] = 0 + spy["replies"].clear() + got = db.query("sql", "SELECT id, d FROM Doc ORDER BY id").to_json_list() + flags = [restored for restored, _ in spy["replies"]] + assert flags[0] is True and False in flags[1:] + assert spy["walk"] == 300 + assert got[0]["d"] == _ms(2026, 1, 2) and type(got[0]["d"]) is int + json.dumps(got) + + def test_a_flag_read_from_a_lazy_command_result_covers_every_batch( + self, temp_db, spy + ): + rows = [{"id": i} for i in range(300)] + for row in rows[270:]: + row["d"] = date(2026, 1, 2) + db = _docs(temp_db, rows, d="DATE") + spy["walk"] = 0 + got = db.command("sql", "SELECT id, d FROM Doc ORDER BY id").to_json_list() + assert spy["walk"] == 300 + assert got[299]["d"] == _ms(2026, 1, 2) + + +class TestBinaryVectorReply: + def _vectors(self, db, count): + rows = [ + {"id": i, "vec": [f32((i * DIM + k) / 1000.0 - 3.0) for k in range(DIM)]} + for i in range(count) + ] + _docs(db, rows, vec="ARRAY_OF_FLOATS") + return rows + + def test_a_reply_with_only_float_tail_tags_is_not_walked_and_stays_plain( + self, temp_db, spy + ): + rows = self._vectors(temp_db, 1000) + spy["walk"] = 0 + spy["replies"].clear() + got = temp_db.query("sql", "SELECT id, vec FROM Doc ORDER BY id").to_json_list() + assert any(tail for _, tail in spy["replies"]) # the float tail was used + assert all(restored is False for restored, _ in spy["replies"]) + assert spy["walk"] == 0 + assert [r["id"] for r in got] == [r["id"] for r in rows] + for have, want in zip(got, rows): + assert type(have["vec"]) is list + assert all(type(x) is float for x in have["vec"]) + assert have["vec"] == want["vec"] + json.dumps(got) + + def test_a_reply_with_vectors_and_a_date_walks_and_keeps_the_vectors( + self, temp_db, spy + ): + rows = [ + { + "id": i, + "d": date(2026, 1, 2), + "vec": [f32(i + k / 8.0) for k in range(DIM)], + } + for i in range(20) + ] + _docs(temp_db, rows, d="DATE", vec="ARRAY_OF_FLOATS") + spy["walk"] = 0 + spy["replies"].clear() + got = temp_db.query( + "sql", "SELECT id, d, vec FROM Doc ORDER BY id" + ).to_json_list() + assert any(tail for _, tail in spy["replies"]) + assert spy["walk"] == 20 + assert all(r["d"] == _ms(2026, 1, 2) for r in got) + assert [r["vec"] for r in got] == [r["vec"] for r in rows] + json.dumps(got) + + +class TestLoadsTagged: + def _text(self, rows): + meta = [[f"#3:{i}", "T", "d"] for i in range(len(rows))] + return json.dumps( + {"ok": True, "rs": 1, "rows": rows, "meta": meta, "drained": True}, + separators=(",", ":"), + ) + + @pytest.mark.parametrize( + "rows, restored", + [ + ([{"a": 1, "s": "x"}], False), + ( + [{"s": "a\u0001b"}], + False, + ), # the control character in a string is not a tag + ([{codec.TAG: "plain", "x": 1}], False), # a column named like the tag + ([{"d": {codec.TAG: ["D", "2026-10-31"]}}], True), + ([{"n": {codec.TAG: ["n", "1.50"]}}], True), + ([{"r": {codec.TAG: ["R", "#3:4"]}}], True), + ([{"m": {codec.TAG: ["m", [[1, "a"]]]}}], True), + ], + ) + def test_restored_flag_and_same_value_as_loads(self, rows, restored): + text = self._text(rows) + value, flag = codec.loads_tagged(text) + assert flag is restored + assert value == codec.loads(text) + + def test_a_float_tail_tag_alone_is_not_a_restored_value(self): + floats = np.array([0.25, 0.5, 0.75], dtype=np.float32) + text = self._text([{"v": {codec.TAG: ["B", 0, 3]}}]) + value, flag = codec.loads_tagged(text, floats) + assert flag is False + assert value["rows"][0]["v"] == [0.25, 0.5, 0.75] + assert value == codec.loads(text, floats) + + def test_a_float_tail_tag_next_to_a_date_is_restored(self): + floats = np.array([0.25, 0.5], dtype=np.float32) + text = self._text( + [{"v": {codec.TAG: ["B", 0, 2]}, "d": {codec.TAG: ["D", "2026-10-31"]}}] + ) + value, flag = codec.loads_tagged(text, floats) + assert flag is True + assert value == codec.loads(text, floats) + + +class TestJsonNativeIsJsonSafe: + def test_an_rid_is_its_text(self): + out = codec.json_native({"r": RID("#10:5")}) + assert type(out["r"]) is str and out["r"] == "#10:5" + + def test_an_rid_in_a_list_is_its_text(self): + out = codec.json_native({"l": [RID("#1:0"), RID("#1:1")]}) + assert [type(x) for x in out["l"]] == [str, str] + assert out == {"l": ["#1:0", "#1:1"]} + json.dumps(out) + + def test_an_rid_key_of_a_map_is_its_text(self): + out = codec.json_native({"m": {RID("#1:0"): 1, RID("#1:1"): [RID("#2:2")]}}) + assert [type(k) for k in out["m"]] == [str, str] + assert out == {"m": {"#1:0": 1, "#1:1": ["#2:2"]}} + assert type(out["m"]["#1:1"][0]) is str + json.dumps(out) + + def test_other_keys_that_are_not_strings_become_the_text_the_jvm_build_writes(self): + out = codec.json_native( + { + "m": {1: "a", 2.5: "b", None: "d", "k": "e"}, + "t": {True: "c"}, + "f": {False: 0}, + } + ) + assert out == { + "m": {"1": "a", "2.5": "b", "null": "d", "k": "e"}, + "t": {"true": "c"}, + "f": {"false": 0}, + } + assert all(type(k) is str for m in out.values() for k in m) + json.dumps(out) + + def test_the_other_conversions_are_unchanged(self): + out = codec.json_native( + { + "d": date(2026, 1, 2), + "n": Decimal("1.5"), + "s": {7}, + "f": frozenset({8}), + "l": [1, "x", None], + } + ) + assert out == { + "d": _ms(2026, 1, 2), + "n": 1.5, + "s": [7], + "f": [8], + "l": [1, "x", None], + } + assert type(out["d"]) is int and type(out["n"]) is float + + +class TestToListIsUnchanged: + def test_to_list_still_returns_rid_objects_and_non_string_keys(self, temp_db): + db = temp_db + db.command("sql", "CREATE VERTEX TYPE P") + db.command("sql", "CREATE DOCUMENT TYPE D") + db.command("sql", "CREATE PROPERTY D.m MAP") + with db.transaction(): + db.command("sql", "CREATE VERTEX P SET n = 'a'") + db.command("sql", "INSERT INTO D SET m = :m", {"m": {1: "a"}}) + row = db.query("sql", "SELECT @rid AS r FROM P").to_list()[0] + assert isinstance(row["r"], RID) + assert db.query("sql", "SELECT m FROM D").to_list() == [{"m": {1: "a"}}] diff --git a/tests/test_to_json_list_links.py b/tests/test_to_json_list_links.py new file mode 100644 index 0000000..e110491 --- /dev/null +++ b/tests/test_to_json_list_links.py @@ -0,0 +1,67 @@ +"""``to_json_list`` returns the text of a link (``"#10:5"``) on both builds, in a column, in a list and inside a map, and a map +with keys that are not strings comes back with string keys (a JSON object has no other kind). The native build returned its ``RID`` +object for a link, which ``json.dumps`` refuses (#219); the JVM build writes the text. The expected values are the JVM build's. +""" + +import json + +import pytest + + +@pytest.fixture +def linked(temp_db): + db = temp_db + db.command("sql", "CREATE VERTEX TYPE P") + db.command("sql", "CREATE PROPERTY P.friend LINK") + db.command("sql", "CREATE PROPERTY P.friends LIST OF LINK") + db.command("sql", "CREATE DOCUMENT TYPE D") + db.command("sql", "CREATE PROPERTY D.m MAP") + with db.transaction(): + db.command("sql", "CREATE VERTEX P SET n = 'a'") + db.command("sql", "CREATE VERTEX P SET n = 'b'") + db.command( + "sql", + "UPDATE P SET friend = (SELECT FROM P WHERE n = 'a'), friends = (SELECT FROM P) WHERE n = 'b'", + ) + db.command("sql", "INSERT INTO D SET m = :m", {"m": {1: "a", 2: "b"}}) + rids = [ + row["s"] + for row in db.query( + "sql", "SELECT @rid.asString() AS s FROM P ORDER BY n" + ).to_list() + ] + assert len(rids) == 2 and all(r.startswith("#") for r in rids) + return db, rids + + +def test_a_link_in_a_column_is_its_text(linked): + db, (a, b) = linked + got = db.query("sql", "SELECT @rid AS r FROM P ORDER BY n").to_json_list() + assert got == [{"r": a}, {"r": b}] + assert all(type(row["r"]) is str for row in got) + json.dumps(got) + + +def test_a_link_property_and_a_list_of_links_are_texts(linked): + db, (a, b) = linked + got = db.query("sql", "SELECT friend, friends FROM P WHERE n = 'b'").to_json_list() + assert got == [{"friend": a, "friends": [a, b]}] + assert type(got[0]["friend"]) is str + assert [type(x) for x in got[0]["friends"]] == [str, str] + json.dumps(got) + + +def test_a_link_inside_a_map_is_its_text(linked): + db, (a, b) = linked + got = db.query("sql", "SELECT {'k': @rid} AS m FROM P WHERE n = 'b'").to_json_list() + assert got == [{"m": {"k": b}}] + assert type(got[0]["m"]["k"]) is str + json.dumps(got) + + +def test_a_map_with_integer_keys_has_string_keys(linked): + db, _ = linked + got = db.query("sql", "SELECT m FROM D").to_json_list() + assert got == [{"m": {"1": "a", "2": "b"}}] + assert all(type(k) is str for k in got[0]["m"]) + json.dumps(got) From 7c175922d0a8ca8875cea030d5266ccf39096cf4 Mon Sep 17 00:00:00 2001 From: Taewoon Kim Date: Wed, 4 Nov 2026 11:04:48 +0900 Subject: [PATCH 2/2] Docs: to_json_list gives record ids and non-string map keys as text on both builds; native skips the walk for untagged results (PR 223) Results how-to and the to_json_list docstrings (JVM ResultSet, NativeResultSet). Docs and docstrings only. Co-Authored-By: Claude Opus 5.5 --- docs/modules/ROOT/pages/how-to/databases-and-queries.adoc | 2 +- src/arcadedb/_native/database.py | 4 ++++ src/arcadedb/results.py | 8 ++++++++ 3 files changed, 13 insertions(+), 1 deletion(-) diff --git a/docs/modules/ROOT/pages/how-to/databases-and-queries.adoc b/docs/modules/ROOT/pages/how-to/databases-and-queries.adoc index 3b372fd..ba9d927 100644 --- a/docs/modules/ROOT/pages/how-to/databases-and-queries.adoc +++ b/docs/modules/ROOT/pages/how-to/databases-and-queries.adoc @@ -241,7 +241,7 @@ A query vector for `vectorNeighbors` can be a numpy array or a list. The convers === Record ids in rows -A record id inside a row value (`SELECT @rid AS r`, a `LINK` property, a list of links such as `out('Knows').@rid`) comes back as an RID object, not as text. `str(rid)` is the text (`"#10:5"`), so is an f-string, and `get_rid()` always returns text or `None`. Two RIDs of the same record are equal and hash alike, so they work in sets and as dict keys, and RIDs sort by bucket, then position. `getBucketId()`, `getPosition()`, `toString()` and `isValid()` work on the object. It goes back in as a query parameter (`WHERE @rid = ?`, or a list passed by name) and to `db.lookup_by_rid()`, as itself or as its text. +A record id inside a row value (`SELECT @rid AS r`, a `LINK` property, a list of links such as `out('Knows').@rid`) comes back as an RID object, not as text. `str(rid)` is the text (`"#10:5"`), so is an f-string, and `get_rid()` always returns text or `None`. Two RIDs of the same record are equal and hash alike, so they work in sets and as dict keys, and RIDs sort by bucket, then position. `getBucketId()`, `getPosition()`, `toString()` and `isValid()` work on the object. It goes back in as a query parameter (`WHERE @rid = ?`, or a list passed by name) and to `db.lookup_by_rid()`, as itself or as its text. `to_json_list()` is the exception: a record id in its rows is the text (`"#10:5"`), on both builds, and so is a map key that is not a string, because a JSON object has no other kind of key. On the native build the record wrappers send their record id to the engine as a parameter, not in the statement text. The engine keeps a parsed statement in a cache keyed by its text (`arcadedb.sqlStatementCache`, 300 entries by default) and leaves a record id written in a `FROM` target as part of that text, so each record would be a cache entry of its own. `get_out_edges()`, `get_in_edges()`, `get_both_edges()`, `new_edge()`, `Edge.get_in()`, `Edge.get_out()`, `save()` and `delete()` send the id as a parameter, so every record shares one text. `db.lookup_by_rid()` keeps the id in the statement text: measured, the parameter path was 2 % slower for a lookup, so it stays as it is. diff --git a/src/arcadedb/_native/database.py b/src/arcadedb/_native/database.py index 57216b7..f8f6564 100644 --- a/src/arcadedb/_native/database.py +++ b/src/arcadedb/_native/database.py @@ -299,6 +299,10 @@ def iter_dicts(self, convert_types: bool = True) -> Iterator[Dict[str, Any]]: yield row.to_dict() def to_json_list(self, batch_size: int = 10_000) -> List[Dict[str, Any]]: + """JSON-native rows as on the JVM build (:meth:`arcadedb.results.ResultSet.to_json_list`). The walk that converts DATE, DATETIME, + DECIMAL, sets and record ids runs only when a reply of this result carried such a value; otherwise the rows are returned as + parsed. + """ if int(batch_size) < 1: raise ValueError("batch_size must be at least 1") rows = self._drain_dicts() diff --git a/src/arcadedb/results.py b/src/arcadedb/results.py index 4c00cd9..b30dfab 100644 --- a/src/arcadedb/results.py +++ b/src/arcadedb/results.py @@ -568,6 +568,14 @@ def to_json_list(self, batch_size: int = 10_000) -> List[Dict[str, Any]]: floats. Use ``to_list()`` when you want ``date``, ``datetime`` and ``Decimal`` values: it costs about the same. + A record id (a ``LINK`` value, ``@rid``) arrives as its text + (``"#10:5"``), at the top level and inside a list or a map, and a map key + that is not a string arrives as text (``"1"``, ``"true"``, ``"null"``), + on both builds. On the native build the conversion over the rows runs + only for a result in which a reply carried a value that JSON cannot + carry (DECIMAL, DATE, DATETIME, a set, NaN, a record id, ...); any other + result is returned as parsed. + NaN, Infinity and -Infinity arrive as Python floats (``float('nan')``, ``float('inf')``), at the top level and inside a list or a map, as on the native build. JSON has no literal for them, so the bridge writes