Native insert_many: ragged int and float lists go through the columnar entry (kinds I Q E D)

native/src/main/java/org/humemai/nativelib/ArcadeNative.java

@@ -560,7 +560,8 @@ public static CCharPointer insertCols(final IsolateThread thread, final long h,
   /**
    * Columnar insert by pointer: no copy of the columns into one buffer. spec points at 6 longs per column (data address and bytes, aux
    * address and bytes, null-bitmap address and bytes, see BulkOps.insertColumns) and nspec is their count. Kinds q d b f as
-   * arcadedb_n_insert_cols, plus i (int32), e (float32) and s (Arrow UTF-8 string: int32 offsets and the bytes they index), and null bitmaps.
+   * arcadedb_n_insert_cols, plus i (int32), e (float32) and s (Arrow UTF-8 string: int32 offsets and the bytes they index), the ragged
+   * list kinds I Q E D (int32 element offsets and the elements they index, see BulkOps.insertColumns), and null bitmaps.
    * The caller keeps every buffer alive and unchanged until the call returns. Returns {"ok":true,"n":rows}.
    */
   @CEntryPoint(name = "arcadedb_n_insert_colp")
@@ -578,6 +579,15 @@ public static CCharPointer insertColp(final IsolateThread thread, final long h,
     }
   }
 
+  /**
+   * Probe: a library that has this entry decodes the ragged list kinds I Q E D of arcadedb_n_insert_colp (the Python side asks for
+   * the symbol, it does not call it). An older library lacks the symbol and the Python side keeps such columns on the JSON path.
+   */
+  @CEntryPoint(name = "arcadedb_n_colp_lists")
+  public static CCharPointer colpLists(final IsolateThread thread) {
+    return out("{\"ok\":true,\"kinds\":\"IQED\"}");
+  }
+
   // ---- phase 8: fewer crossings and no reply text for a command whose rows are read late or never; batches of statements and rows per crossing
 
   private static final ThreadLocal<String> LAST_ERROR = new ThreadLocal<>();

native/src/main/java/org/humemai/nativelib/BulkOps.java

@@ -4,7 +4,9 @@
  * Phase 7: columnar bulk insert through one crossing. The documents are built here from whole columns that Python lays out
  * in native-endian buffers (int64, float64, bool bytes, int32, float32, float32 vector matrix, UTF-8 strings as Arrow offsets plus
  * bytes, optional Arrow null bitmaps), as DocumentBatcher.insertColumns does on the JPype path from long[], double[], boolean[] and
- * Object[] columns. Values are read straight from the caller's buffers; no JSON text, no parser, no intermediate arrays.
+ * Object[] columns. Values are read straight from the caller's buffers; no JSON text, no parser, no intermediate arrays. Ragged lists of
+ * numbers (the sparse-vector token and weight columns) cross as int32 element offsets plus one buffer of their elements, see
+ * {@link #insertColumns(Database, String, Map, long[])}.
  */
 package org.humemai.nativelib;
 
@@ -18,6 +20,7 @@
 
 import java.nio.ByteBuffer;
 import java.nio.charset.StandardCharsets;
+import java.util.ArrayList;
 import java.util.List;
 import java.util.Map;
 
@@ -67,7 +70,12 @@ static long insertColumns(final Database db, final String type, final String hea
   /**
    * header: {"n":rows,"commit":every,"cols":[[name,kind,dim],..]}. Kinds: q (int64), d (float64), b (bool byte), i (int32), e (float32),
    * f (float32 matrix, dim floats per row), s (UTF-8 string: int32 offsets[n+1] in the data slot, the bytes they index in the aux slot, as
-   * an Arrow string array; offsets[0] need not be 0). spec: per column six longs, [data address, data bytes, aux address, aux bytes, null
+   * an Arrow string array; offsets[0] need not be 0), and the ragged list kinds I Q E D: the data slot holds int32 offsets[n+1] counted in
+   * ELEMENTS (offsets[0] need not be 0, a null or empty row has offsets[i] == offsets[i+1]) and the aux slot the elements they index,
+   * native-endian: I = int32 elements, stored as int[] (a property declared ARRAY_OF_INTEGERS); Q = int64 elements, stored as a
+   * List of Long (what the JSON text of a list of integers parses to); E = float32 elements, stored as float[] (ARRAY_OF_FLOATS); D =
+   * float64 elements, stored as a List of Double. The Python side chooses the kind from the declared property type so that the engine
+   * stores what the JSON path stores. spec: per column six longs, [data address, data bytes, aux address, aux bytes, null
    * bitmap address, null bitmap bytes]; the bitmap is Arrow's (bit i of byte i/8, least significant first; 1 = present), an address of 0
    * means no nulls. The memory belongs to the caller for the duration of the call; nothing is kept. Every length is checked here before
    * the first document is built, so a wrong table is an exception and nothing is written. Same transaction contract as
@@ -90,6 +98,7 @@ private static long insertColumns(final Database db, final String type, final Ma
     final long[] data = new long[k]; // addresses: a Word value cannot live in an array or be compared with null
     final long[] nulls = new long[k]; // 0 = no nulls
     final ByteBuffer[] blobs = new ByteBuffer[k];
+    final long[] elems = new long[k]; // aux address of a ragged list column
     int longest = 0;
     for (int c = 0; c < k; c++) {
       final List<Object> col = (List<Object>) cols.get(c);
@@ -108,7 +117,7 @@ private static long insertColumns(final Database db, final String type, final Ma
             throw new IllegalArgumentException("column '" + names[c] + "': a vector column needs a dimension above 0");
           yield 4L * n * dims[c];
         }
-        case 's' -> 4L * (n + 1L);
+        case 's', 'I', 'Q', 'E', 'D' -> 4L * (n + 1L);
         default -> throw new IllegalArgumentException("unknown column kind " + kinds[c]);
       };
       if (dataBytes < need || (need > 0 && dataAddr == 0))
@@ -119,6 +128,22 @@ private static long insertColumns(final Database db, final String type, final Ma
           throw new IllegalArgumentException("column '" + names[c] + "' has a null bitmap of " + nullBytes + " bytes, " + (n + 7L) / 8 + " needed");
         nulls[c] = nullAddr;
       }
+      if (isList(kinds[c])) {
+        final long width = kinds[c] == 'Q' || kinds[c] == 'D' ? 8L : 4L;
+        final Pointer offsets = WordFactory.pointer(data[c]);
+        int prev = offsets.readInt(WordFactory.signed(0));
+        if (prev < 0)
+          throw new IllegalArgumentException("column '" + names[c] + "': negative list offset");
+        for (int i = 1; i <= n; i++) {
+          final int off = offsets.readInt(WordFactory.signed(4L * i));
+          if (off < prev || width * off > auxBytes)
+            throw new IllegalArgumentException("column '" + names[c] + "': list offsets are not increasing within the " + auxBytes + " bytes of elements (row " + (i - 1) + ")");
+          prev = off;
+        }
+        if (auxBytes > 0 && auxAddr == 0)
+          throw new IllegalArgumentException("column '" + names[c] + "': the list elements are missing");
+        elems[c] = auxAddr;
+      }
       if (kinds[c] == 's') {
         if (auxBytes < 0 || auxBytes > Integer.MAX_VALUE || (auxBytes > 0 && auxAddr == 0))
           throw new IllegalArgumentException("column '" + names[c] + "': the string bytes are missing or beyond 2 GiB");
@@ -166,6 +191,7 @@ private static long insertColumns(final Database db, final String type, final Ma
                 doc.set(names[c], new String(scratch, 0, len, StandardCharsets.UTF_8));
               }
             }
+            case 'I', 'Q', 'E', 'D' -> doc.set(names[c], listValue(kinds[c], base, WordFactory.pointer(elems[c]), i));
             default -> {
               final int dim = dims[c];
               final float[] v = new float[dim];
@@ -197,6 +223,42 @@ private static long insertColumns(final Database db, final String type, final Ma
     return n;
   }
 
+  private static boolean isList(final char kind) {
+    return kind == 'I' || kind == 'Q' || kind == 'E' || kind == 'D';
+  }
+
+  /** Row i of a ragged list column: offsets (int32, in elements) at {@code offsets}, the elements at {@code elems}. */
+  private static Object listValue(final char kind, final Pointer offsets, final Pointer elems, final int i) {
+    final int from = offsets.readInt(WordFactory.signed(4L * i));
+    final int len = offsets.readInt(WordFactory.signed(4L * (i + 1L))) - from;
+    switch (kind) {
+    case 'I': {
+      final int[] v = new int[len];
+      for (int j = 0; j < len; j++)
+        v[j] = elems.readInt(WordFactory.signed(4L * (from + (long) j)));
+      return v;
+    }
+    case 'E': {
+      final float[] v = new float[len];
+      for (int j = 0; j < len; j++)
+        v[j] = elems.readFloat(WordFactory.signed(4L * (from + (long) j)));
+      return v;
+    }
+    case 'Q': {
+      final List<Object> v = new ArrayList<>(len);
+      for (int j = 0; j < len; j++)
+        v.add(elems.readLong(WordFactory.signed(8L * (from + (long) j))));
+      return v;
+    }
+    default: {
+      final List<Object> v = new ArrayList<>(len);
+      for (int j = 0; j < len; j++)
+        v.add(elems.readDouble(WordFactory.signed(8L * (from + (long) j))));
+      return v;
+    }
+    }
+  }
+
   /**
    * Documents (or vertices of a vertex type) from rows [{property: value},..] in the UTF-8 JSON text b[0, n): one record per row through
    * newDocument/newVertex, set per property and save, as DocumentBatcher.insertManyJson does on the JPype path; no transaction is opened

src/arcadedb/_native/bulk.py

@@ -2,7 +2,8 @@
 ints, floats and bools, lists of one value type (None is a null), strings (UTF-8 bytes plus offsets, as Arrow lays them out), float32
 vector columns and Arrow arrays are laid out by :mod:`.columnar`; anything else (nested values, mixed-type columns, date-times,
 decimals, integers past 64 bits) returns None and the caller uses the per-row JSON path. The documents are built image-side as
-``DocumentBatcher.insertColumns`` builds them on JPype. ``ARCADEDB_NATIVE_BULK=0`` turns the columnar path off.
+``DocumentBatcher.insertColumns`` builds them on JPype. Lists of ints and floats (ragged or not, None allowed) cross as int32 offsets plus a buffer of their elements when the library has
+the decoder (``lib.COLP_LISTS``). ``ARCADEDB_NATIVE_BULK=0`` turns the columnar path off.
 """
 
 import os
@@ -21,6 +22,10 @@ def _usable():
     )
 
 
+def _declared(db, type_name, known=None):
+    return columnar.Declared(db, type_name, known, lists=lib.COLP_LISTS)
+
+
 def _send(db, type_name, packed, rows, commit_every, prefix):
     from .database import _call
 
@@ -47,7 +52,7 @@ def insert_columns_fast(
     """
     if not _usable():
         return None
-    packed = columnar.pack(columns, columnar.Declared(db, type_name, declared))
+    packed = columnar.pack(columns, _declared(db, type_name, declared))
     if packed is None:
         return None
     rows, cols = packed
@@ -63,7 +68,7 @@ def insert_rows_columnar(db, type_name, chunk):
     columns = columnar.rows_to_columns(chunk)
     if columns is None:
         return False
-    packed = columnar.pack(columns, columnar.Declared(db, type_name))
+    packed = columnar.pack(columns, _declared(db, type_name))
     if packed is None:
         return False
     rows, cols = packed

src/arcadedb/_native/columnar.py

@@ -9,12 +9,20 @@
 matrix (``dim`` per row), ``s`` UTF-8 string (``int32 offsets[n+1]`` and the bytes they index: Arrow's string layout). A column may
 carry a null bitmap (Arrow's: bit ``i`` of byte ``i // 8``, least significant first, 1 = present).
 
+Ragged lists of numbers (``I`` int32, ``Q`` int64, ``E`` float32, ``D`` float64 elements) lay out like a string column: ``int32
+offsets[n+1]`` counted in elements as the data buffer and the elements they index as the aux buffer; a null or empty row has equal
+neighbouring offsets. The image stores ``I`` as ``int[]`` (a property declared ``ARRAY_OF_INTEGERS``), ``E`` as ``float[]`` (declared
+``ARRAY_OF_FLOATS``), ``Q`` and ``D`` as a list of Long or Double (undeclared or declared ``LIST``): what the JSON path stores for the same
+rows. A library without the decoder (``Declared(lists=False)``) gets none of them.
+
 ``i`` and ``e`` and the float matrix from a list of lists are chosen only for a property declared ``INTEGER``, ``FLOAT`` and
 ``ARRAY_OF_FLOATS``: the engine would convert to those types anyway, so the stored value is the one the JSON path stores. For an
 undeclared property an ``int32`` column crosses as ``q`` (a Long) and a ``float32`` column as ``d`` (a Double), as the JSON path stores
 them.
 """
 
+import itertools
+
 from .schema_objects import (  # the schema-entry reader shared with NativeType (#195)
     _name_type,
 )
@@ -26,7 +34,14 @@
 
 _INT64_MAX = 2**63 - 1
 _INT32_MAX = 2**31 - 1
+_INT32_MIN = -(2**31)
 _NoneType = type(None)
+_OFFSET_KINDS = frozenset(
+    "sIQED"
+)  # the kinds whose data buffer is int32 offsets[n + 1]
+_EMPTY = (
+    []
+)  # stands in for a null row of a list column (the null bitmap says it is absent)
 
 
 class Packed:
@@ -39,8 +54,8 @@ def __init__(self, name, kind, dim, array, aux=None, nulls=None, keep=()):
         self.kind = kind
         self.dim = dim
         self.n = int(array.shape[0]) - (
-            1 if kind == "s" else 0
-        )  # a string column's array is its n + 1 offsets
+            1 if kind in _OFFSET_KINDS else 0
+        )  # a string or list column's array is its n + 1 offsets
         self.data = (array.ctypes.data, array.nbytes)
         self.aux = aux or (0, 0)
         self.nulls = (nulls.ctypes.data, nulls.nbytes) if nulls is not None else (0, 0)
@@ -50,11 +65,13 @@ def __init__(self, name, kind, dim, array, aux=None, nulls=None, keep=()):
 class Declared:
     """The declared property types of a type, read once and only when a column needs them (``INTEGER``, ``FLOAT``, ``ARRAY_OF_FLOATS``).
 
-    ``known`` (a dict) is used as it is; otherwise ``db`` is asked for ``schema:types`` the first time :meth:`get` is called.
+    ``known`` (a dict) is used as it is; otherwise ``db`` is asked for ``schema:types`` the first time :meth:`get` is called. ``lists``:
+    whether the library decodes the ragged list kinds (``lib.COLP_LISTS``); False keeps list columns on the JSON path.
     """
 
-    def __init__(self, db=None, type_name=None, known=None):
+    def __init__(self, db=None, type_name=None, known=None, lists=True):
         self._db, self._type, self._known = db, type_name, known
+        self.lists = lists
 
     def get(self, name):
         if self._known is None:
@@ -207,10 +224,11 @@ def _list_column(name, values, declared):
         try:
             arr = _np.array(values, dtype=_np.float32)
         except (ValueError, TypeError):
-            return None  # ragged, or not numbers
-        if arr.ndim != 2 or arr.shape[1] == 0:
-            return None
-        return Packed(name, "f", int(arr.shape[1]), arr)
+            arr = None  # ragged, or not numbers
+        if arr is not None and arr.ndim == 2 and arr.shape[1] > 0:
+            return Packed(name, "f", int(arr.shape[1]), arr)
+    if only is list:
+        return _ragged_column(name, values, declared, present, nulls)
     if (
         only is _np.ndarray
         and present is None
@@ -229,6 +247,80 @@ def _list_column(name, values, declared):
     return None
 
 
+def _ragged_column(name, values, declared, present, nulls):
+    """A list of lists of ints or of floats (ragged, empty rows and None rows allowed) as an offsets-plus-elements column, or None.
+
+    The kind follows the declared property type so that the engine stores what the JSON text stores: ``ARRAY_OF_INTEGERS`` takes ints
+    in int32 range as ``I``, ``ARRAY_OF_FLOATS`` takes Python floats as ``E`` (float32, as the engine converts a list of doubles), an
+    undeclared or ``LIST`` property takes ``Q`` (ints) or ``D`` (floats), which the image stores as a list of Long or Double. Anything
+    else (another declared type, bools, strings, nested lists, mixed ints and floats, an int past 64 bits, NaN or an infinity, more than
+    2**31 - 1 elements) returns None and the chunk goes the JSON way.
+    """
+    if not declared.lists:
+        return None
+    rows = (
+        values if present is None else [v if v is not None else _EMPTY for v in values]
+    )
+    n = len(rows)
+    decl = declared.get(name)
+    if decl not in (None, "LIST", "ARRAY_OF_INTEGERS", "ARRAY_OF_FLOATS"):
+        return None
+    counts = _np.fromiter(map(len, rows), dtype=_np.int64, count=n)
+    total = int(counts.sum())
+    if total > _INT32_MAX:
+        return None
+    flat = list(itertools.chain.from_iterable(rows))
+    if (
+        total == 0
+    ):  # only empty rows: the engine stores an empty list or array whatever the element type
+        kind, elements = {"ARRAY_OF_INTEGERS": "I", "ARRAY_OF_FLOATS": "E"}.get(
+            decl, "Q"
+        ), None
+    else:
+        element_types = set(map(type, flat))
+        if len(element_types) != 1:
+            return None
+        (element,) = element_types
+        if element is int and decl in (None, "LIST", "ARRAY_OF_INTEGERS"):
+            try:
+                elements = _np.array(flat, dtype=_np.int64)
+            except OverflowError:
+                return None  # an integer beyond 64 bits
+            if decl == "ARRAY_OF_INTEGERS":
+                if int(elements.min()) < _INT32_MIN or int(elements.max()) > _INT32_MAX:
+                    return None  # the engine refuses such a value for the property: leave the error to the JSON path
+                kind, elements = "I", elements.astype(_np.int32)
+            else:
+                kind = "Q"
+        elif element is float and decl in (None, "LIST", "ARRAY_OF_FLOATS"):
+            elements = _np.array(flat, dtype=_np.float64)
+            if not _np.isfinite(elements).all():
+                return None
+            if decl == "ARRAY_OF_FLOATS":
+                with _np.errstate(
+                    all="ignore"
+                ):  # a double beyond float32 becomes Infinity, as in the engine
+                    kind, elements = "E", elements.astype(_np.float32)
+            else:
+                kind = "D"
+        else:
+            return None
+    offsets = _np.zeros(n + 1, dtype=_np.int32)
+    _np.cumsum(counts, out=offsets[1:], dtype=_np.int32)
+    if elements is None or not elements.size:
+        return Packed(name, kind, 0, offsets, nulls=nulls)
+    elements = _np.ascontiguousarray(elements)
+    return Packed(
+        name,
+        kind,
+        0,
+        offsets,
+        aux=(elements.ctypes.data, elements.nbytes),
+        nulls=nulls,
+        keep=(elements,),
+    )
+
+
 def _arrow_column(name, arr, declared):
     """A ``pyarrow`` array as a column (strings in place), or None."""
     import pyarrow as pa

src/arcadedb/_native/database.py

@@ -752,7 +752,7 @@ def insert_many(
         parallel: bool = False,
     ) -> int:
         """Bulk insert. Rows go in chunks of at most 10,000 (a chunk never crosses a commit), one crossing per chunk. A chunk of rows with the same keys whose columns each hold one
-        value type (ints, floats, bools, strings, equal-length float lists for an ``ARRAY_OF_FLOATS`` property; None is a null) goes as
+        value type (ints, floats, bools, strings, lists of ints or of floats, ragged or not; None is a null) goes as
         columns by address (``arcadedb_n_insert_colp``, no JSON text); any other chunk goes as one JSON text (``arcadedb_n_insert_rows``:
         a newDocument/set/save loop inside the image, as ``DocumentBatcher.insertManyJson`` on the JPype path); a chunk with a value the
         JSON text cannot carry unchanged goes through ``INSERT INTO t CONTENT :row`` for each row, in one crossing. Inside the caller's