diff --git a/CHANGELOG.md b/CHANGELOG.md index 841ccb8c2..d2c62968f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,6 +8,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] ### Added +- `DecimalArray.getUnscaledLong(long)` reads a decimal's unscaled value without building a `BigDecimal`. ([#563](https://github.com/dfa1/vortex-java/pull/563)) - `Compute.groupedAggregate(Chunk, ColumnName, RowFilter, ColumnName)` folds an aggregate per value of a dictionary-encoded string, boolean or integer column. ([#555](https://github.com/dfa1/vortex-java/pull/555)) - `Chunk.slice(offset, length)` gives a zero-copy view of a row range. ([#550](https://github.com/dfa1/vortex-java/pull/550)) - The Calcite adapter tells the planner each table's row count, so join order and costing use the real table size. ([#550](https://github.com/dfa1/vortex-java/pull/550)) @@ -17,6 +18,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - Custom extension types: register an `ExtensionDecoder` on `ReadRegistry` and an `ExtensionEncoder` on `WriteRegistry`, with a `validate` hook on both sides, and `Chunk.as` decodes any registered type. ([#530](https://github.com/dfa1/vortex-java/issues/530)) ### Changed +- SQL `SUM` and `AVG` of a decimal column, grouped or not, are 4 to 6 times faster: they are totalled exactly as unscaled longs in the scan instead of by Calcite with a `BigDecimal` per row. ([#563](https://github.com/dfa1/vortex-java/pull/563)) - SQL `COUNT(DISTINCT col)` is up to 4 times faster: distinct values are counted in the scan, integers by their bits in a primitive map, instead of Calcite hashing a boxed value per row. ([#562](https://github.com/dfa1/vortex-java/pull/562)) - A SQL `WHERE` comparing a `SMALLINT` or `TINYINT` column with a literal is pushed into the scan and its aggregates, instead of being left to Calcite because of the cast Calcite adds. ([#560](https://github.com/dfa1/vortex-java/pull/560)) - SQL `MIN`/`MAX` of a date, time or timestamp column is answered from zone maps, and scans read those columns without building a `java.time` object per row. ([#559](https://github.com/dfa1/vortex-java/pull/559)) diff --git a/calcite/src/main/java/io/github/dfa1/vortex/calcite/VortexAggregatePushDownRule.java b/calcite/src/main/java/io/github/dfa1/vortex/calcite/VortexAggregatePushDownRule.java index 88fb08262..6e2839cb5 100644 --- a/calcite/src/main/java/io/github/dfa1/vortex/calcite/VortexAggregatePushDownRule.java +++ b/calcite/src/main/java/io/github/dfa1/vortex/calcite/VortexAggregatePushDownRule.java @@ -217,7 +217,7 @@ private static void rewriteGrouped(RelOptRuleCall call, Aggregate aggregate, Vor callColumns.add(null); continue; } - boolean supported = kind == SqlKind.COUNT || kind == SqlKind.SUM || kind == SqlKind.MIN + boolean supported = kind == SqlKind.COUNT || kind == SqlKind.SUM || kind == SqlKind.SUM0 || kind == SqlKind.MIN || kind == SqlKind.MAX; ColumnName column = aggCall.getArgList().size() == 1 ? resolveColumn(aggCall.getArgList().getFirst(), scanColumns, project) : null; @@ -268,8 +268,8 @@ private static RexLiteral groupedLiteral(AggregateCall aggCall, ColumnName colum long nulls = columnFold.nullCount(); return switch (aggCall.getAggregation().getKind()) { case COUNT -> exact(rexBuilder, columnFold.rows() - nulls, outType); - case SUM -> sumLiteral(rexBuilder, outType, columnFold.sum(), nulls, columnFold.rows(), - isNullable(scanRowType, column.value())); + case SUM, SUM0 -> sum0(aggCall, rexBuilder, outType, sumLiteral(rexBuilder, outType, columnFold.sum(), nulls, columnFold.rows(), + isNullable(scanRowType, column.value()))); case MIN, MAX -> { Object value = aggCall.getAggregation().getKind() == SqlKind.MIN ? columnFold.min() : columnFold.max(); yield value == null ? rexBuilder.makeNullLiteral(outType) @@ -346,7 +346,7 @@ private static RexLiteral evaluate(AggregateCall agg, RelDataType outType, Vorte } yield extremeLiteral(rexBuilder, value, outType, table.columnType(col)); } - case SUM -> { + case SUM, SUM0 -> { if (agg.getArgList().size() != 1) { yield null; } @@ -354,12 +354,15 @@ private static RexLiteral evaluate(AggregateCall agg, RelDataType outType, Vorte if (col == null) { yield null; } - ColumnFold fold = columnFold(table, col, filter, planner); + // zone statistics carry no decimal sum, so a decimal column is totalled by its own pass + ColumnFold fold = table.columnType(col) instanceof io.github.dfa1.vortex.core.model.DType.Decimal + ? decimalFold(table, col, filter, planner) + : columnFold(table, col, filter, planner); if (fold == null) { yield null; } - yield sumLiteral(rexBuilder, outType, fold.sum(), fold.nullCount(), fold.rows(), - isNullable(scanRowType, col.value())); + yield sum0(agg, rexBuilder, outType, sumLiteral(rexBuilder, outType, fold.sum(), fold.nullCount(), + fold.rows(), isNullable(scanRowType, col.value()))); } default -> null; }; @@ -380,9 +383,23 @@ private static ColumnFold columnFold(VortexTable table, ColumnName column, RowFi return new ColumnFold(zoneSum.sum(), stats.stats().min(), stats.stats().max(), stats.stats().nullCount(), stats.totalRows()); } - return filteredFold(planner, table, filter, column) + ColumnFold zones = filteredFold(planner, table, filter, column) .map(f -> new ColumnFold(f.sum(), f.min(), f.max(), f.nullCount(), f.rows())) .orElse(null); + // when the zones cannot answer (the filter cuts every zone it keeps), a decimal column still + // has its own exact pass + if (zones == null && table.columnType(column) instanceof io.github.dfa1.vortex.core.model.DType.Decimal) { + return decimalFold(table, column, filter, planner); + } + return zones; + } + + /// [VortexTable#decimalFold(ColumnName, RowFilter)], computed once per planner run. + private static ColumnFold decimalFold(VortexTable table, ColumnName column, RowFilter filter, + RelOptPlanner planner) { + List foldKey = List.of("decimal", table, filter == null ? List.of() : filter, column); + Optional fold = memo(planner, foldKey, () -> table.decimalFold(column, filter)); + return fold.map(f -> new ColumnFold(f.sum(), f.min(), f.max(), f.nullCount(), f.rows())).orElse(null); } /// [VortexTable#filteredFold(RowFilter, ColumnName)], computed once per planner run. @@ -392,6 +409,15 @@ private static Optional filteredFold(RelOptPlanner pla return memo(planner, foldKey, () -> table.filteredFold(filter, column)); } + /// `$SUM0`, which Calcite reduces `AVG` to, is `SUM` that returns `0` instead of `NULL` over no + /// non-null rows: turns the `SUM` literal's `NULL` into that zero, and leaves `SUM` alone. + private static RexLiteral sum0(AggregateCall agg, RexBuilder rexBuilder, RelDataType outType, RexLiteral sum) { + if (sum == null || agg.getAggregation().getKind() != SqlKind.SUM0 || !sum.isNull()) { + return sum; + } + return numericLiteral(rexBuilder, 0L, outType); + } + /// Builds the literal for a `SUM`, resolving the all-null-vs-genuine-zero ambiguity from the /// null count. SQL `SUM` over zero non-null rows is `NULL`, but the writer records an all-null /// zone's sum as a sum-neutral `0` (validity placeholders are zero), so a `0` fold alone cannot @@ -492,7 +518,10 @@ private static RexLiteral numericLiteral(RexBuilder rexBuilder, Object value, Re return switch (type.getSqlTypeName()) { case TINYINT, SMALLINT, INTEGER, BIGINT -> rexBuilder.makeExactLiteral(BigDecimal.valueOf(number.longValue()), type); - case FLOAT, REAL, DOUBLE, DECIMAL -> + // a DECIMAL literal is exact: a decimal sum or extreme must not round through a double + case DECIMAL -> rexBuilder.makeExactLiteral(number instanceof BigDecimal d ? d + : new BigDecimal(number.toString()), type); + case FLOAT, REAL, DOUBLE -> rexBuilder.makeApproxLiteral(BigDecimal.valueOf(number.doubleValue()), type); default -> null; }; diff --git a/calcite/src/main/java/io/github/dfa1/vortex/calcite/VortexTable.java b/calcite/src/main/java/io/github/dfa1/vortex/calcite/VortexTable.java index 9ececc0d9..29c673695 100644 --- a/calcite/src/main/java/io/github/dfa1/vortex/calcite/VortexTable.java +++ b/calcite/src/main/java/io/github/dfa1/vortex/calcite/VortexTable.java @@ -16,6 +16,7 @@ import io.github.dfa1.vortex.reader.array.Array; import io.github.dfa1.vortex.reader.array.BoolArray; import io.github.dfa1.vortex.reader.array.ByteArray; +import io.github.dfa1.vortex.reader.array.DecimalArray; import io.github.dfa1.vortex.reader.array.DoubleArray; import io.github.dfa1.vortex.reader.array.FloatArray; import io.github.dfa1.vortex.reader.array.IntArray; @@ -545,6 +546,67 @@ private static boolean countChunk(Array data, BoolArray validity, java.util.func return false; } + /// `SUM`/`MIN`/`MAX`/null count of a decimal column over the rows `filter` selects, totalled as + /// unscaled longs ([DecimalArray#getUnscaledLong(long)]) instead of a `BigDecimal` per row. Zone + /// statistics carry no decimal sum, so without this every decimal `SUM` was left to Calcite. + /// + /// @param column the decimal column + /// @param filter the rows to fold, or `null` for every row + /// @return the fold, or empty for a precision past 18 or a sum past a long, which Calcite totals + public Optional decimalFold(ColumnName column, RowFilter filter) { + if (!(columnType(column) instanceof DType.Decimal decimal) || decimal.precision() > 18) { + return Optional.empty(); + } + java.util.LinkedHashSet columns = new java.util.LinkedHashSet<>(); + columns.add(column); + ScanOptions options; + if (filter == null) { + options = ScanOptions.columns(column); + } else { + RexFilterTranslator.collectColumns(filter, columns); + options = ScanOptions.columns(columns.toArray(ColumnName[]::new)).withFilter(filter); + } + long sum = 0; + long min = Long.MAX_VALUE; + long max = Long.MIN_VALUE; + long rows = 0; + long nonNull = 0; + try (VortexReader reader = VortexReader.open(file); + ScanIterator scan = reader.scan(options)) { + while (scan.hasNext()) { + try (Chunk chunk = scan.next()) { + java.util.function.LongPredicate selected = filter == null ? null : Compute.matcher(chunk, filter); + Array array = chunk.column(column); + BoolArray validity = array instanceof MaskedArray masked ? masked.validity() : null; + DecimalArray data = (DecimalArray) (array instanceof MaskedArray masked ? masked.inner() : array); + long n = chunk.rowCount(); + for (long i = 0; i < n; i++) { + if (selected != null && !selected.test(i)) { + continue; + } + rows++; + if (validity != null && !validity.getBoolean(i)) { + continue; + } + long v = data.getUnscaledLong(i); + sum = Math.addExact(sum, v); + min = Math.min(min, v); + max = Math.max(max, v); + nonNull++; + } + } + } + } catch (ArithmeticException overflow) { + return Optional.empty(); + } catch (IOException e) { + throw new UncheckedIOException("cannot sum " + column + " in " + file, e); + } + int scale = decimal.scale(); + return Optional.of(new FilteredFold(java.math.BigDecimal.valueOf(sum, scale), + nonNull == 0 ? null : java.math.BigDecimal.valueOf(min, scale), + nonNull == 0 ? null : java.math.BigDecimal.valueOf(max, scale), rows - nonNull, rows)); + } + /// One `ORDER BY` key of [#topN(int[], RowFilter, List, int)]. /// /// @param field the key's position in the projected row @@ -719,6 +781,7 @@ private static final class Fold { private long longSum; private double doubleSum; private boolean floatingSum; + private java.math.BigDecimal decimalSum; private boolean sumUsable = true; private Object min; private Object max; @@ -739,6 +802,8 @@ private void addSum(Object contribution) { } else if (contribution instanceof Double d) { floatingSum = true; doubleSum += d; + } else if (contribution instanceof java.math.BigDecimal d) { + decimalSum = decimalSum == null ? d : decimalSum.add(d); } else { longSum += ((Number) contribution).longValue(); } @@ -777,6 +842,8 @@ private FilteredFold result() { foldedSum = null; // a contributing zone carried no usable sum } else if (floatingSum) { foldedSum = doubleSum; + } else if (decimalSum != null) { + foldedSum = decimalSum; } else { foldedSum = longSum; } diff --git a/calcite/src/test/java/io/github/dfa1/vortex/calcite/DecimalSumTest.java b/calcite/src/test/java/io/github/dfa1/vortex/calcite/DecimalSumTest.java new file mode 100644 index 000000000..8e221ea06 --- /dev/null +++ b/calcite/src/test/java/io/github/dfa1/vortex/calcite/DecimalSumTest.java @@ -0,0 +1,100 @@ +package io.github.dfa1.vortex.calcite; + +import io.github.dfa1.vortex.core.model.ColumnName; +import io.github.dfa1.vortex.core.model.DType; +import io.github.dfa1.vortex.writer.VortexWriter; +import io.github.dfa1.vortex.writer.WriteOptions; + +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.io.TempDir; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.CsvSource; + +import java.math.BigDecimal; +import java.nio.channels.FileChannel; +import java.nio.file.Path; +import java.nio.file.StandardOpenOption; +import java.sql.Connection; +import java.sql.ResultSet; +import java.sql.Statement; +import java.util.ArrayList; +import java.util.List; +import java.util.Map; + +import static org.assertj.core.api.Assertions.assertThat; + +/// A decimal `SUM` (and the `AVG` reduced to it) is totalled as unscaled longs in the scan instead of +/// by Calcite with a `BigDecimal` per row. Each query must equal the same aggregate over +/// `CASE WHEN id > -1 THEN amount END` (`id` is never null), which the rules leave to Calcite; the test +/// asserts that oracle really scans. +class DecimalSumTest { + + @TempDir + static Path tmp; + private static Path file; + + /// Two chunks of DECIMAL(10, 2) amounts with nulls, negatives and a group column. + @BeforeAll + static void writeFile() throws Exception { + file = tmp.resolve("decimal.vortex"); + DType.Struct schema = DType.structBuilder() + .field(ColumnName.of("id"), DType.I64) + .field(ColumnName.of("s"), DType.UTF8) + .field(ColumnName.of("amount"), new DType.Decimal((byte) 10, (byte) 2, true)) + .build(); + try (var ch = FileChannel.open(file, StandardOpenOption.CREATE, StandardOpenOption.WRITE); + var writer = VortexWriter.create(ch, schema, WriteOptions.defaults())) { + writer.writeChunk(Map.of(ColumnName.of("id"), new long[]{1, 2, 3, 4}, + ColumnName.of("s"), new String[]{"a", "b", "a", "b"}, + ColumnName.of("amount"), new BigDecimal[]{new BigDecimal("12.34"), null, + new BigDecimal("-0.05"), new BigDecimal("9999999.99")})); + writer.writeChunk(Map.of(ColumnName.of("id"), new long[]{5, 6, 7}, + ColumnName.of("s"), new String[]{"a", "c", "b"}, + ColumnName.of("amount"), new BigDecimal[]{new BigDecimal("0.01"), null, + new BigDecimal("-12345.67")})); + } + } + + @ParameterizedTest(name = "{0}") + @CsvSource(delimiter = '|', quoteCharacter = '"', value = { + "select sum(amount) from vtx.t " + + "| select sum(case when id > -1 then amount end) from vtx.t", + "select sum(amount), avg(amount), count(amount) from vtx.t where id < 5 " + + "| select sum(case when id > -1 then amount end), avg(case when id > -1 then amount end), " + + "count(case when id > -1 then amount end) from vtx.t where id + 0 < 5", + "select s, sum(amount), min(amount), max(amount) from vtx.t group by s order by s " + + "| select s, sum(case when id > -1 then amount end), min(case when id > -1 then amount end), " + + "max(case when id > -1 then amount end) from vtx.t group by s order by s"}) + void decimalAggregate_isPushedDownAndEqualsCalcite(String sql, String oracle) throws Exception { + try (Connection conn = VortexCalcite.connect("vtx", Map.of("t", file))) { + // When / Then + assertThat(plan(conn, sql)).contains("Values").doesNotContain("TableScan"); + assertThat(plan(conn, oracle)).contains("TableScan"); + assertThat(rows(conn, sql)).isEqualTo(rows(conn, oracle)).isNotEmpty(); + } + } + + private static String plan(Connection conn, String sql) throws Exception { + try (Statement st = conn.createStatement(); ResultSet rs = st.executeQuery("explain plan for " + sql)) { + rs.next(); + return rs.getString(1); + } + } + + /// Rows as strings, decimals compared by value (`1.50` equals `1.5`). + private static List rows(Connection conn, String sql) throws Exception { + List out = new ArrayList<>(); + try (Statement st = conn.createStatement(); ResultSet rs = st.executeQuery(sql)) { + int columns = rs.getMetaData().getColumnCount(); + while (rs.next()) { + StringBuilder row = new StringBuilder(); + for (int c = 1; c <= columns; c++) { + Object value = rs.getObject(c); + row.append(value instanceof BigDecimal d ? d.stripTrailingZeros().toPlainString() : value).append('|'); + } + out.add(row.toString()); + } + } + return out; + } +} diff --git a/docs/reference.md b/docs/reference.md index a49c44b4c..8cf361aff 100644 --- a/docs/reference.md +++ b/docs/reference.md @@ -637,8 +637,10 @@ column-vs-literal comparisons, `IN` lists, `IS [NOT] NULL` and prefix `LIKE` joi the rows of zones it cuts are decoded. `sum` needs a per-zone sum, which only the legacy `vortex.stats` zone map records (writes targeting an edition before `core2026.08.0`); with Rust's `vortex.zoned` — the default since #447, and every Rust-written file — `sum` is answered by a scan. +A `sum` of a decimal column (precision up to 18) is totalled in that scan as unscaled longs, not by +Calcite with a `BigDecimal` per row, and is exact. A `GROUP BY` on one column (a dictionary-encoded string, a boolean, or an integer other than -`u64`), with `count`/`sum`/`min`/`max` of plain numeric columns and the same kind of `WHERE`, is +`u64`), with `count`/`sum`/`min`/`max` of plain numeric or decimal columns and the same kind of `WHERE`, is folded per group into one row per group instead of building a row per record; it falls back to the scan when a chunk of a string column is not dictionary-encoded, there are more than 10,000 groups, or a group value is outside Latin-1 (Calcite's `VARCHAR` charset). An ungrouped `count(distinct col)` (with the same kind of `WHERE`) counts distinct values diff --git a/integration/src/test/java/io/github/dfa1/vortex/integration/CalciteInteropIntegrationTest.java b/integration/src/test/java/io/github/dfa1/vortex/integration/CalciteInteropIntegrationTest.java index 402f67a8a..2085418b8 100644 --- a/integration/src/test/java/io/github/dfa1/vortex/integration/CalciteInteropIntegrationTest.java +++ b/integration/src/test/java/io/github/dfa1/vortex/integration/CalciteInteropIntegrationTest.java @@ -435,6 +435,77 @@ void jniWriter_groupByInteger_isPushedDownAndEqualsTheScan(String sql, String or } } + @ParameterizedTest(name = "{0}") + @CsvSource(delimiter = '|', quoteCharacter = '"', value = { + "select sum(amount) from vtx.t " + + "| select sum(case when id > -1 then amount end) from vtx.t", + "select sum(amount), count(amount) from vtx.t where id < 1700 " + + "| select sum(case when id > -1 then amount end), count(case when id > -1 then amount end) " + + "from vtx.t where id + 0 < 1700", + "select s, sum(amount), min(amount), max(amount) from vtx.t group by s order by s " + + "| select s, sum(case when id > -1 then amount end), min(case when id > -1 then amount end), " + + "max(case when id > -1 then amount end) from vtx.t group by s order by s"}) + void jniWriter_decimalAggregates_arePushedDownAndEqualTheScan(String sql, String oracle, @TempDir Path tmp) + throws Exception { + // Given — Rust writes a nullable DECIMAL(10, 2) column; its sums are totalled as unscaled longs + // instead of a BigDecimal per row, and a CASE around the column keeps Calcite's own path + Path file = tmp.resolve("jni_decimal.vortex"); + Schema schema = new Schema(List.of( + Field.notNullable("id", new ArrowType.Int(64, true)), + Field.notNullable("s", new ArrowType.Utf8()), + Field.nullable("amount", new ArrowType.Decimal(10, 2, 128)))); + String[] symbols = {"AAPL", "MSFT", "NVDA"}; + try (VortexWriter writer = VortexWriter.builder(SESSION, file.toAbsolutePath().toUri().toString(), schema, + ALLOCATOR).build()) { + for (int batch = 0; batch < BATCHES; batch++) { + try (VectorSchemaRoot root = VectorSchemaRoot.create(schema, ALLOCATOR)) { + BigIntVector ids = (BigIntVector) root.getVector("id"); + org.apache.arrow.vector.VarCharVector s = (org.apache.arrow.vector.VarCharVector) root.getVector("s"); + DecimalVector amount = (DecimalVector) root.getVector("amount"); + ids.allocateNew(BATCH_ROWS); + s.allocateNew(BATCH_ROWS); + amount.allocateNew(BATCH_ROWS); + for (int i = 0; i < BATCH_ROWS; i++) { + long id = (long) batch * BATCH_ROWS + i; + ids.set(i, id); + s.setSafe(i, symbols[(int) (id % symbols.length)].getBytes(java.nio.charset.StandardCharsets.UTF_8)); + if (id % 13 == 0) { + amount.setNull(i); + } else { + amount.setSafe(i, BigDecimal.valueOf((id * 7919 % 2_000_003) - 1_000_000, 2)); + } + } + root.setRowCount(BATCH_ROWS); + try (ArrowArray array = ArrowArray.allocateNew(ALLOCATOR); + ArrowSchema arrowSchema = ArrowSchema.allocateNew(ALLOCATOR)) { + Data.exportVectorSchemaRoot(ALLOCATOR, root, null, array, arrowSchema); + writer.writeBatch(array.memoryAddress(), arrowSchema.memoryAddress()); + } + } + } + } + + try (Connection conn = VortexCalcite.connect("vtx", Map.of("t", file))) { + // When + String plan; + String oraclePlan; + try (Statement st = conn.createStatement(); ResultSet rs = st.executeQuery("explain plan for " + sql)) { + rs.next(); + plan = rs.getString(1); + } + try (Statement st = conn.createStatement(); ResultSet rs = st.executeQuery("explain plan for " + oracle)) { + rs.next(); + oraclePlan = rs.getString(1); + } + List result = sqlRows(conn, sql); + + // Then + assertThat(plan).containsIgnoringCase("Values").doesNotContain("TableScan"); + assertThat(oraclePlan).contains("TableScan"); + assertThat(result).isEqualTo(sqlRows(conn, oracle)).isNotEmpty(); + } + } + private static List sqlRows(Connection conn, String sql) throws Exception { List out = new java.util.ArrayList<>(); try (Statement st = conn.createStatement(); ResultSet rs = st.executeQuery(sql)) { diff --git a/reader/src/main/java/io/github/dfa1/vortex/reader/array/DecimalArray.java b/reader/src/main/java/io/github/dfa1/vortex/reader/array/DecimalArray.java index 0f78bdadc..ca714af6d 100644 --- a/reader/src/main/java/io/github/dfa1/vortex/reader/array/DecimalArray.java +++ b/reader/src/main/java/io/github/dfa1/vortex/reader/array/DecimalArray.java @@ -14,4 +14,15 @@ public non-sealed interface DecimalArray extends Array { /// @param i zero-based row index (must be in `[0, length())`) /// @return the [java.math.BigDecimal] value at position `i` BigDecimal getDecimal(long i); + + /// Returns the unscaled value at `i` as a `long`: the decimal is this times ten to the minus + /// scale. Every value of a decimal with precision 18 or less fits, so a fold over such a column + /// can total longs instead of building a [BigDecimal] per row. + /// + /// @param i zero-based row index (must be in `[0, length())`) + /// @return the unscaled value at position `i` + /// @throws ArithmeticException if the value does not fit a `long` (precision past 18) + default long getUnscaledLong(long i) { + return getDecimal(i).unscaledValue().longValueExact(); + } } diff --git a/reader/src/main/java/io/github/dfa1/vortex/reader/array/DecimalBytePartsArrays.java b/reader/src/main/java/io/github/dfa1/vortex/reader/array/DecimalBytePartsArrays.java index bd4dc8b65..e917b5fa9 100644 --- a/reader/src/main/java/io/github/dfa1/vortex/reader/array/DecimalBytePartsArrays.java +++ b/reader/src/main/java/io/github/dfa1/vortex/reader/array/DecimalBytePartsArrays.java @@ -25,16 +25,21 @@ private DecimalBytePartsArrays() { /// @return cell value as a [BigInteger] /// @throws VortexException for null cells or unsupported array types static BigInteger readMantissa(Array arr, long i) { + return BigInteger.valueOf(readMantissaLong(arr, i)); + } + + /// Reads the mantissa at `i` at the child's own width; every supported child fits a `long`. + static long readMantissaLong(Array arr, long i) { return switch (arr) { - case ByteArray a -> BigInteger.valueOf(a.getByte(i)); - case ShortArray a -> BigInteger.valueOf(a.getShort(i)); - case IntArray a -> BigInteger.valueOf(a.getInt(i)); - case LongArray a -> BigInteger.valueOf(a.getLong(i)); + case ByteArray a -> a.getByte(i); + case ShortArray a -> a.getShort(i); + case IntArray a -> a.getInt(i); + case LongArray a -> a.getLong(i); case MaskedArray a -> { if (!a.isValid(i)) { throw new VortexException("DecimalByteParts: null cell at index " + i); } - yield readMantissa(a.inner(), i); + yield readMantissaLong(a.inner(), i); } default -> throw new VortexException( "DecimalByteParts: unsupported mantissa child type: " + arr.getClass().getSimpleName()); diff --git a/reader/src/main/java/io/github/dfa1/vortex/reader/array/LazyDecimalArray.java b/reader/src/main/java/io/github/dfa1/vortex/reader/array/LazyDecimalArray.java index 66f976aeb..a0da632e4 100644 --- a/reader/src/main/java/io/github/dfa1/vortex/reader/array/LazyDecimalArray.java +++ b/reader/src/main/java/io/github/dfa1/vortex/reader/array/LazyDecimalArray.java @@ -45,6 +45,24 @@ public BigDecimal getDecimal(long i) { return new BigDecimal(readSignedLe(buf, i * byteWidth, byteWidth), d.scale()); } + /// Reads widths up to 8 bytes straight from the buffer, without a [BigInteger]. + /// + /// @param i row index, `0 <= i < length()` + /// @return the unscaled value at `i` + /// @throws ArithmeticException if a 16- or 32-byte value does not fit a `long` + @Override + public long getUnscaledLong(long i) { + Objects.checkIndex(i, length); + long offset = i * byteWidth; + return switch (byteWidth) { + case 1 -> buf.get(ValueLayout.JAVA_BYTE, offset); + case 2 -> buf.get(LE_SHORT, offset); + case 4 -> buf.get(LE_INT, offset); + case 8 -> buf.get(LE_LONG, offset); + default -> readSignedLe(buf, offset, byteWidth).longValueExact(); + }; + } + private static BigInteger readSignedLe(MemorySegment buf, long offset, int width) { return switch (width) { case 1 -> BigInteger.valueOf(buf.get(ValueLayout.JAVA_BYTE, offset)); diff --git a/reader/src/main/java/io/github/dfa1/vortex/reader/array/LazyDecimalBytePartsArray.java b/reader/src/main/java/io/github/dfa1/vortex/reader/array/LazyDecimalBytePartsArray.java index f5e7f278e..6c6ac0e44 100644 --- a/reader/src/main/java/io/github/dfa1/vortex/reader/array/LazyDecimalBytePartsArray.java +++ b/reader/src/main/java/io/github/dfa1/vortex/reader/array/LazyDecimalBytePartsArray.java @@ -38,6 +38,16 @@ public BigDecimal getDecimal(long i) { return new BigDecimal(DecimalBytePartsArrays.readMantissa(msp, i), d.scale()); } + /// Reads the mantissa child at its own width, without a [java.math.BigInteger]. + /// + /// @param i row index, `0 <= i < length()` + /// @return the unscaled value at `i` + @Override + public long getUnscaledLong(long i) { + Objects.checkIndex(i, length); + return DecimalBytePartsArrays.readMantissaLong(msp, i); + } + @Override public Array limited(long rows) { return new LazyDecimalBytePartsArray(dtype, rows, Array.limited(msp, rows)); diff --git a/reader/src/main/java/io/github/dfa1/vortex/reader/compute/Compute.java b/reader/src/main/java/io/github/dfa1/vortex/reader/compute/Compute.java index 66116a92c..42b791c5a 100644 --- a/reader/src/main/java/io/github/dfa1/vortex/reader/compute/Compute.java +++ b/reader/src/main/java/io/github/dfa1/vortex/reader/compute/Compute.java @@ -128,8 +128,9 @@ public static LongPredicate matcher(Chunk chunk, RowFilter filter) { /// /// The result is empty when the shapes do not qualify: another group column type, a string /// column not dictionary-encoded in this chunk, more than 65,536 groups in the chunk, or an - /// `aggColumn` that is not an integer or floating column (`f16` and `u64` included, which this - /// fold does not total). The caller then groups the rows itself. + /// `aggColumn` that is not an integer, floating or decimal column (`f16`, `u64` and decimals past + /// precision 18 included, which this fold does not total; a decimal folds exactly, as unscaled + /// longs, and a decimal sum past a long declines too). The caller then groups the rows itself. /// /// @param chunk the decoded chunk holding every referenced column /// @param groupColumn the column to group by diff --git a/reader/src/main/java/io/github/dfa1/vortex/reader/compute/GroupedAggregate.java b/reader/src/main/java/io/github/dfa1/vortex/reader/compute/GroupedAggregate.java index 15b23bb36..6a9557757 100644 --- a/reader/src/main/java/io/github/dfa1/vortex/reader/compute/GroupedAggregate.java +++ b/reader/src/main/java/io/github/dfa1/vortex/reader/compute/GroupedAggregate.java @@ -9,12 +9,14 @@ import io.github.dfa1.vortex.reader.RowFilter; import io.github.dfa1.vortex.reader.array.Array; import io.github.dfa1.vortex.reader.array.BoolArray; +import io.github.dfa1.vortex.reader.array.DecimalArray; import io.github.dfa1.vortex.reader.array.DoubleArray; import io.github.dfa1.vortex.reader.array.FloatArray; import io.github.dfa1.vortex.reader.array.VarBinDictArray; import io.github.dfa1.vortex.reader.array.VarBinOffsetArray; import io.github.dfa1.vortex.reader.array.VarBinSlicedArray; +import java.math.BigDecimal; import java.util.Arrays; import java.util.HashMap; import java.util.Map; @@ -61,17 +63,26 @@ static Optional> aggregate(Chunk chunk, ColumnNam Array group = chunk.column(groupColumn); BoolArray validity = NumericColumns.validityOf(group); LongPredicate selected = filter == null ? null : FusedFilterAggregate.rowTest(chunk, filter); + try { + return group(group, validity, selected, lane, chunk.rowCount()); + } catch (ArithmeticException overflow) { + return Optional.empty(); // a decimal sum past a long: the caller folds the rows exactly + } + } + + private static Optional> group(Array group, BoolArray validity, + LongPredicate selected, Lane lane, long rowCount) { return switch (NumericColumns.unwrap(group)) { - case VarBinDictArray d -> byCode(d, 0, validity, selected, lane, chunk.rowCount()); + case VarBinDictArray d -> byCode(d, 0, validity, selected, lane, rowCount); case VarBinSlicedArray(var _, var _, VarBinDictArray d, long offset) -> - byCode(d, offset, validity, selected, lane, chunk.rowCount()); - case BoolArray b -> byKey(i -> b.getBoolean(i) ? 1 : 0, validity, selected, lane, chunk.rowCount(), + byCode(d, offset, validity, selected, lane, rowCount); + case BoolArray b -> byKey(i -> b.getBoolean(i) ? 1 : 0, validity, selected, lane, rowCount, key -> key != 0); case Array data when NumericColumns.isLongDomain(data) && data.dtype() instanceof DType.Primitive p && p.ptype() != PType.U64 -> { boolean unsigned = p.ptype().isUnsigned(); yield byKey(i -> NumericColumns.widenLong(data, unsigned, i), validity, selected, lane, - chunk.rowCount(), key -> key); + rowCount, key -> key); } default -> Optional.empty(); }; @@ -174,6 +185,8 @@ private static FilteredAggregate combine(FilteredAggregate a, FilteredAggregate sum = null; } else if (a.sum() instanceof Double x) { sum = x + b.sum().doubleValue(); + } else if (a.sum() instanceof BigDecimal x) { + sum = x.add((BigDecimal) b.sum()); } else { sum = a.sum().longValue() + b.sum().longValue(); } @@ -194,13 +207,20 @@ private static Object extreme(Object a, Object b, boolean min) { return (min ? cmp <= 0 : cmp >= 0) ? a : b; } - /// How the aggregate column's values are read: not at all (a row count), as longs, or as doubles. - private record Lane(Array data, BoolArray validity, boolean unsigned, boolean floating, boolean isFloat) { + /// How the aggregate column's values are read: not at all (a row count), as longs, as doubles, or + /// as the unscaled longs of a decimal (`scale >= 0`). + private record Lane(Array data, BoolArray validity, boolean unsigned, boolean floating, boolean isFloat, + int scale) { - private static final Lane COUNT_ONLY = new Lane(null, null, false, false, false); + private static final Lane COUNT_ONLY = new Lane(null, null, false, false, false, -1); + + boolean decimal() { + return scale >= 0; + } /// Picks the lane for `column`, or `null` when no lane folds it: a non-numeric column, an - /// `f16`, or a `u64` whose values past 2^63 have no signed-long home. + /// `f16`, a `u64` whose values past 2^63 have no signed-long home, or a decimal of precision + /// past 18, whose unscaled values need not fit a long. static Lane of(Array column) { if (column == null) { return COUNT_ONLY; @@ -208,11 +228,14 @@ static Lane of(Array column) { Array data = NumericColumns.unwrap(column); BoolArray validity = NumericColumns.validityOf(column); if (data instanceof DoubleArray || data instanceof FloatArray) { - return new Lane(data, validity, false, true, data instanceof FloatArray); + return new Lane(data, validity, false, true, data instanceof FloatArray, -1); } if (NumericColumns.isLongDomain(data) && data.dtype() instanceof DType.Primitive p && p.ptype() != PType.U64) { - return new Lane(data, validity, p.ptype().isUnsigned(), false, false); + return new Lane(data, validity, p.ptype().isUnsigned(), false, false, -1); + } + if (data instanceof DecimalArray && data.dtype() instanceof DType.Decimal d && d.precision() <= 18) { + return new Lane(data, validity, false, false, false, d.scale()); } return null; } @@ -261,6 +284,16 @@ void add(int g, long i) { if (first || Double.compare(v, doubleMax[g]) > 0) { doubleMax[g] = v; } + } else if (lane.decimal()) { + long v = ((DecimalArray) data).getUnscaledLong(i); + // a decimal sum must be exact: an overflow throws, and the caller declines the chunk + longSum[g] = Math.addExact(longSum[g], v); + if (first || v < longMin[g]) { + longMin[g] = v; + } + if (first || v > longMax[g]) { + longMax[g] = v; + } } else { long v = NumericColumns.widenLong(data, lane.unsigned(), i); longSum[g] += v; @@ -282,6 +315,12 @@ FilteredAggregate result(int g) { return new FilteredAggregate(rows[g], nonNull[g], doubleSum[g], any ? doubleMin[g] : null, any ? doubleMax[g] : null); } + if (lane.decimal()) { + int scale = lane.scale(); + return new FilteredAggregate(rows[g], nonNull[g], BigDecimal.valueOf(longSum[g], scale), + any ? BigDecimal.valueOf(longMin[g], scale) : null, + any ? BigDecimal.valueOf(longMax[g], scale) : null); + } return new FilteredAggregate(rows[g], nonNull[g], longSum[g], any ? longMin[g] : null, any ? longMax[g] : null); } diff --git a/reader/src/test/java/io/github/dfa1/vortex/reader/array/LazyDecimalArrayTest.java b/reader/src/test/java/io/github/dfa1/vortex/reader/array/LazyDecimalArrayTest.java index 324fb9c52..f09ad0b54 100644 --- a/reader/src/test/java/io/github/dfa1/vortex/reader/array/LazyDecimalArrayTest.java +++ b/reader/src/test/java/io/github/dfa1/vortex/reader/array/LazyDecimalArrayTest.java @@ -4,6 +4,8 @@ import io.github.dfa1.vortex.core.model.DType; import io.github.dfa1.vortex.core.error.VortexException; import org.junit.jupiter.api.Test; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.ValueSource; import java.lang.foreign.Arena; import java.lang.foreign.MemorySegment; @@ -143,4 +145,40 @@ void getDecimal_unsupportedByteWidth_throws() { .hasMessageContaining("unsupported element width"); } } + + /// A decimal fold totals unscaled longs read straight from the buffer; at every width they must + /// equal the unscaled value [LazyDecimalArray#getDecimal(long)] decodes, negatives included. + @ParameterizedTest + @ValueSource(ints = {1, 2, 4, 8, 16}) + void getUnscaledLong_matchesTheDecodedDecimalAtEveryWidth(int width) { + // Given — -7 and 42, each written sign-extended to `width` little-endian bytes + try (Arena arena = Arena.ofConfined()) { + MemorySegment buf = arena.allocate(2L * width); + long[] values = {-7, 42}; + for (int row = 0; row < 2; row++) { + for (int b = 0; b < width; b++) { + buf.set(ValueLayout.JAVA_BYTE, (long) row * width + b, (byte) (values[row] >> Math.min(8 * b, 63))); + } + } + LazyDecimalArray sut = new LazyDecimalArray(new DType.Decimal((byte) 2, (byte) 1, false), 2, buf, width); + + // When / Then + assertThat(sut.getUnscaledLong(0)).isEqualTo(-7L).isEqualTo(sut.getDecimal(0).unscaledValue().longValue()); + assertThat(sut.getUnscaledLong(1)).isEqualTo(42L); + } + } + + /// A 16-byte value past a long cannot be totalled as one; the caller must see it, not a wrapped value. + @Test + void getUnscaledLong_wideValuePastALong_throws() { + // Given — 2^64 as a 16-byte little-endian value + try (Arena arena = Arena.ofConfined()) { + MemorySegment buf = arena.allocate(16); + buf.set(ValueLayout.JAVA_BYTE, 8, (byte) 1); + LazyDecimalArray sut = new LazyDecimalArray(new DType.Decimal((byte) 38, (byte) 0, false), 1, buf, 16); + + // When / Then + assertThatThrownBy(() -> sut.getUnscaledLong(0)).isInstanceOf(ArithmeticException.class); + } + } } diff --git a/reader/src/test/java/io/github/dfa1/vortex/reader/array/LazyDecimalBytePartsArrayTest.java b/reader/src/test/java/io/github/dfa1/vortex/reader/array/LazyDecimalBytePartsArrayTest.java index efe06382c..8019814ca 100644 --- a/reader/src/test/java/io/github/dfa1/vortex/reader/array/LazyDecimalBytePartsArrayTest.java +++ b/reader/src/test/java/io/github/dfa1/vortex/reader/array/LazyDecimalBytePartsArrayTest.java @@ -120,4 +120,16 @@ void getDecimal_nonDecimalDtype_throws() { .hasMessageContaining("non-decimal"); } } + + /// The decimal fold reads the mantissa child at its own width instead of through a BigInteger. + @Test + void getUnscaledLong_readsTheMantissaChild() { + // Given + LazyDecimalBytePartsArray sut = new LazyDecimalBytePartsArray( + new DType.Decimal((byte) 10, (byte) 2, false), 2, TestArrays.longs(-12_345L, 7L)); + + // When / Then + assertThat(sut.getUnscaledLong(0)).isEqualTo(-12_345L); + assertThat(sut.getUnscaledLong(1)).isEqualTo(7L); + } } diff --git a/reader/src/test/java/io/github/dfa1/vortex/reader/compute/ComputeTest.java b/reader/src/test/java/io/github/dfa1/vortex/reader/compute/ComputeTest.java index e72b356ad..1f800c10c 100644 --- a/reader/src/test/java/io/github/dfa1/vortex/reader/compute/ComputeTest.java +++ b/reader/src/test/java/io/github/dfa1/vortex/reader/compute/ComputeTest.java @@ -6,6 +6,7 @@ import io.github.dfa1.vortex.reader.Chunk; import io.github.dfa1.vortex.reader.RowFilter; import io.github.dfa1.vortex.reader.array.Array; +import io.github.dfa1.vortex.reader.array.LazyDecimalBytePartsArray; import io.github.dfa1.vortex.reader.array.MaskedArray; import io.github.dfa1.vortex.reader.array.VarBinArray; import io.github.dfa1.vortex.reader.array.VarBinDictArray; @@ -17,6 +18,7 @@ import org.junit.jupiter.params.provider.MethodSource; import java.lang.foreign.Arena; +import java.math.BigDecimal; import java.util.List; import java.util.Map; import java.util.function.LongPredicate; @@ -376,6 +378,46 @@ void integerGroupColumnWithTooManyValues_isEmpty() { assertThat(Compute.groupedAggregate(chunk, G, null, null)).isEmpty(); } + /// A decimal of precision up to 18 folds as unscaled longs and comes back exact, at its scale. + @Test + void decimalAggregate_foldsExactlyAtTheColumnScale() { + // Given — groups a, b, a; v = 1.25, -0.50, 2.00 as DECIMAL(10, 2) + Array group = ComputeArrays.utf8DictArray(ARENA, new String[]{"a", "b"}, new int[]{0, 1, 0}); + Array v = new LazyDecimalBytePartsArray(new DType.Decimal((byte) 10, (byte) 2, false), 3, + ComputeArrays.longArray(ARENA, 125, -50, 200)); + + // When + Map result = Compute.groupedAggregate(chunk(group, v), G, null, V).orElseThrow(); + + // Then + assertThat(result.get("a")).isEqualTo(new FilteredAggregate(2, 2, new BigDecimal("3.25"), + new BigDecimal("1.25"), new BigDecimal("2.00"))); + assertThat(result.get("b").sum()).isEqualTo(new BigDecimal("-0.50")); + } + + /// A decimal sum must be exact: one past a long declines rather than wrap. + @Test + void decimalSumPastALong_isEmpty() { + // Given + Array group = ComputeArrays.utf8DictArray(ARENA, new String[]{"a"}, new int[]{0, 0}); + Array v = new LazyDecimalBytePartsArray(new DType.Decimal((byte) 18, (byte) 0, false), 2, + ComputeArrays.longArray(ARENA, Long.MAX_VALUE, 1)); + + // When / Then + assertThat(Compute.groupedAggregate(chunk(group, v), G, null, V)).isEmpty(); + } + + @Test + void decimalPastPrecision18_isEmpty() { + // Given + Array group = ComputeArrays.utf8DictArray(ARENA, new String[]{"a"}, new int[]{0}); + Array v = new LazyDecimalBytePartsArray(new DType.Decimal((byte) 19, (byte) 0, false), 1, + ComputeArrays.longArray(ARENA, 1)); + + // When / Then + assertThat(Compute.groupedAggregate(chunk(group, v), G, null, V)).isEmpty(); + } + /// The caller groups the rows itself when the kernel cannot: it must say so, not guess. @Test void groupColumnNotDictionaryEncoded_isEmpty() {