From 905f6b35a525051d316423cc77c27fe350f41518 Mon Sep 17 00:00:00 2001 From: Davide Angelocola Date: Sun, 18 Oct 2026 16:31:14 +0200 Subject: [PATCH 01/13] test(fuzz): reader and FSST fuzz targets, seeded with one file per encoding openMalformedFile stopped after open(), so no encoding decoder had seen fuzzed bytes. The fuzz module now has five targets: - scanMalformedFile: columnStats(), then every value of every chunk through ArrayWalker, which reads each leaf by bulk materialize and by the per-row getters (separate code in the Lazy* arrays), recursing through masked, struct, list, map, union and variant arrays, and again limited to half the rows for row-range slicing. - filterMalformedFile: the scan with a filter the fuzzer chooses (column, predicate, a value of the type the column compares against), so zone maps are parsed and chunks pruned. Its input is a 16-byte choice header then the file: a FuzzedDataProvider takes numbers from the end of the input, where the trailer is. - parseMetadataBlobs: the footer, layout and dtype FlatBuffers themselves, wrapped by MetadataBlobs in a valid postscript and trailer, so every mutation lands in the metadata parse; public API only, no access seam into the reader's package. - compressDecompressRoundTrip: an FSST table trained on fuzzed rows must compress every row identically through the byte[] and MemorySegment paths and decompress it back. A correctness oracle: Decompressor is an internal boundary, the reader's FSST validation is the scan target's. A byte mutator never builds a valid FSST symbol table or Pco header from the five hand-made seeds, so SeedCorpusGenerator writes one small file per encoding (FastLanes bitpacked/FOR/delta/RLE, RunEnd, ZigZag, Sparse, Pco, ALP, ALP-RD, Dict, FSST, OnPair, Zstd), each forced through its encoding and checked to really use it, and mirrors every seed into the filter and metadata-blob corpora. Opt-in, regenerated by a system property. Refs #493 Co-Authored-By: Claude Sonnet 5 Claude-Session: https://claude.ai/code/session_01WtuVGQ5h5Lk6Zn4JnkxF1K --- CLAUDE.md | 1 + fuzz/pom.xml | 5 + .../github/dfa1/vortex/fuzz/ArrayWalker.java | 165 ++++++++++++++++++ .../github/dfa1/vortex/fuzz/FsstFuzzTest.java | 59 +++++++ .../dfa1/vortex/fuzz/MetadataBlobs.java | 67 +++++++ .../dfa1/vortex/fuzz/SeedCorpusGenerator.java | 142 +++++++++++++++ .../vortex/fuzz/VortexReaderFuzzTest.java | 143 ++++++++++++++- .../filterMalformedFile/seed-booleans | Bin 0 -> 2980 bytes .../filterMalformedFile/seed-chunked | Bin 0 -> 24516 bytes .../seed-fastlanes-bitpacked | Bin 0 -> 3529 bytes .../filterMalformedFile/seed-fastlanes-delta | Bin 0 -> 18473 bytes .../filterMalformedFile/seed-fastlanes-for | Bin 0 -> 10604 bytes .../filterMalformedFile/seed-fastlanes-rle | Bin 0 -> 6696 bytes .../filterMalformedFile/seed-null | Bin 0 -> 2004 bytes .../filterMalformedFile/seed-primitives | Bin 0 -> 4744 bytes .../filterMalformedFile/seed-varbin | Bin 0 -> 2560 bytes .../filterMalformedFile/seed-vortex-alp | Bin 0 -> 10730 bytes .../filterMalformedFile/seed-vortex-alprd | Bin 0 -> 16202 bytes .../filterMalformedFile/seed-vortex-dict | Bin 0 -> 3112 bytes .../filterMalformedFile/seed-vortex-fsst | Bin 0 -> 8601 bytes .../filterMalformedFile/seed-vortex-onpair | Bin 0 -> 9377 bytes .../filterMalformedFile/seed-vortex-pco | Bin 0 -> 2435 bytes .../filterMalformedFile/seed-vortex-runend | Bin 0 -> 3816 bytes .../filterMalformedFile/seed-vortex-sparse | Bin 0 -> 1929 bytes .../filterMalformedFile/seed-vortex-zigzag | Bin 0 -> 6184 bytes .../filterMalformedFile/seed-vortex-zstd | Bin 0 -> 2400 bytes .../filterMalformedFile/seed-zstd-buffers | Bin 0 -> 8848 bytes .../parseMetadataBlobs/seed-booleans | Bin 0 -> 1428 bytes .../parseMetadataBlobs/seed-chunked | Bin 0 -> 1308 bytes .../seed-fastlanes-bitpacked | Bin 0 -> 836 bytes .../parseMetadataBlobs/seed-fastlanes-delta | Bin 0 -> 868 bytes .../parseMetadataBlobs/seed-fastlanes-for | Bin 0 -> 868 bytes .../parseMetadataBlobs/seed-fastlanes-rle | Bin 0 -> 868 bytes .../parseMetadataBlobs/seed-null | Bin 0 -> 1292 bytes .../parseMetadataBlobs/seed-primitives | Bin 0 -> 1648 bytes .../parseMetadataBlobs/seed-varbin | Bin 0 -> 1304 bytes .../parseMetadataBlobs/seed-vortex-alp | Bin 0 -> 892 bytes .../parseMetadataBlobs/seed-vortex-alprd | Bin 0 -> 924 bytes .../parseMetadataBlobs/seed-vortex-dict | Bin 0 -> 868 bytes .../parseMetadataBlobs/seed-vortex-fsst | Bin 0 -> 932 bytes .../parseMetadataBlobs/seed-vortex-onpair | Bin 0 -> 940 bytes .../parseMetadataBlobs/seed-vortex-pco | Bin 0 -> 828 bytes .../parseMetadataBlobs/seed-vortex-runend | Bin 0 -> 868 bytes .../parseMetadataBlobs/seed-vortex-sparse | Bin 0 -> 836 bytes .../parseMetadataBlobs/seed-vortex-zigzag | Bin 0 -> 868 bytes .../parseMetadataBlobs/seed-vortex-zstd | Bin 0 -> 900 bytes .../parseMetadataBlobs/seed-zstd-buffers | Bin 0 -> 1432 bytes .../scanMalformedFile/seed-booleans | Bin 0 -> 2964 bytes .../scanMalformedFile/seed-chunked | Bin 0 -> 24500 bytes .../seed-fastlanes-bitpacked | Bin 0 -> 3513 bytes .../scanMalformedFile/seed-fastlanes-delta | Bin 0 -> 18457 bytes .../scanMalformedFile/seed-fastlanes-for | Bin 0 -> 10588 bytes .../scanMalformedFile/seed-fastlanes-rle | Bin 0 -> 6680 bytes .../scanMalformedFile/seed-null | Bin 0 -> 1988 bytes .../scanMalformedFile/seed-primitives | Bin 0 -> 4728 bytes .../scanMalformedFile/seed-varbin | Bin 0 -> 2544 bytes .../scanMalformedFile/seed-vortex-alp | Bin 0 -> 10714 bytes .../scanMalformedFile/seed-vortex-alprd | Bin 0 -> 16186 bytes .../scanMalformedFile/seed-vortex-dict | Bin 0 -> 3096 bytes .../scanMalformedFile/seed-vortex-fsst | Bin 0 -> 8585 bytes .../scanMalformedFile/seed-vortex-onpair | Bin 0 -> 9361 bytes .../scanMalformedFile/seed-vortex-pco | Bin 0 -> 2419 bytes .../scanMalformedFile/seed-vortex-runend | Bin 0 -> 3800 bytes .../scanMalformedFile/seed-vortex-sparse | Bin 0 -> 1913 bytes .../scanMalformedFile/seed-vortex-zigzag | Bin 0 -> 6168 bytes .../scanMalformedFile/seed-vortex-zstd | Bin 0 -> 2384 bytes .../scanMalformedFile/seed-zstd-buffers | Bin 0 -> 8832 bytes 67 files changed, 573 insertions(+), 9 deletions(-) create mode 100644 fuzz/src/test/java/io/github/dfa1/vortex/fuzz/ArrayWalker.java create mode 100644 fuzz/src/test/java/io/github/dfa1/vortex/fuzz/FsstFuzzTest.java create mode 100644 fuzz/src/test/java/io/github/dfa1/vortex/fuzz/MetadataBlobs.java create mode 100644 fuzz/src/test/java/io/github/dfa1/vortex/fuzz/SeedCorpusGenerator.java create mode 100644 fuzz/src/test/resources/io/github/dfa1/vortex/fuzz/VortexReaderFuzzTestInputs/filterMalformedFile/seed-booleans create mode 100644 fuzz/src/test/resources/io/github/dfa1/vortex/fuzz/VortexReaderFuzzTestInputs/filterMalformedFile/seed-chunked create mode 100644 fuzz/src/test/resources/io/github/dfa1/vortex/fuzz/VortexReaderFuzzTestInputs/filterMalformedFile/seed-fastlanes-bitpacked create mode 100644 fuzz/src/test/resources/io/github/dfa1/vortex/fuzz/VortexReaderFuzzTestInputs/filterMalformedFile/seed-fastlanes-delta create mode 100644 fuzz/src/test/resources/io/github/dfa1/vortex/fuzz/VortexReaderFuzzTestInputs/filterMalformedFile/seed-fastlanes-for create mode 100644 fuzz/src/test/resources/io/github/dfa1/vortex/fuzz/VortexReaderFuzzTestInputs/filterMalformedFile/seed-fastlanes-rle create mode 100644 fuzz/src/test/resources/io/github/dfa1/vortex/fuzz/VortexReaderFuzzTestInputs/filterMalformedFile/seed-null create mode 100644 fuzz/src/test/resources/io/github/dfa1/vortex/fuzz/VortexReaderFuzzTestInputs/filterMalformedFile/seed-primitives create mode 100644 fuzz/src/test/resources/io/github/dfa1/vortex/fuzz/VortexReaderFuzzTestInputs/filterMalformedFile/seed-varbin create mode 100644 fuzz/src/test/resources/io/github/dfa1/vortex/fuzz/VortexReaderFuzzTestInputs/filterMalformedFile/seed-vortex-alp create mode 100644 fuzz/src/test/resources/io/github/dfa1/vortex/fuzz/VortexReaderFuzzTestInputs/filterMalformedFile/seed-vortex-alprd create mode 100644 fuzz/src/test/resources/io/github/dfa1/vortex/fuzz/VortexReaderFuzzTestInputs/filterMalformedFile/seed-vortex-dict create mode 100644 fuzz/src/test/resources/io/github/dfa1/vortex/fuzz/VortexReaderFuzzTestInputs/filterMalformedFile/seed-vortex-fsst create mode 100644 fuzz/src/test/resources/io/github/dfa1/vortex/fuzz/VortexReaderFuzzTestInputs/filterMalformedFile/seed-vortex-onpair create mode 100644 fuzz/src/test/resources/io/github/dfa1/vortex/fuzz/VortexReaderFuzzTestInputs/filterMalformedFile/seed-vortex-pco create mode 100644 fuzz/src/test/resources/io/github/dfa1/vortex/fuzz/VortexReaderFuzzTestInputs/filterMalformedFile/seed-vortex-runend create mode 100644 fuzz/src/test/resources/io/github/dfa1/vortex/fuzz/VortexReaderFuzzTestInputs/filterMalformedFile/seed-vortex-sparse create mode 100644 fuzz/src/test/resources/io/github/dfa1/vortex/fuzz/VortexReaderFuzzTestInputs/filterMalformedFile/seed-vortex-zigzag create mode 100644 fuzz/src/test/resources/io/github/dfa1/vortex/fuzz/VortexReaderFuzzTestInputs/filterMalformedFile/seed-vortex-zstd create mode 100644 fuzz/src/test/resources/io/github/dfa1/vortex/fuzz/VortexReaderFuzzTestInputs/filterMalformedFile/seed-zstd-buffers create mode 100644 fuzz/src/test/resources/io/github/dfa1/vortex/fuzz/VortexReaderFuzzTestInputs/parseMetadataBlobs/seed-booleans create mode 100644 fuzz/src/test/resources/io/github/dfa1/vortex/fuzz/VortexReaderFuzzTestInputs/parseMetadataBlobs/seed-chunked create mode 100644 fuzz/src/test/resources/io/github/dfa1/vortex/fuzz/VortexReaderFuzzTestInputs/parseMetadataBlobs/seed-fastlanes-bitpacked create mode 100644 fuzz/src/test/resources/io/github/dfa1/vortex/fuzz/VortexReaderFuzzTestInputs/parseMetadataBlobs/seed-fastlanes-delta create mode 100644 fuzz/src/test/resources/io/github/dfa1/vortex/fuzz/VortexReaderFuzzTestInputs/parseMetadataBlobs/seed-fastlanes-for create mode 100644 fuzz/src/test/resources/io/github/dfa1/vortex/fuzz/VortexReaderFuzzTestInputs/parseMetadataBlobs/seed-fastlanes-rle create mode 100644 fuzz/src/test/resources/io/github/dfa1/vortex/fuzz/VortexReaderFuzzTestInputs/parseMetadataBlobs/seed-null create mode 100644 fuzz/src/test/resources/io/github/dfa1/vortex/fuzz/VortexReaderFuzzTestInputs/parseMetadataBlobs/seed-primitives create mode 100644 fuzz/src/test/resources/io/github/dfa1/vortex/fuzz/VortexReaderFuzzTestInputs/parseMetadataBlobs/seed-varbin create mode 100644 fuzz/src/test/resources/io/github/dfa1/vortex/fuzz/VortexReaderFuzzTestInputs/parseMetadataBlobs/seed-vortex-alp create mode 100644 fuzz/src/test/resources/io/github/dfa1/vortex/fuzz/VortexReaderFuzzTestInputs/parseMetadataBlobs/seed-vortex-alprd create mode 100644 fuzz/src/test/resources/io/github/dfa1/vortex/fuzz/VortexReaderFuzzTestInputs/parseMetadataBlobs/seed-vortex-dict create mode 100644 fuzz/src/test/resources/io/github/dfa1/vortex/fuzz/VortexReaderFuzzTestInputs/parseMetadataBlobs/seed-vortex-fsst create mode 100644 fuzz/src/test/resources/io/github/dfa1/vortex/fuzz/VortexReaderFuzzTestInputs/parseMetadataBlobs/seed-vortex-onpair create mode 100644 fuzz/src/test/resources/io/github/dfa1/vortex/fuzz/VortexReaderFuzzTestInputs/parseMetadataBlobs/seed-vortex-pco create mode 100644 fuzz/src/test/resources/io/github/dfa1/vortex/fuzz/VortexReaderFuzzTestInputs/parseMetadataBlobs/seed-vortex-runend create mode 100644 fuzz/src/test/resources/io/github/dfa1/vortex/fuzz/VortexReaderFuzzTestInputs/parseMetadataBlobs/seed-vortex-sparse create mode 100644 fuzz/src/test/resources/io/github/dfa1/vortex/fuzz/VortexReaderFuzzTestInputs/parseMetadataBlobs/seed-vortex-zigzag create mode 100644 fuzz/src/test/resources/io/github/dfa1/vortex/fuzz/VortexReaderFuzzTestInputs/parseMetadataBlobs/seed-vortex-zstd create mode 100644 fuzz/src/test/resources/io/github/dfa1/vortex/fuzz/VortexReaderFuzzTestInputs/parseMetadataBlobs/seed-zstd-buffers create mode 100644 fuzz/src/test/resources/io/github/dfa1/vortex/fuzz/VortexReaderFuzzTestInputs/scanMalformedFile/seed-booleans create mode 100644 fuzz/src/test/resources/io/github/dfa1/vortex/fuzz/VortexReaderFuzzTestInputs/scanMalformedFile/seed-chunked create mode 100644 fuzz/src/test/resources/io/github/dfa1/vortex/fuzz/VortexReaderFuzzTestInputs/scanMalformedFile/seed-fastlanes-bitpacked create mode 100644 fuzz/src/test/resources/io/github/dfa1/vortex/fuzz/VortexReaderFuzzTestInputs/scanMalformedFile/seed-fastlanes-delta create mode 100644 fuzz/src/test/resources/io/github/dfa1/vortex/fuzz/VortexReaderFuzzTestInputs/scanMalformedFile/seed-fastlanes-for create mode 100644 fuzz/src/test/resources/io/github/dfa1/vortex/fuzz/VortexReaderFuzzTestInputs/scanMalformedFile/seed-fastlanes-rle create mode 100644 fuzz/src/test/resources/io/github/dfa1/vortex/fuzz/VortexReaderFuzzTestInputs/scanMalformedFile/seed-null create mode 100644 fuzz/src/test/resources/io/github/dfa1/vortex/fuzz/VortexReaderFuzzTestInputs/scanMalformedFile/seed-primitives create mode 100644 fuzz/src/test/resources/io/github/dfa1/vortex/fuzz/VortexReaderFuzzTestInputs/scanMalformedFile/seed-varbin create mode 100644 fuzz/src/test/resources/io/github/dfa1/vortex/fuzz/VortexReaderFuzzTestInputs/scanMalformedFile/seed-vortex-alp create mode 100644 fuzz/src/test/resources/io/github/dfa1/vortex/fuzz/VortexReaderFuzzTestInputs/scanMalformedFile/seed-vortex-alprd create mode 100644 fuzz/src/test/resources/io/github/dfa1/vortex/fuzz/VortexReaderFuzzTestInputs/scanMalformedFile/seed-vortex-dict create mode 100644 fuzz/src/test/resources/io/github/dfa1/vortex/fuzz/VortexReaderFuzzTestInputs/scanMalformedFile/seed-vortex-fsst create mode 100644 fuzz/src/test/resources/io/github/dfa1/vortex/fuzz/VortexReaderFuzzTestInputs/scanMalformedFile/seed-vortex-onpair create mode 100644 fuzz/src/test/resources/io/github/dfa1/vortex/fuzz/VortexReaderFuzzTestInputs/scanMalformedFile/seed-vortex-pco create mode 100644 fuzz/src/test/resources/io/github/dfa1/vortex/fuzz/VortexReaderFuzzTestInputs/scanMalformedFile/seed-vortex-runend create mode 100644 fuzz/src/test/resources/io/github/dfa1/vortex/fuzz/VortexReaderFuzzTestInputs/scanMalformedFile/seed-vortex-sparse create mode 100644 fuzz/src/test/resources/io/github/dfa1/vortex/fuzz/VortexReaderFuzzTestInputs/scanMalformedFile/seed-vortex-zigzag create mode 100644 fuzz/src/test/resources/io/github/dfa1/vortex/fuzz/VortexReaderFuzzTestInputs/scanMalformedFile/seed-vortex-zstd create mode 100644 fuzz/src/test/resources/io/github/dfa1/vortex/fuzz/VortexReaderFuzzTestInputs/scanMalformedFile/seed-zstd-buffers diff --git a/CLAUDE.md b/CLAUDE.md index 09422ee36..13fdb4d10 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -86,6 +86,7 @@ scripts/hydrate-raincloud-corpus.sh --max-mb 200 # hydrate real-world conforma ./mvnw verify -pl integration -am -Dvortex.it.excludedGroups= -Dit.test="RaincloudConformanceIntegrationTest" ./mvnw test -pl fuzz -am -Dvortex.fuzz.excludedGroups= # Jazzer regression mode (ADR 0020) JAZZER_FUZZ=1 ./mvnw test -pl fuzz -am -Dvortex.fuzz.excludedGroups= # actually fuzz (unbounded) +./mvnw test -pl fuzz -am -Dtest=SeedCorpusGenerator -Dsurefire.failIfNoSpecifiedTests=false -Dvortex.fuzz.regenerateSeeds=true # rewrite per-encoding seeds ``` Tag-excluded from a routine local build, even when named with `-Dit.test`/`-Dtest`: diff --git a/fuzz/pom.xml b/fuzz/pom.xml index 70a66671b..702f820e1 100644 --- a/fuzz/pom.xml +++ b/fuzz/pom.xml @@ -30,6 +30,11 @@ vortex-reader test + + io.github.dfa1.vortex + vortex-writer + test + org.junit.jupiter junit-jupiter diff --git a/fuzz/src/test/java/io/github/dfa1/vortex/fuzz/ArrayWalker.java b/fuzz/src/test/java/io/github/dfa1/vortex/fuzz/ArrayWalker.java new file mode 100644 index 000000000..d5880700d --- /dev/null +++ b/fuzz/src/test/java/io/github/dfa1/vortex/fuzz/ArrayWalker.java @@ -0,0 +1,165 @@ +package io.github.dfa1.vortex.fuzz; + +import java.lang.foreign.Arena; +import java.util.function.Supplier; + +import io.github.dfa1.vortex.core.error.VortexException; +import io.github.dfa1.vortex.reader.Chunk; +import io.github.dfa1.vortex.reader.array.Array; +import io.github.dfa1.vortex.reader.array.BoolArray; +import io.github.dfa1.vortex.reader.array.ByteArray; +import io.github.dfa1.vortex.reader.array.DecimalArray; +import io.github.dfa1.vortex.reader.array.DoubleArray; +import io.github.dfa1.vortex.reader.array.FixedSizeListArray; +import io.github.dfa1.vortex.reader.array.Float16Array; +import io.github.dfa1.vortex.reader.array.FloatArray; +import io.github.dfa1.vortex.reader.array.IntArray; +import io.github.dfa1.vortex.reader.array.ListArray; +import io.github.dfa1.vortex.reader.array.ListViewArray; +import io.github.dfa1.vortex.reader.array.LongArray; +import io.github.dfa1.vortex.reader.array.MapArray; +import io.github.dfa1.vortex.reader.array.MaskedArray; +import io.github.dfa1.vortex.reader.array.ShortArray; +import io.github.dfa1.vortex.reader.array.StructArray; +import io.github.dfa1.vortex.reader.array.UnionArray; +import io.github.dfa1.vortex.reader.array.VarBinArray; +import io.github.dfa1.vortex.reader.array.VariantArray; + +/// Reads every value of a decoded chunk, so a fuzz target reaches the code that only runs on value +/// access: `Lazy*` arrays decode when read, not when the chunk is built, and the reader's security +/// contract holds there too. +/// +/// Each leaf is read twice over, through the two decode paths it has: bulk +/// [Array#materialize(java.lang.foreign.SegmentAllocator)] and the per-row getters, which are separate +/// code in the lazy arrays. Every column is walked again limited to half its rows, for row-range +/// slicing (where a buffer-alignment abort once hid). A [VortexException] ends only the walk of the +/// array that threw, so one rejected column does not hide the others from the fuzzer. +final class ArrayWalker { + + private ArrayWalker() { + } + + /// Reads every value of every column of `chunk`, whole and limited to half its rows. + /// + /// @param chunk the decoded chunk + /// @param arena allocator for materialized values + static void walk(Chunk chunk, Arena arena) { + for (Chunk.Column column : chunk.columns().values()) { + Array array = column.array(); + walk(array, arena); + walk(() -> Array.limited(array, array.length() / 2), arena); + } + } + + private static void walk(Supplier array, Arena arena) { + try { + walk(array.get(), arena); + } catch (VortexException _) { + // rejected as malformed: the contract holds + } + } + + private static void walk(Array array, Arena arena) { + switch (array) { + case MaskedArray masked -> { + walk(masked::validity, arena); + walk(masked::inner, arena); + } + case StructArray struct -> { + for (int i = 0; i < struct.fieldCount(); i++) { + int field = i; + walk(() -> struct.field(field), arena); + } + } + case ListArray list -> { + walk(list::offsets, arena); + walk(list::elements, arena); + } + case ListViewArray list -> { + walk(list::offsets, arena); + walk(list::sizes, arena); + walk(list::elements, arena); + } + case FixedSizeListArray list -> walk(list::elements, arena); + case MapArray map -> walk(map::entries, arena); + case UnionArray union -> { + walk(union::typeIds, arena); + for (int i = 0; i < union.variantCount(); i++) { + int variant = i; + walk(() -> union.variant(variant), arena); + } + } + case VariantArray variant -> { + walk(variant::coreStorage, arena); + if (variant.shredded() != null) { + walk(variant::shredded, arena); + } + } + default -> leaf(array, arena); + } + } + + private static void leaf(Array array, Arena arena) { + try { + array.materialize(arena); + } catch (VortexException _) { + // rejected as malformed: the per-row path below is separate code, so still walk it + } + long rows = array.length(); + switch (array) { + case LongArray a -> { + for (long i = 0; i < rows; i++) { + a.getLong(i); + } + } + case IntArray a -> { + for (long i = 0; i < rows; i++) { + a.getInt(i); + } + } + case ShortArray a -> { + for (long i = 0; i < rows; i++) { + a.getShort(i); + } + } + case ByteArray a -> { + for (long i = 0; i < rows; i++) { + a.getByte(i); + } + } + case DoubleArray a -> { + for (long i = 0; i < rows; i++) { + a.getDouble(i); + } + } + case FloatArray a -> { + for (long i = 0; i < rows; i++) { + a.getFloat(i); + } + } + case Float16Array a -> { + for (long i = 0; i < rows; i++) { + a.getFloat(i); + } + } + case BoolArray a -> { + for (long i = 0; i < rows; i++) { + a.getBoolean(i); + } + } + case DecimalArray a -> { + for (long i = 0; i < rows; i++) { + a.getDecimal(i); + } + } + case VarBinArray a -> { + for (long i = 0; i < rows; i++) { + a.getBytes(i); + } + } + default -> { + // NullArray, UnknownArray, GenericArray: nothing per row beyond materialize + } + } + } +} diff --git a/fuzz/src/test/java/io/github/dfa1/vortex/fuzz/FsstFuzzTest.java b/fuzz/src/test/java/io/github/dfa1/vortex/fuzz/FsstFuzzTest.java new file mode 100644 index 000000000..3cd245389 --- /dev/null +++ b/fuzz/src/test/java/io/github/dfa1/vortex/fuzz/FsstFuzzTest.java @@ -0,0 +1,59 @@ +package io.github.dfa1.vortex.fuzz; + +import static org.junit.jupiter.api.Assertions.assertArrayEquals; +import static org.junit.jupiter.api.Assertions.assertEquals; + +import java.lang.foreign.MemorySegment; +import java.util.Arrays; + +import com.code_intelligence.jazzer.api.FuzzedDataProvider; +import com.code_intelligence.jazzer.junit.FuzzTest; +import io.github.dfa1.vortex.fsst.Compressor; +import io.github.dfa1.vortex.fsst.CompressorBuilder; +import io.github.dfa1.vortex.fsst.Decompressor; +import org.junit.jupiter.api.Tag; + +/// FSST compressor correctness (#493): a table trained on fuzzed rows must compress every row, +/// fuzzed or not, to codes that decompress back to it, through both the `byte[]` and the +/// `MemorySegment` paths, which must emit the same codes. Not crash fuzzing of [Decompressor]: +/// it documents itself as an internal boundary, not an untrusted-input one; the reader's FSST +/// validation of untrusted files is covered by [VortexReaderFuzzTest]. +@Tag("fuzz") +class FsstFuzzTest { + + @FuzzTest + void compressDecompressRoundTrip(FuzzedDataProvider data) { + // Given + byte[][] rows = new byte[data.consumeInt(0, 64)][]; + for (int i = 0; i < rows.length; i++) { + rows[i] = data.consumeBytes(data.consumeInt(0, 128)); + } + Compressor compressor = new CompressorBuilder().seed(data.consumeLong()).train(rows); + Decompressor decompressor = compressor.toDecompressor(); + // A row the table was not trained on: all escapes is still a valid encoding. + byte[] unseen = data.consumeRemainingAsBytes(); + + // When / Then — every row round-trips (asserted per row) + for (byte[] row : rows) { + roundTrip(compressor, decompressor, row); + } + roundTrip(compressor, decompressor, unseen); + } + + private static void roundTrip(Compressor compressor, Decompressor decompressor, byte[] row) { + // Worst case: one escape pair per input byte; a code expands to at most 8 bytes. + byte[] codes = new byte[2 * row.length]; + int codesEnd = (int) compressor.compress(row, 0, row.length, codes, 0); + + MemorySegment segmentCodes = MemorySegment.ofArray(new byte[2 * row.length]); + long segmentEnd = compressor.compress(MemorySegment.ofArray(row), 0, row.length, segmentCodes, 0); + assertEquals(codesEnd, segmentEnd, "byte[] and MemorySegment code lengths"); + assertArrayEquals(Arrays.copyOf(codes, codesEnd), + segmentCodes.asSlice(0, segmentEnd).toArray(java.lang.foreign.ValueLayout.JAVA_BYTE), + "byte[] and MemorySegment codes"); + + byte[] decoded = new byte[8 * codesEnd]; + int decodedEnd = (int) decompressor.decompress(codes, 0, codesEnd, decoded, 0); + assertArrayEquals(row, Arrays.copyOf(decoded, decodedEnd), "decompress(compress(row))"); + } +} diff --git a/fuzz/src/test/java/io/github/dfa1/vortex/fuzz/MetadataBlobs.java b/fuzz/src/test/java/io/github/dfa1/vortex/fuzz/MetadataBlobs.java new file mode 100644 index 000000000..4f4d3b32d --- /dev/null +++ b/fuzz/src/test/java/io/github/dfa1/vortex/fuzz/MetadataBlobs.java @@ -0,0 +1,67 @@ +package io.github.dfa1.vortex.fuzz; + +import java.nio.ByteBuffer; +import java.nio.ByteOrder; + +import io.github.dfa1.vortex.core.fbs.FbsBuilder; +import io.github.dfa1.vortex.core.fbs.FbsPostscript; +import io.github.dfa1.vortex.core.fbs.FbsPostscriptSegment; +import io.github.dfa1.vortex.core.io.VortexFormat; + +/// Moves a Vortex file's three metadata blobs (footer, layout, dtype FlatBuffers) between their +/// place in a file and the flat form [VortexReaderFuzzTest#parseMetadataBlobs(byte[])] fuzzes. +final class MetadataBlobs { + + private MetadataBlobs() { + } + + /// Lays the blobs `data[from..]` (footer, layout, then dtype) out as a file: the blobs, a + /// postscript pointing at them, and the trailer. + /// + /// @param data the bytes holding the blobs + /// @param from where the footer starts in `data` + /// @param footerLength footer length + /// @param layoutLength layout length + /// @param dtypeLength dtype length (0 for none) + /// @return the file's bytes + static byte[] assemble(byte[] data, int from, int footerLength, int layoutLength, int dtypeLength) { + int body = footerLength + layoutLength + dtypeLength; + FbsBuilder builder = new FbsBuilder(256); + int footer = FbsPostscriptSegment.createFbsPostscriptSegment(builder, 0, footerLength, 0, 0, 0); + int layout = FbsPostscriptSegment.createFbsPostscriptSegment(builder, footerLength, layoutLength, 0, 0, 0); + int dtype = FbsPostscriptSegment.createFbsPostscriptSegment(builder, footerLength + layoutLength, + dtypeLength, 0, 0, 0); + FbsPostscript.finishFbsPostscriptBuffer(builder, FbsPostscript.createFbsPostscript(builder, dtype, layout, 0, footer)); + byte[] postscript = builder.sizedByteArray(); + + ByteBuffer file = ByteBuffer.allocate(body + postscript.length + VortexFormat.TRAILER_SIZE) + .order(ByteOrder.LITTLE_ENDIAN); + file.put(data, from, body).put(postscript) + .putShort((short) VortexFormat.VERSION).putShort((short) postscript.length) + .put(VortexFormat.MAGIC.asByteBuffer()); + return file.array(); + } + + /// The flat fuzz input for a valid file: the blob-length header, then its footer, layout and + /// dtype blobs as its postscript locates them. + /// + /// @param file a Vortex file's bytes + /// @return the header and the three blobs + static byte[] extract(byte[] file) { + ByteBuffer trailer = ByteBuffer.wrap(file, file.length - VortexFormat.TRAILER_SIZE, VortexFormat.TRAILER_SIZE) + .order(ByteOrder.LITTLE_ENDIAN); + trailer.getShort(); + int postscriptLength = Short.toUnsignedInt(trailer.getShort()); + int postscriptStart = file.length - VortexFormat.TRAILER_SIZE - postscriptLength; + var postscript = FbsPostscript.getRootAsFbsPostscript( + java.lang.foreign.MemorySegment.ofArray(file).asSlice(postscriptStart, postscriptLength)); + FbsPostscriptSegment[] segments = {postscript.footer(), postscript.layout(), postscript.dtype()}; + ByteBuffer out = ByteBuffer.allocate(VortexReaderFuzzTest.BLOB_HEADER_BYTES + + (int) (segments[0].length() + segments[1].length() + segments[2].length())); + out.putShort((short) segments[0].length()).putShort((short) segments[1].length()); + for (FbsPostscriptSegment segment : segments) { + out.put(file, (int) segment.offset(), (int) segment.length()); + } + return out.array(); + } +} diff --git a/fuzz/src/test/java/io/github/dfa1/vortex/fuzz/SeedCorpusGenerator.java b/fuzz/src/test/java/io/github/dfa1/vortex/fuzz/SeedCorpusGenerator.java new file mode 100644 index 000000000..66606ae6e --- /dev/null +++ b/fuzz/src/test/java/io/github/dfa1/vortex/fuzz/SeedCorpusGenerator.java @@ -0,0 +1,142 @@ +package io.github.dfa1.vortex.fuzz; + +import static org.junit.jupiter.api.Assertions.assertTrue; + +import java.io.IOException; +import java.nio.ByteBuffer; +import java.nio.channels.FileChannel; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.StandardOpenOption; +import java.util.Arrays; +import java.util.List; +import java.util.Map; +import java.util.Random; +import java.util.function.LongFunction; +import java.util.stream.Stream; + +import io.github.dfa1.vortex.core.model.ColumnName; +import io.github.dfa1.vortex.core.model.DType; +import io.github.dfa1.vortex.core.model.EncodingId; +import io.github.dfa1.vortex.writer.ColumnEncoding; +import io.github.dfa1.vortex.writer.VortexWriter; +import io.github.dfa1.vortex.writer.WriteOptions; +import org.junit.jupiter.api.AfterAll; +import org.junit.jupiter.api.condition.EnabledIfSystemProperty; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.Arguments; +import org.junit.jupiter.params.provider.MethodSource; + +/// Writes the per-encoding seed corpus of [VortexReaderFuzzTest#scanMalformedFile(byte[])] (#493), +/// [VortexReaderFuzzTest#filterMalformedFile(byte[])]'s copy of every scan seed behind a +/// filter-choice header, and [VortexReaderFuzzTest#parseMetadataBlobs(byte[])]'s metadata blobs of +/// every scan seed. +/// +/// A byte mutator never synthesizes a valid FSST symbol table or Pco chunk header from the +/// hand-made seeds, so each encoding gets a small file of its own that already decodes: the +/// fuzzer then mutates from inside the decoder instead of bouncing off its first check. Each +/// column is forced through one encoding with [ColumnEncoding#candidates(EncodingId...)], and +/// shaped so that encoding wins (a candidate that does not pay is silently left canonical, which +/// the assertion on the written bytes catches). Every 13th row is null so validity is exercised. +/// +/// Not part of any build; regenerate (then commit) with +/// `./mvnw test -pl fuzz -am -Dtest=SeedCorpusGenerator -Dsurefire.failIfNoSpecifiedTests=false -Dvortex.fuzz.regenerateSeeds=true` +@EnabledIfSystemProperty(named = "vortex.fuzz.regenerateSeeds", matches = "true") +class SeedCorpusGenerator { + + private static final Path INPUTS = Path.of("src/test/resources/io/github/dfa1/vortex/fuzz/VortexReaderFuzzTestInputs"); + private static final Path OUT = INPUTS.resolve("scanMalformedFile"); + private static final Path FILTER_OUT = INPUTS.resolve("filterMalformedFile"); + private static final Path BLOBS_OUT = INPUTS.resolve("parseMetadataBlobs"); + private static final ColumnName VALUE = ColumnName.of("v"); + // One full 1024-value FastLanes block plus a partial one, still only a few KB per file. + private static final int ROWS = 1_100; + + static Stream seeds() { + Random random = new Random(42); + double[] noise = random.doubles(ROWS).toArray(); + return Stream.of( + seed(DType.I32, i -> (int) (i % 100), EncodingId.FASTLANES_BITPACKED), + seed(DType.I64, i -> 1_000_000L + i % 100, EncodingId.FASTLANES_FOR), + seed(DType.I64, i -> i * 3 + i % 2, EncodingId.FASTLANES_DELTA), + seed(DType.I32, i -> (int) (i / 50), EncodingId.FASTLANES_RLE), + seed(DType.I64, i -> i / 100, EncodingId.VORTEX_RUNEND), + seed(DType.I32, i -> (int) (i % 2 == 0 ? i % 50 : -(i % 50)), EncodingId.VORTEX_ZIGZAG), + seed(DType.I64, i -> i % 97 == 0 ? i : 0L, EncodingId.VORTEX_SPARSE), + seed(DType.I64, i -> (i * 2_654_435_761L) % 1_000_003, EncodingId.VORTEX_PCO), + seed(DType.F64, i -> (i % 1000) / 100.0, EncodingId.VORTEX_ALP), + seed(DType.F64, i -> noise[(int) i], EncodingId.VORTEX_ALPRD, EncodingId.FASTLANES_BITPACKED), + seed(DType.UTF8, i -> "category-" + i % 5, EncodingId.VORTEX_DICT), + seed(DType.UTF8, i -> "https://example.com/items/" + i % 300 + "/detail", EncodingId.VORTEX_FSST), + seed(DType.UTF8, i -> "https://example.com/items/" + i % 300 + "/detail", EncodingId.VORTEX_ONPAIR), + seed(DType.UTF8, i -> "row " + i % 40 + " of the zstd seed", EncodingId.VORTEX_ZSTD)); + } + + /// Mirrors every scan seed (not the crash inputs) into the filter target's corpus behind a + /// header choosing column 0 and a predicate kind that cycles across seeds, against 3. + @AfterAll + static void writeFilterSeeds() throws IOException { + Files.createDirectories(FILTER_OUT); + List seeds = scanSeeds(); + for (int i = 0; i < seeds.size(); i++) { + ByteBuffer header = ByteBuffer.allocate(VortexReaderFuzzTest.FILTER_HEADER_BYTES) + .putInt(0).put((byte) i).putLong(3L); + byte[] file = Files.readAllBytes(seeds.get(i)); + byte[] seed = Arrays.copyOf(header.array(), header.capacity() + file.length); + System.arraycopy(file, 0, seed, header.capacity(), file.length); + Files.write(FILTER_OUT.resolve(seeds.get(i).getFileName()), seed); + } + } + + /// Mirrors every scan seed's footer, layout and dtype blobs into the metadata-blob target's + /// corpus, in the flat form it fuzzes. + @AfterAll + static void writeBlobSeeds() throws IOException { + Files.createDirectories(BLOBS_OUT); + for (Path seed : scanSeeds()) { + Files.write(BLOBS_OUT.resolve(seed.getFileName()), MetadataBlobs.extract(Files.readAllBytes(seed))); + } + } + + private static List scanSeeds() throws IOException { + try (Stream files = Files.list(OUT)) { + return files.filter(f -> f.getFileName().toString().startsWith("seed-")).sorted().toList(); + } + } + + @ParameterizedTest(name = "{0}") + @MethodSource("seeds") + void writeSeed(String name, DType dtype, Object values, EncodingId[] candidates) throws IOException { + // Given + Path file = OUT.resolve("seed-" + name); + DType.Struct schema = new DType.Struct(List.of(VALUE), List.of(dtype.asNullable()), false); + WriteOptions options = WriteOptions.defaults().withoutEditions() + .withColumnEncoding(VALUE, ColumnEncoding.candidates(candidates)); + + // When + try (var ch = FileChannel.open(file, StandardOpenOption.CREATE, StandardOpenOption.WRITE, + StandardOpenOption.TRUNCATE_EXISTING); + var writer = VortexWriter.create(ch, schema, options)) { + writer.writeChunk(Map.of(VALUE, values)); + } + + // Then + String result = new String(Files.readAllBytes(file), StandardCharsets.ISO_8859_1); + assertTrue(result.contains(candidates[0].toString()), candidates[0] + " lost to the canonical encoding"); + } + + private static Arguments seed(DType dtype, LongFunction value, EncodingId... candidates) { + Object[] values = switch (dtype) { + case DType.Primitive p when p.equals(DType.I32) -> new Integer[ROWS]; + case DType.Primitive p when p.equals(DType.I64) -> new Long[ROWS]; + case DType.Primitive p when p.equals(DType.F64) -> new Double[ROWS]; + default -> new String[ROWS]; + }; + for (int i = 0; i < ROWS; i++) { + values[i] = i % 13 == 0 ? null : value.apply(i); + } + String name = candidates[0].toString().replace('.', '-'); + return Arguments.of(name, dtype, values, candidates); + } +} diff --git a/fuzz/src/test/java/io/github/dfa1/vortex/fuzz/VortexReaderFuzzTest.java b/fuzz/src/test/java/io/github/dfa1/vortex/fuzz/VortexReaderFuzzTest.java index d93aa7a2f..5ae339fd5 100644 --- a/fuzz/src/test/java/io/github/dfa1/vortex/fuzz/VortexReaderFuzzTest.java +++ b/fuzz/src/test/java/io/github/dfa1/vortex/fuzz/VortexReaderFuzzTest.java @@ -1,32 +1,46 @@ package io.github.dfa1.vortex.fuzz; import java.io.IOException; +import java.lang.foreign.Arena; +import java.nio.ByteBuffer; import java.nio.file.Files; import java.nio.file.Path; +import java.util.Arrays; import com.code_intelligence.jazzer.junit.FuzzTest; import io.github.dfa1.vortex.core.error.VortexException; +import io.github.dfa1.vortex.core.model.ColumnName; +import io.github.dfa1.vortex.core.model.DType; +import io.github.dfa1.vortex.reader.Chunk; +import io.github.dfa1.vortex.reader.RowFilter; +import io.github.dfa1.vortex.reader.ScanIterator; +import io.github.dfa1.vortex.reader.ScanOptions; import io.github.dfa1.vortex.reader.VortexReader; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Tag; import org.junit.jupiter.api.io.TempDir; -/// Coverage-guided fuzzing of the full-file open path (ADR 0020). The reader's security contract -/// requires every malformed input to surface as a [VortexException] — anything else escaping this -/// driver (an [Error], a raw `IndexOutOfBoundsException`, a FlatBuffer/Protobuf parser exception) -/// is a real finding, so the fuzzer's pass/fail oracle is simply "no other throwable escapes". +/// Coverage-guided fuzzing of the reader (ADR 0020). The reader's security contract requires every +/// malformed input to surface as a [VortexException] — anything else escaping a target (an [Error], +/// a raw `IndexOutOfBoundsException`, a FlatBuffer/Protobuf parser exception) is a real finding, so +/// the fuzzer's pass/fail oracle is simply "no other throwable escapes". /// /// There is no `VortexReader.open(byte[])`: the public API memory-maps a real file, so the fuzzer /// bytes are written to a temp file and fed through the actual [VortexReader#open(Path)] entry -/// point — exactly the trailer / postscript / footer / layout / dtype parse path a malformed file -/// on disk would hit. +/// point — exactly the path a malformed file on disk would hit. /// -/// Tagged `fuzz` so a routine build skips it; run with -/// `./mvnw test -pl fuzz -am -Dvortex.fuzz.excludedGroups=` -/// (add `JAZZER_FUZZ=1` to actually explore new inputs rather than replay the corpus). +/// Every target replays its saved inputs in a routine `./mvnw test`; run with `JAZZER_FUZZ=1` to +/// explore new inputs instead (see CLAUDE.md). @Tag("fuzz") class VortexReaderFuzzTest { + /// Length of the filter-choice header in front of the file in + /// [#filterMalformedFile(byte[])]'s inputs. + static final int FILTER_HEADER_BYTES = 16; + + /// Length of the blob-length header in front of [#parseMetadataBlobs(byte[])]'s inputs. + static final int BLOB_HEADER_BYTES = 4; + // @FuzzTest defaults to PER_TEST lifecycle: the test instance — and therefore this injected // temp directory — is created once and reused across every fuzzing execution. @TempDir @@ -39,9 +53,14 @@ void setUp() { fuzzFile = tempDir.resolve("fuzz.vortex"); } + /// The open path alone: trailer, postscript, footer, layout and dtype. Fastest target, so it + /// explores the file-level metadata deepest. @FuzzTest void openMalformedFile(byte[] data) throws IOException { + // Given Files.write(fuzzFile, data); + + // When / Then — anything but a VortexException escaping is a finding try (var _ = VortexReader.open(fuzzFile)) { // Successfully opened: fine. The security contract only requires that MALFORMED input // fails as VortexException, not that every fuzzer-generated byte string is rejected. @@ -49,4 +68,110 @@ void openMalformedFile(byte[] data) throws IOException { // Malformed input correctly rejected per the reader's security contract. } } + + /// Past the open into the data: file statistics, then every value of every chunk through + /// [ArrayWalker], so the encoding decoders and the `Lazy*` arrays see fuzzed input. Seeded with + /// one small file per encoding ([SeedCorpusGenerator]). + @FuzzTest + void scanMalformedFile(byte[] data) throws IOException { + // Given + Files.write(fuzzFile, data); + + // When / Then — anything but a VortexException escaping is a finding + try (VortexReader reader = VortexReader.open(fuzzFile)) { + reader.columnStats(); + scan(reader, ScanOptions.all()); + } catch (VortexException _) { + // Malformed input correctly rejected per the reader's security contract. + } + } + + /// The scan again, filtered: zone-map tables are parsed and chunks pruned against a filter the + /// fuzzer chooses. The input is a [#FILTER_HEADER_BYTES]-byte header of filter choices followed + /// by the file, rather than numbers drawn from a `FuzzedDataProvider`, which takes them from the + /// end of the input, where the file's trailer is: seeds stay valid files. + @FuzzTest + void filterMalformedFile(byte[] data) throws IOException { + if (data.length < FILTER_HEADER_BYTES) { + return; + } + // Given + ByteBuffer header = ByteBuffer.wrap(data, 0, FILTER_HEADER_BYTES); + Files.write(fuzzFile, Arrays.copyOfRange(data, FILTER_HEADER_BYTES, data.length)); + + // When / Then — anything but a VortexException escaping is a finding + try (VortexReader reader = VortexReader.open(fuzzFile)) { + if (reader.dtype() instanceof DType.Struct schema && !schema.fieldNames().isEmpty()) { + int column = Math.floorMod(header.getInt(), schema.fieldNames().size()); + RowFilter filter = filter(schema.fieldNames().get(column), schema.fieldTypes().get(column), + header.get(), header.getLong()); + scan(reader, ScanOptions.all().withFilter(filter)); + } + } catch (VortexException _) { + // Malformed input correctly rejected per the reader's security contract. + } + } + + /// `PostscriptParser.parseBlobs` reached directly: the input is the footer, layout and dtype + /// FlatBuffers themselves (behind a 4-byte header of the first two lengths), wrapped here in a + /// valid postscript and trailer, so every mutation lands in the metadata parse instead of being + /// spent breaking the file framing [#openMalformedFile(byte[])] has to get past first. Built + /// through the public API: no access seam into the reader's package. + @FuzzTest + void parseMetadataBlobs(byte[] data) throws IOException { + // Given + if (data.length < BLOB_HEADER_BYTES) { + return; + } + ByteBuffer header = ByteBuffer.wrap(data, 0, BLOB_HEADER_BYTES); + int footerLength = Short.toUnsignedInt(header.getShort()); + int layoutLength = Short.toUnsignedInt(header.getShort()); + int dtypeLength = data.length - BLOB_HEADER_BYTES - footerLength - layoutLength; + if (dtypeLength < 0) { + return; + } + Files.write(fuzzFile, MetadataBlobs.assemble(data, BLOB_HEADER_BYTES, footerLength, layoutLength, dtypeLength)); + + // When / Then — anything but a VortexException escaping is a finding + try (var _ = VortexReader.open(fuzzFile)) { + // Parsed: fine, as for openMalformedFile. + } catch (VortexException _) { + // Malformed input correctly rejected per the reader's security contract. + } + } + + private static void scan(VortexReader reader, ScanOptions options) { + try (ScanIterator chunks = reader.scan(options)) { + while (chunks.hasNext()) { + try (Chunk chunk = chunks.next(); Arena arena = Arena.ofConfined()) { + ArrayWalker.walk(chunk, arena); + } + } + } + } + + /// A predicate on `column` with a value of the type its dtype compares against: a mismatched + /// value is a caller error, not malformed input, so it would only produce false findings. + private static RowFilter filter(ColumnName column, DType dtype, byte kind, long bits) { + Comparable value = switch (dtype) { + case DType.Primitive p when p.ptype().isFloating() -> Double.longBitsToDouble(bits); + case DType.Primitive _ -> bits; + case DType.Utf8 _ -> Long.toString(bits); + case DType.Bool _ -> (bits & 1) != 0; + default -> null; + }; + if (value == null) { + return (kind & 1) == 0 ? RowFilter.isNull(column) : RowFilter.isNotNull(column); + } + return switch (Math.floorMod(kind, 8)) { + case 0 -> RowFilter.eq(column, value); + case 1 -> RowFilter.neq(column, value); + case 2 -> RowFilter.gt(column, value); + case 3 -> RowFilter.gte(column, value); + case 4 -> RowFilter.lt(column, value); + case 5 -> RowFilter.lte(column, value); + case 6 -> RowFilter.isNull(column); + default -> RowFilter.isNotNull(column); + }; + } } diff --git a/fuzz/src/test/resources/io/github/dfa1/vortex/fuzz/VortexReaderFuzzTestInputs/filterMalformedFile/seed-booleans b/fuzz/src/test/resources/io/github/dfa1/vortex/fuzz/VortexReaderFuzzTestInputs/filterMalformedFile/seed-booleans new file mode 100644 index 0000000000000000000000000000000000000000..6486211d6066ea185c58dc998e3f21425841e760 GIT binary patch literal 2980 zcmbtWKZqPv82{$7*_&)OmkX{T$`Q8+;ff<%B`Ja&A;=XWqzIBivNyZ)Hjd8Ct~0Zn zbE_0rh=qlSSOz&n#4-en6e5R+99LUdNeB`wEu-u2_x@zxWjBezZ}vCuz3=bl#_}}YZ#vg zmVi@uCUjV;@Dd^lpG!I8aC8`*?_u;V0$B%s!;-9wG8LA~iXR`ZZk*QC1N@PueH>Z4(3MSGSOJo&)p! z)cW5*-(SDmE>H0P)A}K)S-!|SNDIiJimd9$tsx6qSd}fbeIN!*{wpuc|Nq3L-FmM- z5I^hDH+j$3(f7~%=NMDYy9M;^lO83r|bW@&C*`W}K_Zy;eiK{3_LPKbSqhwP~P) z2vx7)>Br+SD|!?d_VQ9~ox`%0f{&Z*!S)WoZ%_=Z1Is`i_!B|z0p9`JKn$D*mH;O9 zD;(Sh?f|!e4d4>63{-#zP}>D|fNdZK)_^8Z0q#x5H7d$j_AQ@#(#AVVZjSVxd}L)O zCbD6ax07s`s9WIG@XeuZZ?2JpcXnDwLdCV3$-o_$bPYU{3cl6R!`)im2vMoSyZYr*;H=2)Nm%Bszaxj z6=jr_a@NW`+nx6YnJ(bP>3NK+-Rq@t!^*t+Y5vgidGFYGqO_lLIpi}d^Xdbc$hVfy z{n?F*GL14-oXb;gh@WQ4y1g8-->r=6JDC@&w2Z(xc*nkSK&PFQ{iwqvaU^k182St6 z8^0%aZXI`%xkVQBJ<2n26<7hxeOSXCTgUu)eb0b>XdQ*#@&24WLX70=d*I21U4!L= Z^H!nM@Z^oBD^Fqx+r;^X7v6ApGnK%H83%XtR<=Z!AB5y|_COOgM&)Gi7S z+lvT$6T3JHq9PiiBL-q37Gfg~;vyd6BLNa35fUQ_k|G(BBLz|-6;dM&e32IEkRBQE z4KgAVG9wGJA{(;fTjW4aC$Jh_(nqJG4g!1S15Y=!j0}j4tSkZs?94=!stFjXvm$e&~+@7>Gd_j3F3`VHl1P z7>Q9BjWHODaTt#Yn21T3j47CkX_$@~n2A}KjX9W$d6pfzIEhm@jWall zb2yI+xQI)*j4QZ`Yq*XZxQSc1jXSuDd$^AW_yrH~2#@guPw^|B;W>W83%tbdc!k$^ zgFo;Vf8rh9;{!h86F%b$zQXP_!XZ2&AR;0mGNK?Vq9HnBASPlVHsT;I;vqf~AR!VV zF_IuDk|8-#ASF^EHPXNrX^{@;kpbTzBQhZ~vLGw6Av?ZB4&+2Ge23h~gS^Ox{3w8e zD1^c&f}$vf;wXWVD237}gR=M@5e=zw5^AQTcFP2#c`vcx3ahaOYq1XNu>l*g37fG6Td@t>u>(7?3%jugd$AAu zaR3K#2#0Y5M{x|taRMiC3a4=fXK@baaRC=`372sNS8)y3aRWDT3%79xcX1E*@c_Tz zAs*o|p5Q5d#WOs|Z+L;1_#LnC8t#ao7b2_C)R<}$s!3~DCT&szW@-!7B~UN4Q-jrxY8SP;>e5#) z3{VHF_I=Tgk=}T|AL>O0Y#hu#0$VfVdu*!`{@cAsm9-QU___qAaUyN|WQ?qBV&`&K*be$@`U zPqoADPs1K|KWc~FhuUHHpLW=NryX{`X~!R`cjB><<}Yfv@ZL)zs!`PFs@sdJIf0s3 zO{S())2MFm8_k*3Y-$emJ2kKB_6ln*rj}I8sO8ijRJT`Ea}Bk&T34;FHd5VQbImQ) z)@oa|y&9sry{?*jsJ+#G>Oggf>h?xy9;;4JC#%!cnX22HuX&NWR9&I2R@bR+Z;R&b z>MnJ!dO$s_y1kQ{!sOX^kihU)h2Y5qlhtp2M0rv9$Fy+1X7P(QoN^^HG*8d-IF zF*V0gf zZm)^vW@-yHKn+yescx^M<}PY?wU^pg9iY0sVVXy(W7P5LBz3Cl_GW9Ir!G{NsLRz= zs@vP3d9%7r-Kp+T_p5I2sOA&uY4x0XQN5zNy<3{^st?ph>QnW(>h@l1eyhG$KdE2U z@DbgHw^236P-Cm{)P!mh)orKJ?5n0%GpbqC?5f+#tvR1sP%WYsS4*jGue|1pYGt*W zT2rl~y1fRP8>>xKf3=m`Ms<4~G>594)oyA}wU6rd25BCuj!;Ld^`IY*I z`cD0*eo@_C#7N$kqNvf;SZZ7~f$H{>X-=u8QPZj4sF_u_mqYV+YF@Q~T39Wny1g=* z%c(!8mDH+g4b|<{)m&d~r247N)t0K;Ypc1v8lrYmyQ)1@x7SbeKy`>ZTpgv3Ro&iX z&C}GG>Kt{xx=3|ScG*Qp!TE$VjF?d{ckKs~G;Q%|aARJV6Y^HsIi?%OZ^{P_8Q znOHt=csy#4Ro~)Iyu*8ZfIVjYgwL?YtzTjPW8WUThDQY03~(fsq)6(HMiV7>DtgfQgud$(Vwvn1<Q~(IEVANfQz_<%eaE8xQ6Svft$F6+qi?fxQF|AfM4(skMI~z@D#t| z8J^=eyueHRj#qe%H~0f@@h9HlJwD(gKH)RI;46H1oD~k?Vdn%95ear)5Cu^Y4bc$; zc76~Gv0>*3aS;!8o{#_uVdn~okpyzOZwLbVv_7fA|I&VdoH; zkp*@hkqy~l=Mp)P6LvoF9daWN@**GXyrKXK!p%yv7^+fw%Y*@9-WU@DZQz8DH=fK0JO62RqM-fQX2M$cO?v--?Fluyd}M zhy^?Eii5bYbFcVF06YImgv79Ou%t+aE_ ztjGpCKl>IrkQ2G^9qc?U5Awp!)$*eN?0l^d3d7FXilP|oysZRE!p_}FqYTR8dz6En z!&Sf!u=BVdQ3-Y~R|QpJ=X2Fj19nbV3$fvY9hn?RwL?hTaUK9Ai&hwg~ zIsDNAEzt@AXpJ@qL|X)*9onMcP#W|eE1zf}>T*eh##Wh^V4cx>n+{PW;#Xa1|1N?%Ac!bAzf~WWu&+r_- z;RRmecf7)DyulxMi$C!W@9_a2@d=;t1z+LAc}F;eM+DgUYa~R5ox?^&G}w7;48(+; z%f?0=*!gTc#D|^JCPX46MiL~2o!cfy3fTE=Dx`*;^wI;GQiGtGa?i0d^ZcS zA{(;fTiAJTPUM1}`{qU-*!gchb zr~o@Ru81FD=f{;%1$K^H4b@@i$u&_6wNVE@!OoZK;b+)6a|1Mloi{f|6ZoMinxQ%T z(E=^e3IS-1HV8yp1fd<;qXU9r=hUI-h)(E?F6fGG=ngx-?ulOLjXvm$e&~+@7>Gd_ zj3F3`VHl1P7>Q9BjWHODaTt#Yn21T3j47CkX_$@~n2A}KjX9W$d6pfz zIEhm@jWallb2yI+xQI)*j4QZ`Yq*XZu;aFRPyI#pUdrEf&A+L?t8dgl)eoxM^U)9V zBdC$pXlhI~j_URjX-=x9P*baE)eNfJ%c}WXHJ6%4&94?x-ChaJrPc4%3hIw)71iz4 z()^S9v)WK?qBc|AUV!F6wVfKQc2v8lZm*Z-zUlyVusTc~sk*)KnkT7K)fwt+b)M?> zmS|qCu2R>k8`RCJ+uNymkGfwyq#jjIsBZ6^=8NhT^}2dXy{o#tN1C6i&()XeYxS+_ z_C9I;s)mo^y(E$vRgIy#y?B}vs!7!3YAV%Nb$b~#XHm1OIn~^1KGp3N(Og_DrIuC8 zs})tZS50$GwT@a(ZJ;(*-JZYZR%#nHNbR78s&22F=ALRFwZA$@9jdy$(VEAp6V)l| zbaj^M_7-SftS(bms%zBss@vPDd55}N-KQQ@kEm|%l;*SQ1@*FeO}(kQz5ALUs!!Bs z>I?Oi>h|7g{-}OY!$tMJMMO1<>h@x3j;kh66RXM8l&afHr}-N-vzkrKp?;^jy#ksG ztHsokY8kbh>h>yWuBz5hYpZqD`l{RW)7)HbskTT1>PZPdI)-LCFZ_o@d}w|7kQN%f3+UcIDVRo&ig z&G*z_)W_Tjysd!zYJ^@IA^y`7%O7r{NDPm9qs$5i8}@zq3XQq|pFYRzfY3~DAd ztNN|#_VQ@XuNG2^swLFYs@toe`A4;iT3xNB{-nCShMJqG&D0iZfEuW}yQmM2z0~|#eXG7#KdE0;w-+h8_ob+63^leIPfe)0z2urx zslIA@HKUqEb$dBA=T`Hn1=S*IanTwXxb%^;cV|ZZAl4 z2Q^gftaej-s&22p=0WOEb%Z)v9jCgzDVnFNv(&lj0(G(K_Eu_Mqpnvssaw?@s@vPA z`Jj44J+7Wo&#G?kvgT{*P4$j?Uwx>$y=R(VsISyN)OYGf)$N6g;e9Eh8byt+#!};| zZZEOsWNJz^jhasVMs<7HH0MyiQ}e0?)WWLUE2+7RT2B2zt)x~}-Ck|Yb=CT6Bh^oB zuDZR}n%k=F)eyCl+EsOXy*2k!2dYEV;p!;W?M=`;S)HcNROhJkRkycP^9psfx=!7w zZc*LdF3o$@1L|S*n0iumd*?M@Qm?8v)Z6Mk)$Kjj{Hyw#`n&o@{Zn;&pWUn{@JCQ1 ztI^b$s@sdNIgy%FO`)b%)2ePSljf}Iw`wjmkD6a~dqp*uP)n=ds}R@%4>h{KH9Zm5*{a)HsCkLH zTwSHERX3<^Z=2?w>K=8!dPqI0y1mny#HE9!Oimg@E%Xnv$VRiCRb)z_-qd$0ME z`c(}d%lj6Q)TpZ4i>*1Hnov!mCRbCbZZEy&jA|A&yP8wYt-8H}nv1B#)lzC%wY=)~ zDr>H$)>P}L_0$Hc+iR-XUu~tfQG?VDs@vPmHu>h?Bi-m30UcdPrO=L3`b>SH zy1hR%zf(V|U(|51y>A&&b$ih@$5P{}3Dm@DGS%&+(VR~GM$N2dQ*)?pFR$hTYGJjQ zT2d{ey1gGXS5m90HPqT_UDfS1((I=;S6iyB)wZhJ3(?$3?W*=rd#n9aw>L!daCMYA zR-K?uR^8rA&2!ZG>LPWixojjvx2W6IUFu%d?H$&9Og*WdQO~QFRJV6S^KJE> z`iuHl{Z(~)ziWP@{;7UYKdYX9bnljEWc@?FXlhI~jv8N0q`K7YdyD$Q%v4eDlfo4QkVd;2vXQje-9)YIxY z)$Lu;d|kbz-c=u{k5sq!T=Prywfa_luYOY9Uii4)mm;ZA)fj4QHJ z+F9+Uy1hP{`>TW0q3Q^AwCeUIYM!D_S7)hn)di~CTc&xXx<*~EZc?|ZZg02dedzL{y`w(bZV0+e@H1 zv6@UxsislWsctW`=4@&X^*c4MT0nJs#Wa^x%c$klAJj^!+pD3uwpv%MuQpQsRJYes zb8EG&+FlJ&JE?B3hvwdDKXsrwL>;cWy|J4AKb{=_zkO2uUwgbW_VfFjxCQ$e{vF(f z{T%;39>9K<{}7Mx7*FsN_A~uwcn{cQggUc-LA{|DZ}e#ZX}?_odZ|A?a|lYJ6zq8fWl$FOT!L~a4|_hr52y%xPC+G9hCQ#K zDypG6YM>_U`31F62lgC;x~Ka3~ftF~60JKIM1fng1 zV9!Blj}8b%2tv^jozNLw&=uX#9X-$!z0ezd&=>vC9|JHDgD@CFFciZu93wCiqc9p{ zFc#x59uqJTlQ0=mFcs4<9WyW!voITTFcQCy=YD3lSHPhTe4NwEsc51Nd_PS{9uJ%&< zssq%)s@of>d5k(5_P${N?ohEz0I1psXNs@>VEZ*>h?}(KCPZp zFREA6>#EzktNDTYNPVh4S6`}b@2%$d>L>N98a{#dEhDLJFNWsWYCJWennX>mx;D7#C7B#z?Q+0d!G#6BhsKwP%YFX9oRn%Nrt)|vg>!|fqx7S#6Q`KK>rM6LnRJRwZ zxwG0$?Wy)r`>SqmsOAysXmy-AQJtc?y;+*)steS`>N0hu>h{)a-lT3-cc{D7eX83# zqWQRbNV5U0`b2%Ey1iGL|4`qlAJs2vxP)%Q+bEi&tFhF$Y63N} z>b6sAPNSw%zfm)**;Kdpo#wo10kyDNOf9Lpy>gm=P%Eib)f#GT)$P^S+(`9Po2xC= z)~efUuQ^2Rq;^$%sJ+$y$G@tL;7?zK=`;;sPm%ZjC^&+TFE8^*+vkku zIbjie=IH-rl;E}Dc)fRy;8QY<=M6o-M==7gw|~RKqjT-?n3?I}hp+sKqo|(eJ7;FL z*V%2&`DCyE>v&M(ub1s|d%k}Lu5S$gcGu<{AMNdvdwy?Q*z0-xAJ=om{d|xe`)7Z5 zVEaLy{luQ%`&5g$f?j{dyC065@%&CrUuvJq^swWlo)>4=k@VJ^-u59T{ZgHek}fI7 z=fgfa`@DE>=aT1kBKtI9u8T@~{SnXWeLT05i`Qj<`yFGv=;Leu(P7u^%P;n^_dhOI z>?tqXZT!d6NqowA-v7k^xV?hz)9};+wi{sj~VTD-s=6mz{~dMu9{x2ucPPllb6@o zb=&7>H0;=^KWp}~oen0wb<+O1tM}_Lubj7Al6c-{yWHM)yS)~l?fv@a*Q2iI_5c0r z(UVoL4v5B|ZM z`pYLF|Kg)By ziG3dMav++(d%c;t=ZL~JMdj~FH10EEaGk#HdOcsi?7Xcn?C((p*l}C;A$HwKKaI&` zA6wFONj#s^qv{cu|GJgnI^5y;{P7PdJ%4|=Vo$lyZo@7M zdyMj2??1n8ecbK+m#>@mKGJ(lp0C@fQ&Bv(%Rf@I+w+Cpp85L`o6lKfK8rE=TqgAV z9%24(gy;9vw&x9-!~6Np{(t{`|Cjg6H_VNB-k(LOynl`^#roU(73R%6FZ%Meu#e+{ z_v4OYKfeCA?-lm3+22!lx&7&i%FN+;N`_6|KXdk~C_Jq=w&zb^*tfC2mr;4G{tkt= z>$H#AZr`pqsAFKDU(-PU#-02E89ncJ0H2t@?l)uex%1xt+Lrxj$o4Mj3f<@2-o|^_ z?M3w5p52Dso_((EHtloo>v^BOf4V)d;_H&w^Fp|A*{Lt=y6ktzy2Eo{wB`IA`y3nl z-eA|8G;(6D*FFdTetjA2ZD8-OJM4Y(9lW8T=j+A&uw9QY?C+8Hh|cZXckP&-@74A` z+2=DMi}tbFeroPtt@drOQ@h|$|1LRN1p0+~e#f-^n_l~O#{}di z*zZPYj)F)Ddq=Kt1NJ-X=AaLnz#|OM!A%2#ItBQ5=5^7zPqpg|v+KQ_*Jb?YWg(%# z9h-%6S?Pbdti4}wh(DJ#(q(pcrP~bg@6gddsF}Y{w}0*hcMS3mvX6C=F0(%$+G`&i z&^916pp#v#&n8`NkFnh4&D!z*XY~3opBMXM@AL4Vx7F4!q>aDbR(S54?c)yfJ_ZJa zgnB+JsdZVH>$A&z^8fSoEkbyG6}{g6oa_B+5zxiIdE=0PZvKrut3JVcn=tRYe-}QH z!~S`l%>(!h&(rIZ{dJwq{hI}}^$Tp=v}>q;V?NiRAzXG$Z)bmRb=!J79yjVT`?K#Z z^9%J44QT7{)_tOLAN3!fhi2`9LPGt5LVYsovM|@(tX0P#&nK|-KlkkYYTB+{pid)R z7UuKkd3`6n-u}Mo{qYNIA8cO^pK-b@%;$&8_T5E0M}(QH9LL^LFz1(kMPS4GxE*o8=&W_{%Q zfgyt?CcerPOEWVktu+-NnPFPK%BZZ&tfA5#PDS6}y}P-g)AV06O+V(d=lwgsbI$ji z3uBD&fXs`20+yecJ?6!WyyMihx$k~^OC~VCr*ivuldlZaE;wC2M4_(rTa>qRoU*R9 zY{{ASS*qYJOC9-%jREyzn**H{`-Kp-b7hFORo#%+R(DR_9;~Tj_4?f`WX&0E!-XFb zG+l#J*Ax#N%hu)_*Iu$$bZ7Om*Nxw^GOUMl!}^bI?$UKRsLQ&CbL55|Nz?Z@I^5Wc zOD~P`I#Lnio%wh_pQDaM->gOV`5oty{j=9R5cu{6PRMy7Bj}x7BRRHj^u6yM92=T@ zyrA3Z(?vbFsojhRz^BG~*_4b)M zd&HEggHs;!<`*t_Ds5}Svbo2euRnBk?>gptY)g8~`5oT6t9y=={(L~LWG$K7G^ZAv zP_!PsmV8;^=gYGChF)DXDyIE-+2Eg*OGh$YbCF`n$x^|u?QspO(T2^NxuIks3tzwV z$xR>EIwG~J=4>&no)^8LalYe|riFdNH!qISZCx6>sd@SKh@G`@k?iRmlU`dDzhysr z>2v4mgy_T1?3#S6G3oh}&+h)RwTbmPyK&Fd^P2}bK74NPS8ZF9W3O!6H;px?Y`^~E z>tBDzhQ{4|`GDn@SBCBQ?bSEG32~hFsK8h9^ZuevPcT)2(@DQQ-i zX87FjrHjB2HVvh&EJ@v|&Y~zhf{2bYp zyp6%v3*U;oQT*w^UrShq%oI_en&M>$E;ChiFQ4blW^Ark*$;1#)m+)r{fT6Q%0K0k z9RHycmyI6w**nv95M&wzna-~`l?g#ICIn_fkU4V4Lt`MwT!q?^2SMg5>x4oGQV&5E zFUwCH4?&i!c)!8`K{7{S(j*9Sa*(rfDg-&LUejcOAgv7-f@VOFl@O%8>)QNk2y(HO z+gIDi`}J)Jz6X@a{%@XuATP4uL)Re4TY9d=4}$Dg8pZX6AcsJZqh=@ij)Nc#D+l|} zf*@5*sa+Z&$Yw6AP0;FV*pt$7z^^i!yq3NfI{#H_}~7`9S}H=*;U9Zn2O2pt3aME ziwt-H@w4e8X8!zu`iaE=CWtAGQn3K!sZ8U~iuKY5ca^JN#e$h!>?xo(8PBx%r7<)Y z#Y50Xn&zhmK@OBMMS%tlD*RDdij-2>pW|puTSrF+MNGcQ9~s4> ze-3=m8l?5Q&q0ch+*6!ncTWSvotE}V5YL4%Z*fHwxBKAfY@2|4t{Utg#o^vN@=a$z zWxcq!axqU!^ahqrb<}VF-}q?k)Su?x*{P1^q;n>dR3)8)XH0#>HpZ@Z8cT8Asgc_; zGn}C0nY_DAI#+jTA)Exe3rvYUlcG?f?j(o<6pMS`k!3~LR5~XIZ7ysLd{CM8$N>yM z2Ohemvxh8&Qxz#)gMRv;7M)%9rHmTHPXJ5GL{cqRX?3Gn$K`ht@ z@<;d2HO4fx%34Mq!(441{w7} zNnD_Kqh;cLQ`{agO*h)49#nq}evtUko_oG?o^KCnois%KDXj+7*9X6@qxF);D=;6` zlF2!@UQ|qjdiM;}#*23@ha9?${7Nqi^%X?_G=gTO`dr}dPoH8=;n1lCrP6f<(~UK1 zUcq_Rlv>4_AY~ULaaD zdhexm%S64=D9?TRmH)tdchP9(?K>C&RQNfGHui{J&P1uh0Agu4g^t literal 0 HcmV?d00001 diff --git a/fuzz/src/test/resources/io/github/dfa1/vortex/fuzz/VortexReaderFuzzTestInputs/filterMalformedFile/seed-fastlanes-delta b/fuzz/src/test/resources/io/github/dfa1/vortex/fuzz/VortexReaderFuzzTestInputs/filterMalformedFile/seed-fastlanes-delta new file mode 100644 index 0000000000000000000000000000000000000000..6f4dda829582b360a6654ba27f178944718a118d GIT binary patch literal 18473 zcmeHPQD{_06rGK0vfZwBn+C~;NDwJfLMT>5ij^QDR)UnGlKNq@CM%8IY|`u|gwT&g zL`0;BA7YCiK}xV7L8Kp2f&>X7rHF_hB1J?*ibx4Co1J}cD(=X5 z-u5-5IRS~*S4K#f{bUp3zVe@^PVwtsuKpa>|MLp zc`EOfmXqMY0^(n;{d*YL@mM=tt{uN|c?9Sh;_uhrwtffp-4ym+SpF;r|EvZNl#9Co zyc+`#lxxRt+zfbN>vXXD=P2_3g-2#MXL0X1iM;1GzlG|B2QGt$zYLg%4R6OZ-8$8fucn^639@rf4{p%v`U*99| z$vi48wzwdXbdF?e8G zKs@Y)TWyJd#?$15JvK8^!%s6srvFW~%h3i)T$F#pK>OO;*VpWEP{QpdUb z9R%-Q4EXmx4aj>f0pEXbfCsXM_upyoz;?s?uOGiB5&yLT^RS;ErRt}^13L`!j~`wE z@t-vOxpxHl=i7kuPY3qhJ=f2@KZAe#=3B0QHTY+_;XPz49GIz4-t(HzP4GY_;QQ~% z3f+G%;Qo8G%=@pKZ@u2Hg9rKo=3y`Vg|?qZ{yAotfBbMaA^yschxcON-3j>T-j&#Q zP2hodz(0)v-$NdN2igtqA#w2T`U>5@yw>*{D9*ej_upaUy(Ff(Ld5d=DAJJ>-Pz9wPHE zRbB-T3>x0Q{P6Z5@7;e%e($ppd9Sv_`!AN=nh^fA<-qmdH38Yew>^Ks7a4XOY~jj0 zZR4M@{N0b6|0FJ9Nx=^b&yBP8e-eMUUtzsV{6&U^D>$}rtN1zs!ha$qE@4SqxH7*g z{oPPqI{L8$3FpmXe;|*-i{V-2X6Dws(~N z?VbM{h9-LEe%!|0xDVI0RJonMv5QAzI~z%vcg8H9?aSY1B&b)dgOZ?UE>TBnpEZ8G zrb*S;G%ZP}2B$BN(S0Loxno1-sIlKg8{3Z+lIo#@TQiHxelnW={*PzNWlpcuGD2eS zi!)4=T9w+#U8!(fSj!lVlkYimZluP;Y`nBW!;+E4STfSIGZ|r9#cS6_?;Hd9Mv-y2 z$eO)aE_EWMEhe3{WJ^3F_FFa%+Z>@PSVI&i_CsRN7#v@IZ+Bu?HycC#h9~1{<9_2W zuyM-OpYhMR+8A?gC+M|t`?Q++FqU4|IiKcV*Ev0}E8BXu=yjDgUcUx;QE`fyJ`eou z(@#{JsWP_2pB~lMoAf$OkaXV32Co~(s6e%YYAe-MRC(-i`cR7{`!d+upHC094EJW! zUG%EPpyM+MT~~K!UR_4}lDYO=zJGu;O5KHy&n(7tem0Q(oGm_i-rl}UD(~!rhtz+P z>Il^W)efqgsji|LrTRXt4a+yEirwC9E}zQg)h_7f^QRxWqB-pC?aino&)dtTzZgho zJJV`(9(#SM{#=@{FG0uq7k>2hclUJXy9ZgV?iShIm&)Zcscbse(v{BSQ>0~fi8|}S zb1Cw-CzW$TZx_ ztkd#V{7v-94rDT)cJ>a?D(ZdQOp;x=NHJh6_JIpau9|m?{f;=Z5I?w&o}QG0{!`TNlg;!%C8FBZ Jt{v@L{sVVqh#>#~ literal 0 HcmV?d00001 diff --git a/fuzz/src/test/resources/io/github/dfa1/vortex/fuzz/VortexReaderFuzzTestInputs/filterMalformedFile/seed-fastlanes-for b/fuzz/src/test/resources/io/github/dfa1/vortex/fuzz/VortexReaderFuzzTestInputs/filterMalformedFile/seed-fastlanes-for new file mode 100644 index 0000000000000000000000000000000000000000..81a5ab6e23e6e3cdf305d7c69a3e547b542f7d77 GIT binary patch literal 10604 zcmbuF&2Lmy7{<>Gg-*r+Mdk5?|%LI-kVGaA@oSo zX;F;q|L4~}nB`jJI^=rf3&;(~jmQ^~FCjM}HzT(oiFyUgTanw4uOeSVZb$Aw4j^|T zcOeInuOrEM6U)1iZz11C?m-SAE6BabVdMz1iX26fvk%KP&N!AQknbTUk?$j?kcW^TAU{MNMjk;9-ml0>^}~C= zA}7@k@BNCLR6o4;D{@l(;P;2+2RUB-@ciKShvf%3Uj6X=;P;2+2RUB-@ciKShvf%3 zUj6X=n8tgE;T8| zgB-7Zcz#sCvHT#%s~?^p4R9WAmY3^)sNlaSbmV>)ep~)8aS38;lK~gPc@9M!>QBAjhj8o*xInvHT#%s~?^phrzM@Ajhj8o*$>c zvHT#%s~?^p7s0XoAjhj8mLKIDN02OIz7R_CBb0*t3*i}Q`h%4CW0XZ<=&{%-A0s@F z@X*$;I{QSI**e~3;HXTV%N-rIs4ZZbOrI9r$w~wbK ztFz1XsHCz6vs>Gn&Ffh%mR4V|PW%1zA7jid=&$@A7Od0bzkAc;?OAwCPe~tT-CD4( z9+&PLuNiOc-DB2Ut7H9pino^QW@L~RTnI(EX7SN4%DUgBJuOYyyxuY!{&*bsX{9cO zFehX1zUg(EmNq19jWq3|`(OBbeqK|LXQJ8aR3}$YH5=6tSqwj7n<|KHBemg9_zmlq z-d*4Bw8mvi2sg1!6~#Wo`^Fplb;P_}CuMbwH(IZBvS;BhS^tZ)tJ0>W4M|%sZKbpx zY0qI_{goQWZ8qASN~06rz_$Ax&&0jVz0GDloVj_0D(oBB zrq3{r<3Ou6R_oLzbaS}bweM)9-Kkd^)%NPqW=rlwLC!?)hn`Q@v5r;R+2zUB|0?^- z=3c!Y-5x4W_tR%fUC`w6gv@K-Uin|Wl4()gzaTtat5a`|&r?bM_$7joTeYK?5o!^MZtr!iiy z?-_26%N>l*@hV9@grDUaC`*q)CrxefPi#E1cr3Z>EO;E+C!S+07xBD%vcEgpzq_9- z;{2jn(b#xxb-z{8{*>d;zfrfI)ZS8RWO+z(7Rv=0LGP*dzb*ado{$-ta$zv++%dS} EAAV8W*8l(j literal 0 HcmV?d00001 diff --git a/fuzz/src/test/resources/io/github/dfa1/vortex/fuzz/VortexReaderFuzzTestInputs/filterMalformedFile/seed-fastlanes-rle b/fuzz/src/test/resources/io/github/dfa1/vortex/fuzz/VortexReaderFuzzTestInputs/filterMalformedFile/seed-fastlanes-rle new file mode 100644 index 0000000000000000000000000000000000000000..d577f19205798aaa5f0bbfe6bfc5339035fd7993 GIT binary patch literal 6696 zcmeI0%TruM6vn?hOkgq$WPpgHMny$MM2s5WCZb}Dii!#m<0DL9FqmP6Ff)kpg>Tdt z-!W=bd`DL)7p^SJg=JZ;TxRJi<-$e&0`YveyI~maxyFUArhj!fxBHwv-F^D@+@b+s zl?Y3XQSgpKN32hncJab=)W!+ZZho4swK0NpowbGOnDq(MtF2F%j$5BFU2lEDbi(>* zy209mbfdL}={43TOed{Rm~OH@VY=D+Xu8GPgY;T!E7#@EG_!t!_4_An-TpJ>$;pYx zDGDYk=Pa07)G5TUx`H?i^++f*piyBBk_t^|E~5oYtd;Br(~1)nQaDNBWSnAIhvl4# z(-cleo5C45Q{gPM3ugy#4$f8RzT4*38k zF^S!nQYfIPP{OprBbce+Q5g2%F@?wRgu;_}N?{gHm+=gicoxqoJdYO?Uc{WjOL*Dx z3YN1MuPW@rYYMO94TU%Hmhg4}?_j^eyEvfm9^O~@00$L5#77DrV_xACd>Y7S_#9s- ze2K4CU_Sc%J_l2S=nBl&`rmw=-PbyfdW8fUs#G&j%|JB+)eKZKP|d*qlmYio{~`O3 zbeCMLyGxFk{7W50OaOXCcqwHrogz5m-*ibY-??V7kGlXf7dnl$olz3q?S<2pDSb1BF1SeE(#trDB7GZx=05nJV9&{XV-OfFKLR*UU0g9n#XXVOa{>9q|e zJ%8E~-fd~0bm0=+J)zsWab7h(VX ze=RIfb3W7gsbXfPdrv->8I>x&wKg78Ya1OODdJb#&bw6GUMNnL!~z_(Htw~xjqEJt zb}?c^a?G*H;*Dj~MgNukL)w25nHQNA85ZdkX%~r#Y_ffM|LWrAbA@6$SHw1J3!2aL z$}xB3^I7a$8LyDpUCQJ}GWaGS-eh{Jkdd)}wl*F@JCDh!@rm)`_%zKpTGn?gT_|SL zxlEyZDw~m=h{;O0A6!q_Tqn{6KRtf?FVbJkb=(iWb?zK;Joe+e#j`-|>eKQ}-#Yor zb0ww5^pDASY-OqS-20S3O4{8Bv_cpW_y~G z@wm;&M0%z<67$Mi#&dp4OT5YY_K|!^cF?@XU7|UKA0!7Hi*sP1rkJ1M@_Bpy zD%?EiW7cRsDQ4YK|96MI%KfBj*JnW!WZ-~FcQariq+Bz__`ERNxRGPgCQIU;b06WLkKaP3P`6{oc%5yHFhRb!Oi8=FK;6W*^lZzf%8BS;FhI4e6q$~!kqniJx7?q@9loSPjrnYVdQxQV4rzo z$3Pl$apS$;0POe7_?1ax>;s#?Brt(!c_=Dk=MogXDr>=Zs~bEemzVnxa(B*#G~tijDca zA8cO6@)##`yM{UQtotTl`GVOci#ggT+xVuku*%(LI0h>bpA`3L^Jio6J#t*WYtN0$ zT?cL7k1!vcs9^5f#z>nS@Lutm;+(~A!fHqx4YB5J_;a3!%ILG;?Clv7@>({7RO$wL z-q)VRvS;5_`Gk-4Oq)2duED+Mpr2o9-e{hMGoPgdajcy+PW|flxgQ5Q%}}eIxw`LY zx{mn)lCuZ&02f#RW`I2K6XFhl_dpMDfdya^;LG}k4L$<9z;j>|SO#W*H1Gp%pMX7} z2e`l@Pyy1w{;;lYq+L1``vv%j-mbURb+#}+7Vgr3>o{T63p$>93|sEEJ!@jE<-tu4 zbCC|)H4Sd*46fybkpj0a+(nE~k9>IQB?EG#; zSKZrD?ndNw)rMJ0q4*@Us~sfr#c)j5Lx$gq{puj5t7=}u@i(eZwAw(fHGh|1#E)x{ z>@9voN5b*jk2yyx?KM=qoN}l!{I(K#s5OH~JAtNY;S#ZH+wH(4aA|lQ>#8=JKCKIv z$e*!y#LnNd=yCj3$k#!ygiGWH9ML7C@d`NtfrZ%9Xr4%0em^2)FT*C1TI33J}&)5{U!AY+9@z+=EPear8IaVb_TBu{d@-<-#); z-d2(boi^!|&jGfa`&-B14hWOL1VH`jpKcuXqU!@i!0LMmZ5D)9l~0-)6G$H)$}u3U zIo|`(Go((V?dq9(jvk0fvHCc#j}Pkth*G0XK7Dt~No)?T?G+F&1LpxBV7^t6#ly&_ zEWFu{NbEpn`Kkhr6c0sWP(|0*H2K=pBMBd013b zBSI`mj?!l4rZ2ReSpOH>Jr3FD<%vv#kCP`tk~}eojBQ(6x$kYcF8NrJ91TG{1#n#} z&rF^mZZ>8`I|#Ln8`rDs>D(`4j4qz+g|h9g^R%(&n4V;p%wLlIIStYzK)r=cdUu0n zb)jj_Dz;2cVq*Fux?eVJ>P4G6Ij1d`jLt<6<^bmGC{R|rJFCD*GJA^P7`>7^70C7T zpL^bS{A4_QfVz3k+A;ze=57cA^_~D|+eM)((ZM*=UssPk59p)lc}@E$ZTRTwW80Q< zMqdhoJ%6;Xm5#6OPH}y8&jmZ5%^BJZ-P=q(=<27>-7&N&owzFf^speu-i-Sb>4;}3EslVPzl3*r@rs6{f(UrBu7+r(JHIGf7 zBJ8(Jzh(Oe#v~V7huz#6kvEd|H6v#;_1S!~IZT?4K}KR=?XE)bS0vaq;53i{zQLdn zc!CkklWBnRKS+*w-ZA07J?uDVx0tm>U$r6srmUOi9YOnl&I`%sza8E+c2*$6 zd@ca4eQ>ewTi`2W$2j<#?EtA$9Djr+eT+qUyb6ljU&qZ^LFNsaQ$bHsL^W#e^8S8n zIsM(N&)$F3W!v1WV=w@odOTmO-+Bi96~7S(=Id5>o8GIQ#~5riPyc(WgneQCwb~1~ zDOfzu$2Sky(*M3GDZeuZSC6IGnS=iMfaRbmA0DT|1cHz4t%~{##$H zwyquh=Hb5UcXo5db;-eQ{*uYd`<5~0CMm)IERF25U3hEK9>!*Ap^-+*`(~3MRd1kw z7YeQcYd{F_cS8lp1COx4cY&{gH6R2I1ET;Bj{8t>3%Cljfob3vPysT)FA%#8Tm#mC z5Euu1AOqa&wQHrBhU%W7^Q~>{U5U@G1ZUBIXn3!5c}qcKCay2V(MO==@LuEG&RER{ zZBH++nKoKAp2tV~@D_tcGXn37;o&)9?P*3IwxW71QY*dV8m)R%r>~zG9#`7` z<1}94wCbke^LL5m*A|qzXXws)@witHdUeeQ%?~2#8o(Jvza1NsxS2{+_?9HknU6d* z)=Qslf_~i4xyNk2#LH1Q)r?o7DH*C7M#ix{T1Fs0?VU4>5#cWk{q^pgA#S(%Aeox^ zIE|(d>$C|T?*!V)-vmatk;88(!(;ASUXVsQ?}t2-`?k~9F9Z;|dj{OX3eXX11*sL@GmNjV!(sh{x9!xrzn zrSmuq^(j)2bBuf91iPY_+n6}p`LUhD%p1q; bhtPdqLQ-1C6h4_|k}-{Os$m%IFlJFo7DJgrkutcDQly}Wiz2g-W?u4cIuGBxr0>0? z)2upW79!%pf-AwQMMMQ%b>&iANa@;zDuhhEFzk)@ZjW!>9`AiS-uryK_x)d=eRY#I$~;g9mI3v>-it#Y z9SChi#o}4K7XiwGRHcX~bDjf=0DUQpB{HXR+2Z)QT)%++y?-VXk(cDRIgyL#zY44X z%lbq^WnM~BMvg@r)9FRNoHn(EFtUrl>@rqhU=4A|uo8Od6QMowoTkv5{)_xhRO3F_acq~FoDHQ0#7%txIxmrS; z{~q7KSa%1ZsdCgl$@T`#75MX}s_H$o1`X#}2XS&B|JA=I6TXiF!(NQioau&MDlemd zAM1A)*av*z63_&yz@J#O`@na=KHvlAffaxY`YRIr2;2c~0$adEpa~R#Kj8Kga2MDI zeBd>p0Th9Iv%E%0>dOPG=h|Anqu}buyNdoJYdbNu4ZXM>M8m*bhpvKqm$7qWGdpyr zXKhIu4?5g47ml#?y*M$jU9q<3(6)RMb8TRvj*-#qxOfnmh;#kK+PI!}Tt5zWf;1R1 ztbQZNmER9%@AO0-*m1e_;ue>Ov$=M>Vr(w0fR*%f}1*F_%L=wzgb-U=#Vq>bXC=UXq4hWRf#+$PMxHk+yCxhV6H2 zpuOFf&hEk|6L zRalj|jAQ1Bt23XGnkuzez}hTi9Tss1)@41`=Z=i*tkQrDxeFVyF`IB#Hf1w5XA4GJ zsbWgV;LEzGM*DSk&{@$$(+KeEafyF%E%0r!#I<( zIGbfG=N!)EJkDnYBMVgy=OP}##azN8c@&T4Fo!0Wai5yqJ+oRW9S@ynK|aKX`3N87V_eV2xq&gyM)8x}#HaW)pW(B7j?eQ2 zzQ~st^SmN{m9O!2zQH&77T@MOe3$QWGh?0)#2@k_e#|ZWgrD*=e$FrWCBI_K^Nsjh ze#h_m1ApXC{F(paFZ`9iG3NPQ{0IN!*4QsQ(=&_Nti&8vW))Ut%u`LA$Lh>y4c25W z7O*x8S%*c8dFqPmu|9X?PTZLd*pR!h5gW4!W1gnsW^B$D+>I^Simkaj+psO$G3IG6 z?!b=RlbyI1JF^S-W>?Gcq4cL&oun`-x33p{vHe++fJiCcovK3o%ceY_$w&Nab z&kpR!n5UC?FLq`Z?#-_3#_rsQJ=l}I81w8a-j99Qm;19H`*Q#f;6M)IV8%QLi4W!w z4&^Wo=LnAED2`?^$1vtOL_Ch;Ie`;7i6xxODV)kuPGih7T|9$_aVBSRHp^JfIh@OR zoX?nNfp{Sg=OP}##azN8c@&T4FC%tsjWJSJYx$GL$|a3i1OCO*Zd`3#?B%=5hX z1-{6a_%dJNt9*^G^9{bqw;1!hBYv0faWmiN2mFv9@nde`Cye7qMIf#RK zAP-_>h{{k7<8Y4PNRHxY7IO^8@(@PGt4!cTPGSisa|#RM-!p|cF4o*sekv!GoBWqb zg%=byNoK=TN=>qY;zr7Bgqp4mSLLT_8Y^7?ZBoJ!`Y%gL7;E{~t;xS{RZ2BZ>LF)1 z7v>R4=ofN^^IZd6pEP8ysr*;p&@VsfRV6*k{G=X^oU~!QaD@DM*^N_$*^O)D zr;5^j!!?<{S*ftOLOu2m`NO#o=hiI952I#22;r{6k^NWxRH|+=Vwh#7CEc8~h7BBF zX2@TZl<@KBu%u{p|M!1u+B?RH@mLfR|MT(JWd&Jr3#AIO8iz8nAS5%W!q^wBF;G~4OQ}Tkt+(JDZVf^r= z3UN{Tu#VOz<%*;%P0FOC^i4{$q~s)}r)$HnYaF+{Y<@*aSw(8Bwtt$>!oTO7 zTwXpab;jSvn_oJ2L222P($uPd7;jF=y!oX`?hV?)UNDb2^JdJRQ88m-Xija>9^RAo zl38;?OZZ&nq~AxlpKazkyJUX)@ucr_V{(1cTs6FpZTEM`7p@CyE9|FWj29;L(6?&x zXV{;~Q7LnMZjvvY4M$#9Tu=GQr>;u+jc3+%SVdv{!C^p07(blLPqKxbA9`liY)*Pz zPEXpl?H_*SvT~R9E-c8)bKkQ|78PdaW^H?BMp^nyVP1hgWhG@3r<5-!t0=6KS390t zFl$!kWHM%0-))i-ruJa+Y{DGE+`Wu8Y)`nwkT zZ`&^k-FIfkWp;ArnTPAzBxQ3lkCw@x!&-*k$q!u;PfGmW|2pNSpI6wgNn6q{yG2q- Oj;vI1YSf70e*Xbrb;B$G literal 0 HcmV?d00001 diff --git a/fuzz/src/test/resources/io/github/dfa1/vortex/fuzz/VortexReaderFuzzTestInputs/filterMalformedFile/seed-vortex-alprd b/fuzz/src/test/resources/io/github/dfa1/vortex/fuzz/VortexReaderFuzzTestInputs/filterMalformedFile/seed-vortex-alprd new file mode 100644 index 0000000000000000000000000000000000000000..78b83648de5f16aaddf7ebe41a7754e0ae09a912 GIT binary patch literal 16202 zcmeHuc|29$`u{ouj$=Az8Oof5%yT)$gv`m9Su)R|J~`%jibRHpR7l1MNl1nU64EFs zGEaqw@80L=e)`>eU-!@Zy071>bzaY2Ydz~(&-1MJn)Y6M9{>QrLIejCMCuG6a}QP( zv?1|mL<}WBGy#Akr~*Jx&qe|^0uuCt2nRfiAp&0q!ZG1wAReF?cnECthG`E0WjHXy z6lyEn;zkhY0VP76fFQ6DB2aC#ssu_wEI`&^0D^#EC?q6uBN*5s5NT~_zzbO7G59J1 zlJ^=O5AaSXK+nm>O+d60R1u|kniwN83^zg#6jIuuGypdnNQ-LIMdLAmQb>pl0m4O( zp%Hc{paHVEvG{%sHIT%u4=`kuJQ!MnHvMTTZi4nJJb}L(O~3>0C_o?sXl-f)8gGXx zQRO#}Rz>tWpZa8<5orLAvJ(&6yQeV&Zzpyn?$;zg)Yqsi0k9^y3s0gXeD z^P@o(9w5>nIJrdu8;@O7367tkB!WK+5O`$(BX1f4?gs?1ClCM)rQjTVMM763AcsmQ zU=bpZnD|Ey?mZHpRFhN#u7%o_{-up*geVP{kaB3q8j{E(nu1+`6~I3-h`n1x68QiB zTE&bIa|m90D{!Q)nL&lF9RHx&nhizu9)-7xFIXMaFp_0IT<;= zNu~7Wy6V}0XbEaU#r;!snp+fcG$ApcWaReny3>fnxl@juT~O+8pZ8T!+ByA_grQyi zqV_G9j%sG zl#?Kda&Fk?3zQ)uw2` zc92r>uw1f!R0eO=l9DzghqwRQuy~;!sY^I*c*)`BAT3-sz)`L>pT&szTV6Ew=I#2c zn++Onn+()r7D%-6-2mQu#iwyy{Y&+uGZq;RAh4k-;IvNZ-#iomzJMre&oqd3Kk%bQ&^w z4*QR|$L1hn#rwD)grhP0180uk9#rgN zUb~r{_ZbvR%LP}`V{ajQI?l-UMSkg`iEWPAt8uTZl72~xv445Yu#J*@){rmtk}nxU zp}p(w+G-|ltX$*yo+qFu==D%eg9D&A(yL)R#?NEIMaXrSqIj0U`0n9kIr^NZK1S7v zcYON@U%qq0_O@u_2p#WyMuC5uV5*~|)i}q|q&Uf5{b_^gNx=hu5y2stt-niO(MhFH zV!Bj)OQ|1PI?i;k`Fuyy54+b!#9e1li6P2Cmw}F(2edA2evtLfukzA3CCGc+=*C{p zmMfIW%kC)Ekt$}8QE>~M}xs#T1U`f{FxotJl{ zyitlTa~p2OuyZDKhcRcA#GRW8DuuXja@Uz!j=} zvN~C>%Qs)8am4XDQP(&HdCmL?$xfVjGORoh%v@xbuGu`shjSErpmI+^+dN;6;Xb64 zP1rwdIDay%WPYyFd&I_vXvZl!zB^u^J}Q4S*R;ayHWR>ovu0!}Kh=Uskv{tE3&T|^ zq8O<0J-J+RRav$XsqgD!O`B1t@7N;s!1v>4+AS%XK-0wT;PvjXyK2r9&ks1rr+TaE zcogk5ROq3OW3yfSdEbeQW)Mf|HvQtEI*W~u%D&rAfn6MXEsu-oac%jciG60y8`LB_ zt*1JfRGsIq`zJ@x2dfI7Dkh_Fbh4cwi|Z9@zjKx1U@vMj$c^@NyicHg)A@nE+^j0z znMw^&JgN3#E{kh7<}6ywgd#z2%p(K~q?;2o^Enx^VW669*U8DL*f!A_ZXMCf~m+G+dZvKG>v&?-q6d!SSTUFcX92ijkYIs|I z{Nmknmnx-|dqm#1`5BnDJYD_{_CK2|bf^U-4JraD-Fl^+nQt&^r;Zc-@s^GUu`33a zcDgur-j2>%h#6hJnvvc=@BRHs#Hf&S$jFdNv1O>W6U|I`Pz&+_!+C-AMB zDpXadvts+p?bLJa{gNHYCv3WW<_?KekHh}l+{=m#VpKfLEB01+90)$=J=;(3J7!6&VjiukDJohnW-Q(7~ zQMlIhYU^T1^*Q$uZ7y*2%By3voNCU8LmQIrUa;cMFhiTA1+MiE_*1jHp@n2TEVr0u zTvTp+UwRKfo&P>f#gc3RKfbb+H6-g%k{#R1w`rHcwiyJ&oPJKEmV_0$ej!L>rOMlT zaO+~H1m@s51ci1?JkP_~iq>uv+V!ID0ymgr#h9F#M{%OWpHl?+(@44iR+WPGDye~k zA^npFuG+bs;KMVmHR*}A9xGa`AE_K(n#?<0zxqV3Yh2nbJhH3s3lw+OI(Vf*yeW1iyqK;&pDuw7He@y)H#Y~O^4BPZ$f#3m4Do4 z7n&2DvTEVJw)y$PJi5jw{huiYt!Ud)2^ZOJ1X3e{m>9sNlo+>Y&%ymLxKmoD>(3EAbx{)k@?gT+r1`_7s|1F zwpK5N@^8sJuhu-{++mULJV|+SzMz2JJnmb|2o;;U&qMs-@xj6gUq`vAx((|-irbu@ z0&^N}mv`UYrN#VAz-~YsD~_)yaM}f66~^A{-jl7)5py#r4@_ z#SkTm`5aPw=W*h(i+z86%vm^!?Xfos5=1^Co1IP9Zj(F!2tREu6+R77Pa2xW5fm_1 zBzM{j#?Bl+{Uf=zkabVJ>`{PRZ+tnyu@R2!!0+w-5ctZ1B(~oC@|#;rfRbfWsKdkY zm0SSC*rv@q?&9q3ZcvUo3+Zjk8qVPRJ#l%#R8tI+{Dw^OV_vJa|hkReJ*`9Nwrg@i8>ZdR~X|=+GH!b{$CksODk5CHZ!>)|e`$@^0 zb|E%grLl(5$&XmxuIe@ovZZeBFLo?8eVF}uu!XNxS#5i7tL^LX*++;dMyVEpn(xql zc$4bqI46WZSlz9v_DZ3C^42KqBB-LzcZl!^Z#>zkcrn)N8cRS_t<9X@G|UdyIuCC&JjkVo%x8QeQ_T$@TZO?>Qq&Ku(y24Si@V9{u<-asv9mg57u zqP$Z;5>0dmXp>v0|4{u4!H`@y$CasKB7NbhlHqviiyL}6;hUe9sD^0zvs$eZ^)8m0 z&$EBd(H(Omo4PgndSCJA%nt*vga<7(jE|J5WRqONB_gDs)Ck-03CjG1?N(UzC~^I+ zX*_38dCxp8)mT73-LW;5NqfnVFG`4p#SogxhgO89V&F#=I84c+4U?Rjv2O*$u8-** zqyzOi>edP`$&YK_+{kZ!D3;$T5{V*|MZ~&xN*JuKB)qtK{-Qs;*6k-Nc2n$@(f}Fd zE6lR(hki;dw8@L_p$fEJuBBhM5B34d+_y}W1orM6lO4cPjh!Vs5K|ou-?k*gP19ILJHNkXcUg0Y7%d2mB<>fz>va0tHKz_Hs0kbkJ^QorXugdel$3K^n1kd2a0VKx_R?_Sq`bpD&$(`8i!*Y9%W zyjc=?UH$UF<-_D2bmQGQWLjf*xdst3&m`6wg4*4nP-Ar?J|DkEbEa60-QngGlkK_f z!J+j@hjwRP;W{4(zWU>q=Y7*FPmd=l-}H&R9T!}udQk^|ljly1jb?^6SS`~sbD!W^ z=+lhfBX9sbMQFDgygiuFODFR^;Q$glY@kczuO#t5o9h> zDR?*e=g*l+TWCD$JMV!B@Lr%_@SljkJcQo2oU#9%tK?$IY-`)s{s7nh4_&t0dw%Xz6gAIi@2ZFM=bem)v-`}`~) zlRM9is~U+c8rLzjUmV=jwj;op;|5;8?>OycFDnC#=U0tgfaLjmLuR^M61mfQFRJ(k zbVYnW{G@;a$#kFIbd_IKaW6^iRu2`|je?XizEWK#_H~&%ugz z*R?b{;gqVK!VtF+^3NIL5^J<+N254~dFDD!hKGkfZ_bo|d?8&r@C~_}YoDQuxr*ok zY4AeVeo$-xnMPlji)9Oo z$ep9Q`1OjGHfhJN%4!jOGos;>BqL!OH2bDdl3$0eZcOzOr(f*%0R_vo^Z2f*A;ix2 zIf{H)iyiM4-B(i$x{IyzM@>XVo7&F~@?cE6YDKO|jCWjU#+Eamu}(Zn3-vsc+W+WXG8kM`Bo9+WIQ`bg+FKizVuJ&&Q(QWM3IVmiTK zJerbJyG^9(!9M$wVyhEz4iC!i>L9`5qf*nd*4V@XGNpTypU<)H2|=kPb>CZ~wPxrb zYpgvhTspi&_;z~3KT;(9t-;+TdY%*GrxQ|K z3tuK7z0#5tf8XRnKR@{CS{3gw@vSqot?OJZ=KlH73$JslMfOyb7ZClawXu4@Qx+D$ z=0F*yh!T^4m>tO?udWS0zi=b?^7+n0;jTB&em?rE|6{7(qsDoivBX0;ecs%aJ})_i zG4{<#-MX0ZJ>T*NTXr(TB(5zWwNCEaK;=~X<)zoylLJZ_N1cSuxRH)MeLSuvNrnRl zoF<}e*|HxUPw={f%%$(5NoK)DpY#1|Pl*_`PXLg^Tf5NvLe1ln*!E-7qSrJq&+KqW zM0e~%o0p9!(Md;PCOz}{_V-SqUFY6hdHiKm|H=IDAtU?vF$AS${S#we`|8sLGp@`P zOMwF`oHaEGH+EeZHo%BHvzbQ$%rB|L((tgoAlOP@OBQ z2utX!lPJQcq`<~}(Cmk@n|9yLMsu~QJE)#6AhRzVgw8{(ap|S9Jq+6T;8k|U#{kRZ7<-4-3&vkhdzF`f5URYJ%MSx zxqDRV!eYFq=Yf++Qzd%BSkG}AsmM`*DEvyKAGs06rL88y#Iv0vH^GpmIl8RIAc1Tv z7x!Ut?)0ml9NyC>HPvW_j$OyJ-!T%qq_-I++oGHsaZhB|Z;-6Dox;npPaMrNKcBR* zFW)7@!Q%^$#$N%>8-BGNq32%o9og|2;RqAor=Z2|&T~>v8G+R~i&^3`GGss979m5R zY;7#iGM4(r%zef*U8Wc}#H%tt}l1DA=NN-}Ho7c9eGbQb%aNFFbN5CMjVOd4l)x`R8Vp4CuCaCuZ*HmCx11+7(Cd z#nT`07L__2Ms+Okyy)jq2IEhbohCB%zAwb&X|KN6bN%gKRZo_Ga3L+?gKdF4T@aPw!5zVMR$2-R+07wtnKR$P{f z213Y~)#U>V%v;9n<@Bd-qI+pR+K)vYdUhm2m5D{EH6n@kGq!vGiEACwBzLm=q9t*h z5-O>x)|?}TuZ014n^t)RL6y*y8tGIg6P=I>%Z>GPMr{iv#$Z$5^iC_wEx;$AYB#4M zFG`pmjaza0{!}>rhPkzfZftbS^?dA|I~%AB?XO)M3S_=F#x^M_UY=LjXR2pK;J&qz z<{Thx7?N=L%{7CXonYMuMp#yZ9##42p3@X&K#kBp+_xOXRdhZd#Jz-QIlQEYTz zyir6!pH-fd@hiM}MDkYDa^7AKxo@WYVz|on3tSa!A9~Yv$ph$dX9Rlgd3*A7{8=@< zD$1C<2_7T3#%*p!Zng8U><+G_hCl0Hm1y& zZ)d+kx1pmGJe~;`PlP2>=*ydG^%r?wH*{j+cCrhajBxK7Ub@M_+qGAzbz0>UAiP7& zM%T>i-(X)Gu}VI>+s}ZXfFns(J&-MyLVH0cknk{f*t8j7CvXm85)+E%!m$2j>( zElid(2Yb>qir+#bqi(ZW?%?U73KttP@f}e9u0Yngi%D`1rO^Whju&Xq^u~4mswxtk zx;0(oyF*TFGkkG&r|VOLUW#Rr^GIzMBY~kC+xCagSTLrQ;vBC|3uM{q{IjwsL01oI zeQ^?6GN=|%lvZ?rSKH`=_A({l^kSIB7$Cq!eXWJ*79{YYKl9UI76a^P>Y6CKcTacz zVeL=i4+8&v1T0tuKf+d}vCENgtW_Bu0>@EuBWiH`D`bv45}&g?(8|zC20*W}Ht{_K zcU}G>obL}I?XTq`vte-FdayeSj{j5We?)rzc}fSbMgZjblHz;=_%-z3f&CR}P9D8S zd@KijZlt5g)i0#D{w?W!kzO?EIp@aSKXdDE^!}v&R}r|@{P<@&FEvQxY2X#np#jzn zTV}^(p>J@~9MT{0lY>*M`LSvl04Nu4a~>X{00mtoY4isu04`JFR63vqJ~Y-&(I`3~ z-};mGxHf6-z-`sKk|ASg0_|(nN3{~_D8bmo3Hp(F=>9`JdvS3Ey1$saG(n$3x`+E; z#paJ+e-QYCz#jzuAn;#6V5Qb7Wf%+Otc{$@hSWgu2Jf|*Ni@hesO?@Z zp#z)`Pj8y+r2qnv?{l4Mp-0ETL$SqWSimvg@T}`N5)h&qou>m)KxeU$Z8H)9QWi5s ztS_=CV71pdbmSd5*`mJFZ-vRMx4 zlS`C9n*3np{v|T7uRHSgL@_xyi4Q3`?n?$L@miJ~o-`oiv*ub4>hFC+KFL<&cfixB?^*`G^vP5^?~^g<9YJtwQm46pnPHO!5NWDlmW= z`r8x+`T?K^5&RJp=~x<)B!0#WU5Ky#5YW*a9Dc=r|D%eubc3@d@P_Z6hS4w<3r*=vM3g)WpD|}3e1NQV1ceDh-nfIOi&vB;zZ&T#i9GND+?;W z@}zGWj>CBjZ2a%=4^bRK;+GT-!3`iffkVLNejhaO$K^19u*-1R{k(;QUvQahmnE;Y;Ou8xYOVcgY(i5u&{4%8V<02PKe+<6A^R_F%7GL z-=_%u6A$jR;Qye*{Ga6E!RPT$JlIytf6$TlPxA19srn}#>^D4!;6R#6q%QuwT|&gN zUjy*(cEN4?*8%u--Ni(edS0D|VU%4?z2|Q>>0SjSI@&Z2Zcmk0KM6fTv`VUG1 z83ocr#X~V1kWe9vCyWsRB0Gri5b;9<*Y(@p*@0u&mmqJy0Ow$_P;W11CnyV2cFW*J zWw*@fjAH;O-Q~ld0`2ns1N;J^66mi4yJheKwOi)s8tCN?YkdE&`dmEh1Bm048fq&% zjyfSKgopr<9Yh)s@k4}$XfM=1a14JD_e;08mw$l0R{&7lE&F#qneCSS)(!`6Zx7)0 zuloF*eFL4n9GyYJztQJo@8|Cf*)H8JgXhAoA3lC(JkJE234)73*KRo@lyQ9_$Oexd=f@YYxNG04zdzJ192Z&0~mo!Sfl? U1vNlT2}(c#0nCA!skzqw0aWS&m;e9( literal 0 HcmV?d00001 diff --git a/fuzz/src/test/resources/io/github/dfa1/vortex/fuzz/VortexReaderFuzzTestInputs/filterMalformedFile/seed-vortex-dict b/fuzz/src/test/resources/io/github/dfa1/vortex/fuzz/VortexReaderFuzzTestInputs/filterMalformedFile/seed-vortex-dict new file mode 100644 index 0000000000000000000000000000000000000000..2ecb1c99477666a0eef1746e32256400a7b3efa5 GIT binary patch literal 3112 zcmeHJJ#Q015S_En_Q{xV1QfYIQG`-ZB*=sm5kklXqDnzQP(VnDV;f>&pCjjN1uB=+ zR1_2`5kG*E5~ZM`qM$&Oh$6oL^WN_4`LGl51IR}2c4p^uXJ^+D5jh8NWcl$jSwE*A zNhqvIN{|(&LS-3SWc@P-XEk8^ob|zfUxRurZa(YwUM$~8sFj4enNX_7pC=8fgqd2LHJq_w!sHJ}qY)|6jLS#2gvfb~Ywfdg1?!CM^r%Zv z&K!b*$S>o|OLVx-448|}hVxi1GKWa?YNJrf2y^&u7@AgM>OhTK#pGyKC-}-9lbIf& zmo^H$FzuugdJ7w&M_1`%dIFx?WM~doG%y|g7R(dmG zXIm*uc5mx+`kuZ{*t34NrN;ZTEjQUxV>w&Ood?Wy*p_nKF`T&jD^kQ+*h|K{{TWyS zxJP%sAY~cFWIe0=V3Z*w$$AZeZJ-J)0<`_`do<$ATn61<+&sAcvKuuUVC9R-_QUB%lm1YL9Msbis1j#w5(IxH|fg{mGL3aJob96 zPAhH=nyhkwY*}mOoZ6$Q8SBftTfkTEYr(C#XP16y%iiJr;q3SU=m1qs+2?3);}%R@ zFrMp|5+2&TIaNmG$e6qAo^r0xKVy~2+h?b#M}!awH7VK z7h0u0@P)5DOiZFxYt^c?Xpvg9)>@_3TBO!mrIuQ?YOQqdJ#z?!!zw?zR#zu$W}lh; z?Z?dd&Yavi0Fa={#g$gUp(_tQ0`x&1`XV3u38nx^6e0zUG>TA+{m~Bxpg&n1hygeV z12G7LaWID95DdjI3@6Q@7=gnu5{KgmjKXLfi7_ZaDQU_Oq8wvUfl7=+6{;~FN8xDF z9D@ls787wCj>i`;2`6AOzK9b^a}uWDWK6{=I2EU18oq?n@nuXW%^8@1GcgmhFdJXN z9DEgLVJ_y8rUvt|01HuzMfe)(kipqF2lb?BKqD5T3Fo32S>&(;OK~2Sk>-3X$JcQI zzJUwzO{~C0xESBUO458At8fV}#bvl0S70@+#8vnXt|rYjScC84T3m@JGCjKj9s`i*5Kb-osz8oiu;N`}i9^z=!w< zf5*r81fSv`_>46F#OK%nD5V_bDo^<;P<>RM0{W_awVz6;0+m#SDy6hat0GlQn*CKg zuYi6ko>#yDDxOzBe-+OwVxo%Y6)-@>^9ndf#q$ansN#7A3{vsD0tTyiUI7QIcwPZR zR6MVMLsUGkfT1d$SHLh8&nsZKisuzQrz)OTzz7x3E8s8{&nsZ0isuz@xQgc$aD6?I1X(h$WMOTN=dxgNLrErA+fci#rlG06c1$Gea=E7L z_>z)HMz}~;SqGt&Wmb-}vck%8D=V$6wzAa9u~tTY_f>}0afAe(L;5$|NF!U_Wj$C?RP zw;95TCK@L`O5mONa2ASol{U;0rV zVO2EmaWPjmfLl$jMkxAfT1Nu7|q|V8|T5E>3I-GByApL61vLri}I1H zu*yt?_5;(eHWMwg>lK=5)PyG8Oju#_tccdDT97u~Ou6-MVVMf!K2v7QGUaA^nF{OA zZg2E!$`rF(fqr3`Vso;6er$GPsc47qUemTNE<5kimrvE@W^KgNqnk z#NZ+Z7csbq-50S%5nB{7IK|);gHsGnF*wEG6oXR?PBA#e?o(`0%ofEAE@p5sgNqql z%-~`M7c;n+!HEt=CptJlqJxbRx85VyY?0_-lf;>3gpCp%V$~y7H^_L8SoLtML=Oi` z++oJWwnwa%%g79APn7n2X(vnDSBTO_+8bqLwX}1kb!EI=GHjPv>twuLVzo=2?Gmfr zj7!9J$+JuHJXglM#I{SUy2PqWth#tb64MMutXjpYRU)>ERjbUdRcvn)C046g z&6bf_(zb|Ii&$MLBMoL;BDP4xW*O<0SltqJ+O^w$j2AEj-b}6D>T^ z9UM`2a6~O^(ZUujJki1vEj-b}6D>T^!V@h#(LEed_i#ilY|+9NEo{-k6D>T^!V@h# z(ZUlgJki1vEj-b8@+`ElMGITButf_`wD3d=PjoxaP76=8@I(twwD3fC@tSC1ix#$M zVT%@?=q_0k;fWTWXyJ(#o@n8T7M|$4Bp+dm7Pe?%ix!?};fWTWXyJ(#p6C`?6XA&# zp6C{dC~VQf7AT^S;m0|z=GW`Fv47E$Kd;0v} zh)*Zbg>>G0Ae}A`qqF2A>Ew7EoxB`J=fWq`DexI|<~x^8cr$dKo2B!RZ_sJ&C3H4> z4V}c^Oy{nxbjo@!ol-nPXQ)rniRp86KKcrshQ2{(pYPJy!H0CRxdR@ZG9~G>vcDRn zhN{Ep44_O^s-x6Ib%L6rPE*s>EOnMzpz2hEYF6i|3)DqwmAXP*t*%oysoPYG>QFuE z0rjwYT&-8ns7>lcwOPHc-c;|X?dl`-nR1*wC+QSB1Dqkw2xqhtI#tdDXOc6;ndZ!J z<~Z}6I;YXeIm?|D&MIfMv&LELtaDnNPUk-7A?Gn?y|dAI-g(J+)!E{_yCFPx|7|h?&X z^iK7rd$YZH-XgEwYxb6T7kVqb%e||;>%CjNJH5NT9`8Z#QSV7_gSW|h(cA33?!D=~ z<8Ajo@;>t%KhID4#r^<)h(E#~?T3DqKf#~mPw}VuGyFOJe80|b^mG1le}%uwU+u5) z*ZS-H7QfTK&wt2&%wO+s^q==%@?Z70_;2~!{P+Ej{m*?b$PZFMzhGc+NH8K86_f@Q z!T8|VU{Ww8m=??k<^=PDx}Y)019=F0$4$&{AB$Rp`^n@+>CcO7-XC%oKf&VQhl_xHuv?PcCfNM3wt3J9g0NytmUokFk5U zXWU4t9GCm~ULFyiY{e!Jd&Y83D`m{zImY=UXcmE;gUQ2{&pc*s<`a@R=r^&3i;cPF zd2JS-YNtqIAV17Snts&e+KF;Jcm-Mhf*CsEIO!& z4lsIu;8)w9R(ztw`HmmDcdY%@RbORpGNE@XDYZ|bQYy7iaZ+l?87ajpuq#97P@h+c zXUHW=hG)tp>PM1i#U<+hyJTiY+%n8$?(y2P|2n1v75+HLAG4-W<=9nn*Y;!~Gne(A z==0>7T`+t4?>5F!SNsP*Q+Kmr#?0GojN@&Tu|C2U?sMUsS5O^EmB-fZX2<${d9tE8 z8TR(hiPzDvw|9)^x+i;k^~_$wXaepmhP_dDJ2_!~o03oW!t5LQG_Z;4XsSG)=swaV zMjd3H=!qn`oJ2}qUtX^jROeBhNOd?>wq5%9jvXvHm!*rFbG6IHEMMGMyO6qAV|DxW z>K0}erHdoV_!vw2~%W1aN?J|$~ix=0U*^U>HzeP3KTzySr zZFbE3Os=VB0VljB#%EJabGDZ7J$w3Gl+EU_VNaWenygG8Tdj_tDVu*&bEYAa%Pb|A zyq|hM zQ4!w~w6_MrifHF`WG|S?4sBuo95X@D0=iAA$|GmX3Gi}JO#Z(LGeKX(B+VeavBFq$f-_ynyr=fXL1Vae6J zxmXj;OG(aAo{TYTkd(QZwf5vq%~reOsB7nMo=o$^o3&><&SeYbQA$ofIhfJ(iHiDl dB(JgkJXtrsgOm|X;IcuC;#_B9#`Kw!{{>P|h6Mls literal 0 HcmV?d00001 diff --git a/fuzz/src/test/resources/io/github/dfa1/vortex/fuzz/VortexReaderFuzzTestInputs/filterMalformedFile/seed-vortex-onpair b/fuzz/src/test/resources/io/github/dfa1/vortex/fuzz/VortexReaderFuzzTestInputs/filterMalformedFile/seed-vortex-onpair new file mode 100644 index 0000000000000000000000000000000000000000..405618c31a9640e670809a315ba68ae71bb367dd GIT binary patch literal 9377 zcmdT~dz_C&-@d+c-@o0p7V8+2}E zNX2lRjS*})2j}8EjKul402g8uF2cpQ1f$t924it4#^Ewtjw>)8SK?oofQf8L!z4_` z6r^J+uEI2AU^=cwCR?(QjU41+2J(=P0?foLT!YzcxfXM99sZ5$aRY9|T-=14F%P$} zM2?qk=7!s6sVrP>VX;fd#k|cj0aybB!j?y|6p!I?EW;Cc63g)vR^Vwo!~vf={s>JFpX;SheuANUi0vE^?Z#u0#s6glKl zh$1MGq6la}(bSM)s1Y@$CKO9esTnn=7Hnxr!MXxkQLwIn6DU|$Kx+!t6)TZ~bp^Dg zU|j(xQn0Rob`-2D;3Nvx70{l7bp>>wU|j(nDOguPCkoaT(3ygD1$3ccT>)_vtSg?U z6s#+tD+TKc=tjZ10=iSMu7Fc0SXaQQ6s#-YGz!)g(1U_?1zxX$b;a5_j;^?Oj-xB^ zdKIiI@Ol-jEAV<1tSj((6|5`pdKIiI@Ol-jEAV<1tSj((6|5`pdKIiI@Ol-jEAV<1 ztSj((6|5`pdKIiI@Ol-jEAV<1tSj((6|5`pdKIiI@Ol-jD7?GuHfy1LArvs4+iN9-aZ(lD|q{0kgnkEgF(82 zw+{yC3f?{#q$_y)V34lh?Su8Y0x8E0MMOq5h;A6usBx3nrp=nSXxZw7)@|CJ*zTnE z9XfXE+$HYhuHCwya_VV4PVX7dZc6-=^aAl|8JWF$=}7P1I?|`l8E2fK<9++;NWcF5 zb!@-@9Z5*gk%0qsWYC$3XC);M9x^m#SnBYzN1Su+c_Yui;KETCU3|&tF=H6ZX^T{yH3>!vsILH0i!NZHf)_(;98S^s6(o zvf{IIa&t3<zdiu&bjX2*WYmC+?#HmcgwA}-9EptsJNuG zth}PKs=B7O?v4d_-gWoFd+xpO{s$g>Xwl+_A6fF~(#IZO_QaFRpIY(sGb^86_1yET zUwHAQH7~#N>e|=Vy}tg94R5~n_QrSKeQ(p|_dob>%SRt?-S)|++js2zY}e1fBgB^--r1ea+C``XT%)$^csA|h&%4dwGW?J z6TI{32tJR>33N1{KIOdWpD*P+Df_&f7{gD9!Tb2a#_HRg_IRluI)xj|yle z&7y1QTDqQYpc`o}-AuRA?Nm%9R7T}gL6uZR)l@^ZR7ZEv0=kp#qPuA!-9z`%eRMxP zKo8PGw1^hd!}JI(p+{*cJw}hyGJ1lZq~-J!t)Qpr8Cprt(kgn6o~PCH0=-Bt(HeT0 zUZGcMExktT=yh68Z_oyMlis4YX(PQu@6vm;i8j;w^Z|WHTj(SDn6}b3`h-5E?X-h- z(r2`bKBwLE1?{0PX)k?6U(+}AEqzDd(>~fyKhTf#6CI$R=@L(=ui5K z{-(opghfM2DMz^~q#{(Lic$?!v}&kgR3p_`HBqsuscNR0s}`!IYNbw4tyLR!3*Qb3 z`3@stxen*^+AZn%e2bB=*5(iP(sedHOqUz(g@t?%lJZ5`UdS5rkqb3au}+uhuvCX- zww#Ytq0{Ajs}jUYoA#Mi*>sYP4fRS^>vW9{=ktxsXC*NoO=5^u#dqM2nti(G$ba{r;%cwbbf5YBg%D9U;TFR^!!b zv|5c-%lA)zOp`6+WXm`?jB&DMoNO7(P1byE`XJe|OxD+g#jajD=>ue9y zY2-T1x6blRu{={O&lHO|pKsM>+)^yh6pJf%!#TzFtlY%?cBI&PQf)n{Vd$wAJ=LP8 zhM}ig^i+$UYSHDkFZC39Zv!Ugt55Qb`XuhRLSkkNCGK;VnBU(L^SMjRM=#RMi*)}K zS==IxTBNxbS?nT>T%^&8EYk#yCbQNsO|XcC{I+4{a)L&clJ%SuEawEvIl*#Hu=SW( zSje-ez8}milw5z+kyt&tnT3*Tdc@2^$us?J=3^n>YD_EY=cD8r^!iySxu(C(ER;Oc z--#N%Soct|MlQDei*)@a2Vt3+d!Xsi;8Rid#v`Wj_>~7^e2Mp)>c_q1qbNy_RT*K7NCdo5Q&1|aDOsh00Z>lwVwZ*R1c-0!MT4U8{tQw6~W3g&9R*lB0u~;>lWsSzH(U|i!R-qjsef1{m}Tjb3TT!R#DW zwvWwOM|#`GuC{F)?%lau{&gURe-CKEZy)XW)jy8k`FrrYem~yt&f>39`F?ah-;l=e ztNfMx3O|)!-n03oeKwEbJU$_p@q76KzW+SPuK>&V<$D#saOgVSo!@DCQ$HF=Ni>v()46mZji$?K0!^Xme6uOw z+sq9#j|%w)Q_c64d-;B{gzqCO`2O({-!?Yz-Qs<|OKj)c!ydjb?BhGaLB16n=Bq#? zpZuHgIlrCith%Wls<-N|&QwFxaCM%#NR3rjsEI0FO;a>0>It<%J*!r$HEOMTL%pruQy;3WYP;H{_NcGbKJ}CORsEq3^Q9xgiFO)0&74+F zTc?B5#p&jp=EOUFoq1xxl%^8RuN-q&ex%bSK-%bFOi&b8d9zIrE)Tr`lQI z+~YjtEOC}OE1Xr%3(hOfI%k9Pj;%swvIJ=#%oPEv#=aBO^UvMJb7&q2!;kI#4 zayz+Q-BaD3ZXb7mo9GU8&vr+;7rU3bbgGN2ph*UuaM$DU=c#5gHj96&e$|A~Yd1IW#Sl8JZEA6`B*eA%xDxb@|V`xc^eu zS$Deo@WgfNG7}PO8zHg>hP~dZOY7ySTQ=HV9 z%U;rpjvV249oOPZp1{lZQm#9DsaMj`ZWOgg!jFa6;7kPypYWkJ~7;i2;By$ zUw-y9c%~ltUgG1~4`bhhy|i7*w_d3_9**u#z4$3*xra{{Kg9NRN9765)}!*|ht}_p z%9D0U&q$m6(Uh`!JtsEc_?41Z?ypZe@`CYOto zvYn4#);yk06O#Y8i=}^5X)MM5qZ(tW}1RqvLD4>>Q3;+?^q3s?#1qU_9&|j(O&DL@}$f7MuMS2>Qv^e2Bm%Zp)@)jKRo!I}$yfkg{bzI*j=|^A3^?A}=$LhG= zE;{z-9T&9D*Re_d$RMJ0-$nCS_^XaASDV<2zNI~8965?k?fvy&Fo)vKVWU5nz<UY;j#eC z+_b#>bS}U2SX)kZZdyhj+twdz%Sy}lx&XViEw;W5xp^7<`$onrhLu&>^~Oxv8OGP^ zce4h_pQPw{A8GQ;z9+cyt!Pj*E1^lx;&>F#?y&Dez7D0{NTU@px{Kk}Ho{YetUThB zwWtyqA~+@`{j#p7-(0qZm%ohD6g78nv-Z)=^cYUcnVCH$eaeKawAsy~ z2D4pUkUb+i%-%e@ss28DW@hGu$qXX;lljnty--}keJxysqc{o0`}?0*OuNYAYd$TLrTM2?PEX_T&dy$vSxCRh V2r?Eops=H#PyEL`V@Ho0{y!i=Z+id$ literal 0 HcmV?d00001 diff --git a/fuzz/src/test/resources/io/github/dfa1/vortex/fuzz/VortexReaderFuzzTestInputs/filterMalformedFile/seed-vortex-pco b/fuzz/src/test/resources/io/github/dfa1/vortex/fuzz/VortexReaderFuzzTestInputs/filterMalformedFile/seed-vortex-pco new file mode 100644 index 0000000000000000000000000000000000000000..e2a9ced7bee426262857c0d0fc447af9e1d14749 GIT binary patch literal 2435 zcmaJ@3rv$&6uy63peq$hQCD@&4v7IJAPD$&C`!a2pa?-!q(BuIwDKq;=w_XOpw>qO zg~1R7H={#{t}=Or_#h0yDT+D>@`w^Oip{*=f#y?>*1&o_o&)LI|+} zM&biY4x167uw&2wwDZWuJIjaI8L^$`M`yX)Kiy-F)>^H|zmd+y;~GfoZ0BAq5E*>x z`Yu;j>383on6M^HSajNX+_zJPPj1h9_uT62)UtBD{kxX#I|2O1W};)VF!SS`xqTy( z1NChVwP9t}G4o~yE&P6OMciL%x4SFEIi2~tUIv$0CVtarS$FSA^5$ZFtl;{<%!*Z- z?(*}#*@l?*_^z+L^0V8$iybEuk~b^$6@U5bflt&TONVK#w|-uktYFP0p-zhP;CqVrmXZQ!4E%Hy@O zld?Rw9N^0ob~@Rc=@S<9CM?kM-u@WyVD8~ zNsF(AJ=)6e_f8x+jq-qw9}bH5C{6xnZ4aKkIanvpQ{QnFtOHy08y5fCB# z2w4Oy;271O_yNl0(UBX7CA+jDGzNN$*_OdI#_3`8grI#cFv`VbJTYf2B``#sgh70!HTQ!x;-!20fSqEZPY|ftx{MvI*s9Wq67JbDV-X`RH8*zlPbu@opTQiWwkCLHX?f=9m8fB09D@VyNC!Q%1y+OGC@BSUQ^|MjF zSlY4o;R9woHyRlX-xEDI<1!n97+-0)B~UYpLIegaz)rxpPvbejlq>`Ro|6|UfC?c7 zuCYd1;0RzZV0&ORsU9&Ha6~Mr>ZBB9nsd5ZrHqCpxy$LWmpEN?Y*Y#v;N;^z$dgl& zG@t>LPzQ{Mf1Ws>s2GjvD~vdU)0xEE60b-h7GQf7xCB@W909x>*dAB_9M1V!!`v8C ztx8T&s8UEer~4R|qhs?$s@3u2+}L=@%0!J)6{RH2ABne3k(8{2-2I#m>(AlX7Nv#- z+qD9=&WP6pOM)Vq-8#FECh!Nf1=iZMzf9j3yyJLxFb!nv^GG zOeL`@R$?o%;cQfz`1s9HY7Lway`N4nB7}OV0p>ysNH`+Xx7QRm!|uX}18wLY>6=dX zEnx2%dYRh6lG`8c7}|;S{Xjp&f^RXtJ$$aH_`g#G$V>4u0twl=V80R41a_U+KtK!$ KB_ZoVef|YOKO$KG literal 0 HcmV?d00001 diff --git a/fuzz/src/test/resources/io/github/dfa1/vortex/fuzz/VortexReaderFuzzTestInputs/filterMalformedFile/seed-vortex-runend b/fuzz/src/test/resources/io/github/dfa1/vortex/fuzz/VortexReaderFuzzTestInputs/filterMalformedFile/seed-vortex-runend new file mode 100644 index 0000000000000000000000000000000000000000..0d759c71f52b726a3567b545d911bbe2470ca2d9 GIT binary patch literal 3816 zcmc)Ne`wTo9LMp`-EQ93>t?sLYBuZ-ckQlrbi13oyRn2~ zS(H@hpeUgQ3k*^)QpljhAf-kHEhs3+s30+ml8Ow9uIKCX`F`DZw-M1FyU%#+d!OGw z-rw){He-yr5K1k`VE_fxFoKC(u?H_=0ot$?hB9#r?RM$Nc=TeX;s|I`F8uM22n%o3Bb zSM^Sk<3$aPlk_K1`(4m@&(!O6ee-1atad6>GqsV`P`MtAM9k|+Q#oY&w4QIPmsgTj zi`*=HR*KoHTeF)#5zUy6Xr?V`I&E9^acv{!GCM}F$Jujhn_e`vPwV>8&K*5Jx5|?I zxCd7|R+_2j>z>)P16O0K>AjQ>Juc&~+Rd5n^sm(I(%}Cur8`rRv`IxWJ*kK$4u^9y zc^ymet_i~T->ysS;kgl0HoO{18&rsc`$!>~L@?JV$EI}z;m}+Sc|9V52=2WPZ+|B8N5?<3awk`_)c3`@dY6gA#?4 z==v$anYmTynV-fgXe~h;_-HoqB2t{j{1?(*#t7o3a~eY*y0HT4|NP(ex~^K!NO`!L zAH8lzxs=b5HHZB=J*s|PuFzjKANb{MGv$@)aE%)NPW0>aDEoc-2Wq7UHDZ)A)?A_S z?k{Gm_Fgzm`3X#741HLS6^P+xzpwsp2*xd!D%EVMYPR}y^Q~uO?wt3P%SE$$?s%2_ z_FBHwpEn2Q8E+^%T*)){dw!knVSgP%!-c^@wJ@S;b82qi;aVwQ$`J{VohTmKPiU=! z*@`_LTRug9s^f}B_&L;k>Zf~4&w{Q&UjHwtZJghFu1GDezsGzkRf&epSYNNANt_xqY4!xrW6fDwN@_k%AKkw03F;)Asv`f41G}hW{w%pyj@gLO6Dg^)l literal 0 HcmV?d00001 diff --git a/fuzz/src/test/resources/io/github/dfa1/vortex/fuzz/VortexReaderFuzzTestInputs/filterMalformedFile/seed-vortex-sparse b/fuzz/src/test/resources/io/github/dfa1/vortex/fuzz/VortexReaderFuzzTestInputs/filterMalformedFile/seed-vortex-sparse new file mode 100644 index 0000000000000000000000000000000000000000..fbac83fa17e98b8e52c34ff6e25c142d3232b549 GIT binary patch literal 1929 zcma)7zfV(96h7~V?ekIaiNw^zp%DfXh;eW=VIWKl4q}Ll)KcS%Z#*@)rGFKMK-dKbQ@x#zLS|n)Y9obJ!dRLG_EI?%{h>j6LOI;bDpr5Vw93hK*rdN=KvXJ2h7;dd%d2BCl%~z zM;Y^CuC%)arn~@-!)Un-WyAP{$F}q-!Kz?TO4ySEvr2nJH81a7_eE%4qc0^^4MADdjJu5b4ZlM}u25;7JSr0aII|zq~!7fAhlEwHxDjz0^ zS4Nd9%ZW#%!Co%m%vxxUio>wXopkm(VJfB(6Vp>p#c*HQvo~OewTU`057z@?T(##F zQ^Ue>4LM`l*S?H?pU#N!dMF0*KMcuSZ=eE}_GUPz>qh7L+Q(<3p&VHCrSJ2 zWIG5lEYR(D!1FGlemHk%E3KN_!Hf7+>M zwYq$@I?kxY+GwTgX`XiL5XsL|@HVQgb{&s;Nri#=_N_|M-XP@w)1@F*n}JvT2Ud=d(#qdK>o62@O*Kue5Inc5~MzRqN5 zDExp?LlE)Ux19ratG_YTdh(SFblM|(egIr<{Cw)C3=|3Du6ww$+*nruSI zDMwG9M0)Vhqk0n0A|6DU}`-5+1=DqLlz4x8(`@PvT z2!h~(K%8@zAk?)ioEIv>E5evCF4P407lkRIE)bW6D?%hp3p2u-!mMyrm=orO1>tRh zxGvlfZVI=AcZ7zpC@cxLg*(DsVObzng!{s(uqLbv4}=eeheA_m2}I~Fg>_sKA00#_ zObdBBu8D&VqNt90;-Z5ns>A#sC(jSlVSbQPREPONPM#m8!~7s8&kxgK`-+^bedX&g zKgh}2SLTQ5uzf|&qOc_R`^wj0`-+^bedX)0`XI-zk1QQlALRJ;k)@*{E;@*!Iv$9N z4x%ZvxKV^cmJX|PcvC`M$kSnUPL9=io(}Va9INvzKTLe86w$FN9y*9TKTL;xKgh}RBN8X=2RT{$%GY7v4|1~hmHA;h*2P5!VRhbe zkGXXYEo)yz>PzdG)iXK>zdo{bm>=Z$^^v86a~B;%Q62XEAjj(5>LXIyARYGoAjj%F zPseR>(Lq?9=jmvQdsZD&%0UNFRL6q2=pc&fxGOF?h_mV-$FGlZwbcak!*pCy8#;(c z80LrRxF#Mt2&;3ek43c&s}FMg`mlY~RNo*!$gw)l({V-Ihy=xqaF+9moT57Ji;E7z z>O4;e=Po*kqB_hEa;(m+K72oJh?Dk%oIF2F$C|k4Ad2c36Bivs*1j@7Oo#bFPEj2V zanV6oom+j}Q`@lmAg3v`xKV@x-}7Ph0ne|GEFD+HMF)}RM?)NRSbgaB|KEvv2YG(@ z-$CA6o*(n#r2QbruaAb>(th~gL2~^1;Cnu-KFF~;xB955ZCHJfGcBCuydvk8aF+9m z9INv|I%*V$1&bys`k7ePzlh@C$1L}sE2UroRf>|k1h2-|iTFbUliUflF&6npalhk# znY54TXU<5>ti?Y_89hq+n2R+5=3sut8WF1H%356=&yJ6eDQDxt8xxB$xx)n3!W9ld zPqh>Ulci|fah*=b4gbSsZsuep=5;0>v8EP7m&)%x-Wty_)LQEw*KRuDX4 zg27R5*1~$L7Dj8ekfo)qSk^Og+>+dIjAB*{zfQz2-Hlj_En*}DuuR(r*!*0ux%Ty{ zS%3sTGSK^t*FyDAj zheA_m2r~ltU;T4@%$4=*^|t%%{kIN!-S#6r20tWylv~pGsI$=zeoxx@CMWGX{q0@# z1i_1>k8(`7g+=I^JgVtj?QIQh-Ku>C|Tdke=^~CMJDPP^A_>rzd<|B{2MJ;f(=RMuS z9o4_omExSsuV_A6iILFu)0qCU7>ONs`rAnA?>2%|k*Q*y2PIq^6^P`R4t{imxcurTA@RQyFbFl`3D{x1+ zQ|8vPg?JL8$R0i z`H6{BhwgoQ^^e87e-A`jhyQx%!H0AHM|UnZtM_shz|5V}z0u^y6~~tX@)O}()~7kUe|%{rnw)s%E)S~F_E=?u_A1ZJ z*DbfpDi#kfH`Tw|-v0W@iC;$}wYH!C*}wMtY9{lb;oz;?l@umaHRT`UAn?_UP#6? z=3**+HCne5RDxadu1Y#BX^Es3QTU{i?IsEvF0xula~}crVY5aEG3Y)dcnR>K)p@ym zDdseD5oII=3O)vOMhLLwA}^)Vhmf^p{XiaWAP@KnmK|q_rmSZ&8DwS2fx;tv3-6$6 zmXG#&$aO@WEIs&e5ZrccR@3Xqg3eX2_QcheJ7@zlgVlApg(Mm7UD%iY{Fjo7WZJ z8C^kf^Bd$G{1$*#1_8)dS3UqK;0tI`BXP)nsW*d>V_`5Ymw3Yyw$pSs(B$`Up zS2X(0IV~0Fqkf{CjNp=s;iR(=*WF1QX+k{#`B>*m zQ?{3qK|R}$6b(Nyu|~bd2mwMHp@sn6iLFcq6tN5^2F>tD!)PKN4$+d$N?v)67wQk1 zY(ef<bTa`RKNYY-gZzw(hizQSy_{y^vi5e#FuQl5LiSQNS5+OinC)5yB z!pl;37tJlZiFnF1;wF1r^0td-xY%cJA`xZda=)7FMU0dgHR9n^LvO#CG=d2ByKQVH zjlooyTs zK!~@f&WIZ^Vt@kThXsz6m%{V2;)IN-K_$;k)LP{~P-x}L(Ng9wZl1W&qV~{3EGra; lpN!td!$xC+)_p{~SmK0wP*OP(ZKKK;1KOFkH13akq}&`x`z~?)CouzQ;a!g|l*24oM#6a5$Wvzy=+dHFNev z_Smt+^?i6@_m`tMk1OM>dmG<16m|dppk~mnI#$1F@aw#}PfZ_aRcM{n`r)}ezM;z7 z>)QQm+X{STF{hR3CwU)RPWf!PC7pe{mjBktp{3}n?5MU$E;V0z+j%yxvfA`oGID8E z(}_Own--kPNSCeY*6itDP-p$dn3^~h|c+Cru{r?TcLuGOg@jjdBx zNEP3y)B_c3Yt^}JHEM}eu~oI*Ug24z9kth;W)_oLP zD%D%HD%FV+#T=E|OyOIho^Z5W-A|&JpxSP%czsU&Ec~3hUaa7$Lc1#Vm8rKIm#LG* ziasi9t|IuX+P}0^Z7)`ERn}Sx&rg5R#J3+2~^+Si4?tPQQG4b81}RG2v`_zsHP0g5a33N!mm z6+d0YPgC*NsQ4>Y{AJCO;2VmRD2f|e^8D%c%3--YjLnd94rtAhqNlpbQOFZMXt8u zik5#8RKi2dZ;G3J# z*E_P=akJgHOs=zuHor{=j%MS;+uph%dSV=g)zM;oboJ(<38}^_bS?ghO50fEllsKGKqofJZ$ptsYPGpxRCQN_^rC^Od(2ZJ z)7BOBFYK4P-28Eb-`b)+sSB3RGoKS-u%^f)b-ejsoZ9f}l|@}rZOmiC{Z?qFwi)U1 zURV^Q?mFXf*yli-ytb#))FPi`X<74uEC>G`eUl@boi<;xnY!HP<+D9im#nACx2clm zGj`+2_93Y)CVO=Ly93P|`)9oKZ{FB%8TYMD_0CaG-;Ar; z>Gk2nvzQH|7ow`6;o!|LK ztR8Li?$O5~rhSBg8{G6NXZ8-76Byc~BBy6iKwv*NetC|uPhbmAaPG2E5HGMq zTxQxOs9#_*ujH&L*CcQxucCCOb`UqP$EGrm36uSUeqZdVupE9|8sDfoeNrN;+dEmd z>i$*bIE*THkLrTuZg_)Jx=ll`QF_Pm!g zHRV;+XWp{%^eV7C?hzlYDm#%ZxwVTg$!>gAzG2;56Hhx!+&&>RbAr-3#=NcZoMvS)Onbk0@+8sVDwhwRi52qtA01vc0D^`K)+;x&BdsL+p-S z$Mv}`zz`dP)XT;6gmURo{3y!FT4O5XS0Tn-J|s4lP( z2E-6Gg6?2U*n6Q~U=EhV3hfIvU`zU=1HcKKi3`dDcNjv3pyn3iBa?M4X~tMbE%lI7cebDyRV^`3=1Qm*5Jy zhTec%peA?FMreVDIRCkQkxep$C|fUZ^=( zf)(kD+JG(eCl2U97!0n&4fTMb;7PpE5ik<|MaH2Mz!&^U06H0_!gLaZ&Vmq_N5ar> zh=gb&L>E9T#E}G43UXLXlF$_R3YL=P=t@`vX=FW`0hzFgWT82b2l?b1bO#i{Zc>cy zg9C7g97ex~qwpg+j-G^H;8$`6Jqzccf>fckP!H$HMf5UUh3n)ds)E~amo%aG;UPRG zPtd2(3T@;C`U>8{JMsbLXyZT7C0x`148e$eftr9B^d=T)AFzfmNk7yM?7@)?LI;B@ zxDgL@D0qT58G(+1F)*G?MEzhA1d=J}G?)Q1NiaGG=0O+D7lOPbLkZI@);KOV(2b~9D5Kbb|Xb{2zvJj023CKtys(@rzLcT_q!3tPS)}rem z9X61S=w`@Tj)<5(1G9#E`*1=!w?uoywKq=68=TTqT^vA_>oCyAWVU2BnX`a zAuyMOqVpjFqKE*E0TINJ1XKbtNF)k08J57;WEr{=)<7Crk7ht7Y$91`4&*^T`3Btq zg|Lh4LHELbI7q%lzk{RjBl!tE0YAejavCj#GAJiiXf4#kZ{z}c39i63as#~uYPd_9 z&=zY4=6_uf44xFa8Uy=1S9eVY67OvimVA#M zgAzDSPNKiSuW*K(MbAM6RFgWi9)5!hIv+V zpAQ+0j)8G7f%u~S5CD_ORCEULVK$kA&Vw)rCy{702w?$Ph{l5$q(qJ`h9pQKU!hB3 z1*|5iXd0}C43dd%f-J}(ThId74m(K^x*LjNA31;?g2Qlx{D79gaX3kSL4SoaaF(1y zE1?FIqyaq-7vVCwie87CpdxqBMreln?o&Gd$f2HB%qzx6kx-dpJ>=YIF8{@oD_Pj~_qc>eal*HpghTUbZe> z&A;~f@YaswjL!ya-H{*I{I;6Uq%-lmAF%Hm*tZboSdxqFp{K<-W6gIH_^YnP)@=mU zzy`Li;o359Dvv&TaM*KAqv@5JZPU|yU(p}q*gDw!r?CyJc+0@CDdck4vh4RyLwC%} z#nu>M3rscNm$0ST@o@$lFa=}Hp5lRK(AVrc9%`}mv2}M01M$GH+u{VNT#)3lI6@jJ ziZ{_V(6-cm)6G^(YoM+rei^83g7*`T^w}{_)oiz|W?!SR>}=>bMmGN(qmDCZ`p+?P zr1{vCjnFZ$eKX%%$KG;s%vU;Sw$1#WzGlDKJZyW}{;+R`EI-?CJD|&6fej;^oE$Ew+wM^XRlM$F4Uv zmJM7jwobNxZ2gQ(E|npg$D(+-<}tg5+K29%^WDB|`!Qhad`Kt0WB)&Gi>_u{I<`F; z^LCt5D{R64cippj*!ir6j)ASC^R>@jJD=9^`E}7B^LM;{I<5=0j{kN|v@dOl=VW&U zBaR{Fa|X7b?7bk{FScyK-|erVu6RsmRu~r{=h)$Q_MX-mjGz_AgM9}05pqBdGl6{; zGlIJ~=1AZ1%%yh=%dt2zYwV z*mv#Q*%OaxJ5yxx==qU}LZJYsHck%P}w%-vj?dj*!a0w1r@cy^q^Xc{}`|M}{~D9%!v&aQ!u=}`&Te;VJb)A$G} zo>0#2Ztch6=g*%vAzmCIl46;wG_BJyk57z?<2?O*nMDz@g#xy%hIG&Fv@MGw#P~}> z<2&tpoJc0u9PLpwt<&?cX`ES~j~B`?KAy&R+ApCfNf13>CQ1>^*DT7(qIo)Phad?j z=-}t;j27X{mC<;MPsb`+5G7g^5jQ_FSuU85bEvtoJf`_NT~jn4?o7BBvTK>$o9H}5 zN5}|;vDvOn9bAaI7B7H%jE2$oJi9;tvhN#V!YKR76oy}dAt97fM1?@A3RBggsyZRYs$%GXn3-xW_PLz6*v7u3ArnI;MueCV z>VOa<>eLBU45;eNj8F%}$Owf0*)NKbo8-&gz5m|%-8V;q;@g>A==fM zHc(mD!~>)EVC(!T_ge8Sb!aPfEq7TzaL~5S&{CD<8rqI%SJ0Pua{aWg<3Q7>?w9pr z9rIpai?%THOEc3m&J1{2IuM=DP1a3=gy=-ihy8)rug~I&%ut7ZZbsTKTq%<|mDu$ViF{FCGc?HXTe07qtr_aT z^py3x&z$y=YnMa&BJ<{J^N3DG<9A>36)LBl>FKFEbtbhu1WBAb6+2oHt+ICTxE~h@ z+^YL=T-_wGv?E$2e}#P@_PJR+%BC3~2fY-nk{@V9@5RpVXQXmxRjl*%%<>EI`AjWJ zGHBmKRZKJ%y}Q{5xZ z5%24Kp_R3SyL@j`#Db>9I@cYpK3sEN0oK`v@6>&;0hR#U-<5t%?td!#jl+uVBl^dc z@e%rMMADM=;nw75miuiL`^BKaF;wSpEMD;~V|j@2CRhW!Uw%H%QP%p!#X|q0j3Z@> bJ3rM?ABwxm=f!hX*J>ZtlWP&LBVGCj6y;;f literal 0 HcmV?d00001 diff --git a/fuzz/src/test/resources/io/github/dfa1/vortex/fuzz/VortexReaderFuzzTestInputs/parseMetadataBlobs/seed-chunked b/fuzz/src/test/resources/io/github/dfa1/vortex/fuzz/VortexReaderFuzzTestInputs/parseMetadataBlobs/seed-chunked new file mode 100644 index 0000000000000000000000000000000000000000..654c620cd2a516bcf4de2eccd2a165e370c8ad7f GIT binary patch literal 1308 zcmZvcv1=4T6vkgJi+7x7j2J{AM5~Y@1Pc*M2^OjB1<5(PI~ND%cF&!;^}NdBDoYD% zkyK)x%2F&uQi)iLq!9lE_59xMJ9c$5@VmKr-@G?(-t0`UN7vVA(Ymxs0d=THU6{5( z2do7|OX$(j#;^ID`?DZVGp{E*1FJk;#rV@Q(IJ?D80>*9&;vhP(7{_U12MP-)&YU9 zSl}Ia1)hNi;11XV0r-w-@4+FMff(Ed*FXS{nt4r~kLg(K=P*a|otlHGI>7jcXcsCi zS6OJ1+~_B;b$^$8t@)NZw9TgGd^V0ewB1v*k;j?_^ycsh#sPJpCU#VioIx<6L z`{7fs`>3^d&_2q%mBu`x6VdqFmwLs@Yj1{n;ZEIoqYhD;IIj{zTCG0Nh~9{uzt4ek-l{~qoy_tF@!?c$kY>=n ziN@z!`Ni6L1 literal 0 HcmV?d00001 diff --git a/fuzz/src/test/resources/io/github/dfa1/vortex/fuzz/VortexReaderFuzzTestInputs/parseMetadataBlobs/seed-fastlanes-bitpacked b/fuzz/src/test/resources/io/github/dfa1/vortex/fuzz/VortexReaderFuzzTestInputs/parseMetadataBlobs/seed-fastlanes-bitpacked new file mode 100644 index 0000000000000000000000000000000000000000..4f1dd0337e4186c17669f90bbe627c2d7e9f15be GIT binary patch literal 836 zcmZuvJ5Iwu5FH1TSjGgCq@X}lDWlu~DFr136%rNMIF5*L>|n2>h{`9Rr=#Kk9Dss~ z0wGR7$uS6TcGrqUGSbt|=k4s<*|okbzeA&J9@tZeVk>hTYp*}j{N`t literal 0 HcmV?d00001 diff --git a/fuzz/src/test/resources/io/github/dfa1/vortex/fuzz/VortexReaderFuzzTestInputs/parseMetadataBlobs/seed-fastlanes-delta b/fuzz/src/test/resources/io/github/dfa1/vortex/fuzz/VortexReaderFuzzTestInputs/parseMetadataBlobs/seed-fastlanes-delta new file mode 100644 index 0000000000000000000000000000000000000000..843e4fc0978c0414cc24d7d0e38f0a80dd7dd620 GIT binary patch literal 868 zcmZuwu};G<5WSQ@qYz3c3=E7Nt9}3yQiT{9m{@5@T@jHcC2^{XsWT&6Hzr0t0U?AC z17FdZfiGxz7duK&d(xAA_s-{czQp)7+yg4iCYMaI$RQh{Z=emBKn*Btc1pAb=M5{{ z%uxO&_EVq-bb%JYXCD{KC9e)JjuR;+o%=WveYo_Zx$Tnc2fn0F9p5SAsZ263M03qG zO3uH@qFb)GSgkh*J&B%(et-|)377&s;0$O1CUCq|I(M&Qu2;=d4AB?v!oJQtzXtyWe2(9PwQ$WNy%zo$KIhhTF5C@u-hw`k Hvek`0aqgO~ literal 0 HcmV?d00001 diff --git a/fuzz/src/test/resources/io/github/dfa1/vortex/fuzz/VortexReaderFuzzTestInputs/parseMetadataBlobs/seed-fastlanes-for b/fuzz/src/test/resources/io/github/dfa1/vortex/fuzz/VortexReaderFuzzTestInputs/parseMetadataBlobs/seed-fastlanes-for new file mode 100644 index 0000000000000000000000000000000000000000..9beda946fb3a84799caf98c040b64b66ddc1c0f6 GIT binary patch literal 868 zcmZuwJ5Iwu5FH0YEMtNR6clu%L=hJNrJ$ssrXj|0goR@Vf1-#gC!nOFpr+&iR0ts| zS}qV#astA8_F6GYMta(v|C=3K`_b$jDa909bTPyd6Q)n#5YT}dP|DZ7$QqSaVR_J$ z4s{in09~L1FogN{zq1)n3m8Rl>W$mCQRwxc%CqvV&wV}LP31$`H|F*vjWc*erpnit z``oK6yk^DPtmsv#*AJW&bBTNdufQWP0lL5m&;oSeSmkcwu6Vjpn50ga%DM6_<1X&3@==KX9qY*;RQ3Y7dcRP5*8--&s)p+mpm9# z%-F~|a__9jZ-w=$afL_Z9lda`)6du7U%))R2f483o;pi?1f8*U6$^Ex$Tz@`R!Uv_ E0~pnqWB>pF literal 0 HcmV?d00001 diff --git a/fuzz/src/test/resources/io/github/dfa1/vortex/fuzz/VortexReaderFuzzTestInputs/parseMetadataBlobs/seed-fastlanes-rle b/fuzz/src/test/resources/io/github/dfa1/vortex/fuzz/VortexReaderFuzzTestInputs/parseMetadataBlobs/seed-fastlanes-rle new file mode 100644 index 0000000000000000000000000000000000000000..17de9465890c999d85a648de850a6f49c36338d7 GIT binary patch literal 868 zcmZuwF;2rk5L^dCEMtNR6cm&xQN#y8DJUtZX^3$gVd0#EeYPU1ynqr>(n84t5CuYr zidRIGynrz4JIhANN~62o+1;DFwT&Oc-J#NKa>*o%9I_Gm0uBHZXaJ=>qF@cqt1i4Z zl+XMsFa`QR59k0~fA>3|^XdS3oXB9(y@^FIfJ@JsTd%l5=u7(4@r^2;$|M6rG}BzG z;{3}@Tye+7T%A?xH;g=qdl7vDZ@>dE1^U1-&;d-~NY`#+uXXydNTnwvooQ~Fo_z7l zi#U$xcClX?TxWss1A1E0Z|o&$fV1B<=Tn6f;(4f?w?n(grWnimJDukE3jB F#vf?*n9KkG literal 0 HcmV?d00001 diff --git a/fuzz/src/test/resources/io/github/dfa1/vortex/fuzz/VortexReaderFuzzTestInputs/parseMetadataBlobs/seed-null b/fuzz/src/test/resources/io/github/dfa1/vortex/fuzz/VortexReaderFuzzTestInputs/parseMetadataBlobs/seed-null new file mode 100644 index 0000000000000000000000000000000000000000..4ecb7310a5569baad7832eea00906f7acf98f792 GIT binary patch literal 1292 zcmZvcJ#W-N5QZm-;iBU>f`|}EC{RTyP(* zBa{kJP^F?o6bPlJMkod1CjiGg_MB|wZ1mh_=iM34&VG2ibZea!txs#@(JBq757SMs z3f4WMCG_ZMgH8i7tpcT&ts0{Tblh+?PL5?w2O^a zsBCQF!sr9o`hV)RwtQck0hOb?}qeIu$$G6s;B8f1Jlv0e8B7 zoYy!>BH9zJRlmw!h<(uHqoOq9?Vwkpwdw(3^&JxG+s=qoG S=f!b3UlUk5gPt!0pO=3%r9El@ literal 0 HcmV?d00001 diff --git a/fuzz/src/test/resources/io/github/dfa1/vortex/fuzz/VortexReaderFuzzTestInputs/parseMetadataBlobs/seed-primitives b/fuzz/src/test/resources/io/github/dfa1/vortex/fuzz/VortexReaderFuzzTestInputs/parseMetadataBlobs/seed-primitives new file mode 100644 index 0000000000000000000000000000000000000000..623abe14ad126325e8e767ee58c7e42c5c2e4604 GIT binary patch literal 1648 zcmZvcu}>616viL0;f@m#BQcr~q#_iAC|n4!GK7MPSXdg)x!pOKxVLxg&YtHLhTp_hxqHy`Aya=;j3Vo)V3d zM@1T?5?phj2qrwD0kmjn>etyBw--RG5o_6=uZ2R>4cJFRMEjr%0V!6-N#gbv<< zE(pLfm;?mA;sx))9@qg7z$%yp9{7Q<4`3g3K>+T6Y2bmw{JJ`c4(QnWFCxaSx1%;Y zViWdHt6lDEEfH5$)Kc;he5K!Jug+MbLz~HKiH=iWLt8mVYl=9L&^D}g34ObsL~f@t z@+Ech8&0LOTps*JlxPuY znz5QQx4)G}Sprw`<9J=wMkAyRt2z10{4MJr?XO3K&6vwU&#dO;2O7~E>*w#YCK4Tr zNG9`f$REVVbG2F{hW6cRT;Kkdb&rPBi1bwE>Ko$npKefZ8PBZzg*(**jbXpg`FcnP{7 z1ouE0OahLd#@^5jasRgIm&%TB-SkJ+tk1l+i2WAqeuf7EC;o})m)eex>w)PdK9vHz!=~hxiK8;Wj~Xv7qEB1 z4$xo=+y{5TDwqRYC(aM&wzgERBu$(i5gL1O;h)X#=gUG=Ls?Oa3ox@9VvK3w;+iUJ Ua|qk1wbPz`Yl*LTN|%ZL0?s0m2><{9 literal 0 HcmV?d00001 diff --git a/fuzz/src/test/resources/io/github/dfa1/vortex/fuzz/VortexReaderFuzzTestInputs/parseMetadataBlobs/seed-varbin b/fuzz/src/test/resources/io/github/dfa1/vortex/fuzz/VortexReaderFuzzTestInputs/parseMetadataBlobs/seed-varbin new file mode 100644 index 0000000000000000000000000000000000000000..7c1ed66bf7369d3b44004c53a979c5be91dd38e1 GIT binary patch literal 1304 zcmZvczi$&U6vto7L8=f!5fuWdDoj-eR52jLSXB(2n3x-!JEvGUcj4?DArnKUjttBQ zbwUhGofr`VLYVWtYfbf037u}(&qfdN(U;Fv_`A4)zch+c8+O$d$tx$*BFx>_# zU@anALXC=RuXkVmXx6zO2R5U*OlCMO0?afdxhEsdq$U~f^&a2eZrf40=Sq|;H zXuQAGxI|m8V7Y)hAK|~eS6Bb*(>H#kVqPS!WOy4;h5L>xPS?2+N*C|QLXU`Z#OvaF z%7w>r;r0i_f~v(jzdQW;@SF1pu+GoJT~O*>YKachdA61DwnDsUEU}MzANCc6U+HT- Ww23%pd=XUN>KQm literal 0 HcmV?d00001 diff --git a/fuzz/src/test/resources/io/github/dfa1/vortex/fuzz/VortexReaderFuzzTestInputs/parseMetadataBlobs/seed-vortex-alp b/fuzz/src/test/resources/io/github/dfa1/vortex/fuzz/VortexReaderFuzzTestInputs/parseMetadataBlobs/seed-vortex-alp new file mode 100644 index 0000000000000000000000000000000000000000..df685fbefb96e6a667d3600127f5895d61deef7a GIT binary patch literal 892 zcmZuwF;2rk5L_EWB4dIH6cp)5L4go2KopcTv^2yxMzL_tA-1E4DledtenYv*`b5@*HsdpWDDf$xm0$zbfU;=c2Q=ko4z_H%DiEpjbk5!U-DwPY(Ez>hv zJo74!Be`F!mjt(2p!`6dmed<~!z94jZ<=%GItIhAAEx1mv`jV6Icc3ZFQIqfQ0HCxN^iB=bM&n92Q1rha zBk!x!$DE?~M&1+O11s_kv0gQ=U_{<=7QSobc}x5|xQ};2Ev(tnYw3^RGq L>(Iw2%{Kl34S}8A literal 0 HcmV?d00001 diff --git a/fuzz/src/test/resources/io/github/dfa1/vortex/fuzz/VortexReaderFuzzTestInputs/parseMetadataBlobs/seed-vortex-alprd b/fuzz/src/test/resources/io/github/dfa1/vortex/fuzz/VortexReaderFuzzTestInputs/parseMetadataBlobs/seed-vortex-alprd new file mode 100644 index 0000000000000000000000000000000000000000..781d7a2687ba674077e0eacf639a31fd09128d3c GIT binary patch literal 924 zcmZuwv5wO~5FML9j4?T1as@?1m4YHlM*&p|%4=w7FB`{EESz<)*HJ{3nu2ejy3e3M zrxOL=K#BMY!h7~w@hKVUX?Nzmot+u4-Ozb;C5`zKN=G~i#7F2CFa|n+3pDonjmQ%? zQ>-)>B>l(09+&}BU`iA|tmp{`?p(OkRQ! z`C+-Y5KPxCEPW#lQefg|!4>mBe7*aI`*9WVkqz{^8j>jn0<)+p13QQF8` z%bly~gXLO%%(E;NeVVU`Kb5hLV)=3=-$vy{jJt0w7ar@_Q`@y#xR2`d9_nM1<~#oZf!U~z literal 0 HcmV?d00001 diff --git a/fuzz/src/test/resources/io/github/dfa1/vortex/fuzz/VortexReaderFuzzTestInputs/parseMetadataBlobs/seed-vortex-dict b/fuzz/src/test/resources/io/github/dfa1/vortex/fuzz/VortexReaderFuzzTestInputs/parseMetadataBlobs/seed-vortex-dict new file mode 100644 index 0000000000000000000000000000000000000000..3e8ebaf83a0df0b1a3492389b5542ac13bf1f69c GIT binary patch literal 868 zcmZuwK~95E5FHE^f>>*H(M99JbXko@VByjW=%%GmBauQwfi$jq22bGH1Gsc)x^U$Y zTzUxWd+-UEcEaQH=gkc7&w#epu1=*gOFE*9DYjT}eF8&32ljx+0sXBr-&?Lbzz-3(2T#uUNcut_sr_5{!HR5@~8f2<}SOY1D>{v526Nt zVjZjeC=0`pha#LQzpo)ciSOtEwNN9CD`7*~|8r@@`6``^DSOZ^&SqlIiu_JkuNg;% pK6c2!y^Iy3^6@XAk3FI$*6ypd+UMAtsNTqLF~6LXQOpS&?GHUSk7@t_ literal 0 HcmV?d00001 diff --git a/fuzz/src/test/resources/io/github/dfa1/vortex/fuzz/VortexReaderFuzzTestInputs/parseMetadataBlobs/seed-vortex-fsst b/fuzz/src/test/resources/io/github/dfa1/vortex/fuzz/VortexReaderFuzzTestInputs/parseMetadataBlobs/seed-vortex-fsst new file mode 100644 index 0000000000000000000000000000000000000000..0983b9a84c4826330bf4e2a91cb5ac0ed2f3a375 GIT binary patch literal 932 zcmZuwK~BO@5S@}DwN^!NVIpy1+-Zy_Ff81dcmdr=DJ3?RmbAabxMJZ6ID;z}9>KU` zw(FZ1UQHOsizm(px$iYb;jVk7hgoB<|K14=tR6xo2&#Ywvc zNxu$EfgaEWP5{`A_H1apVu+%1m=xOKuQ&O5SvQdl^qvl7SJKYi?)B zc@J6iNX4I3yw3D= zT;(49I1VMz@iz2^ZmL2z^3(HvpvJDphG%Ou$8M7P$bVne9Hyy~`D)F`P4nx^kLH@o z{>Mo$3REzG%-@Q4t@N7Dyxapv4v@pw;wI2)*b9*x7HPvM^Oz$YhL+memDMTgL}cg>Tz?x<1On@ZN(6cY^zQ2C&5s z%z(YHM;cGUL1+K(W#soO`(#crgPr_t7JMpl&!|_;Bf}Uc)WC0%D~&S7e-UGx8+)R5 XN1vrXfp0;*QeS6%@lK8Wo!}UMnc1Ud literal 0 HcmV?d00001 diff --git a/fuzz/src/test/resources/io/github/dfa1/vortex/fuzz/VortexReaderFuzzTestInputs/parseMetadataBlobs/seed-vortex-onpair b/fuzz/src/test/resources/io/github/dfa1/vortex/fuzz/VortexReaderFuzzTestInputs/parseMetadataBlobs/seed-vortex-onpair new file mode 100644 index 0000000000000000000000000000000000000000..bd2bba527dbcd8edef0daede808ce6e11f3ac49f GIT binary patch literal 940 zcmZuw!AiqG5S^50V~tv?2MvrbNK4xYojka^NEtR#TEv|UtOADb-;0SPm22j~vS7aGZ4=3#u zBz@Wva1UGpJzyW;{p@!-<>>>HBs1ad;a#G`AzXR1-0F-QMuCxM8(*8nbCZq1h`d>D zeZ~dXV|_!#uX(+zSQ%MCuJ4Ha;Cu(3fCu0bI05zn7wAsQS~u|D)*2)_H%c2hw%nqc z&MjB#V~`}VXdB;x-j&KttaO+k4kDAPfCWD;&`ed9hsb}K=MtT&CiHM&9q+!^>9#s{wu5A>hvBe4Qnwwi2cEWwk93XSYHiy4dK}0703zHA?&$%* z5??U`*1{TTJP99_{qM^u`m5?>PC0}1qBjpd6}fBFtLKqnj1y|&x6GAh730^$80W^C asJ&~?%AdgZpkBMb#QWttHH&wGeLMShjfLSINM(+AVu~%UI7od09l!({KxL;$Y$52f0DCf2 zya|5}^nfnV21t4REeftKFiFN)I6avqao9&FU%Ixvtn~*$COsqbfZRBTSf%^_T19PAUoC9sZ1g`YHUHDpKki==`$C=D@ZJn0K)$?wW zB$AiaerfoShj9?f$C`d4f1HLm`&-vIf30H_BnU8Z8%jBM>aA%R`e}J}sov3*L-06l zz9g-IcNMwdMs0YqGN-s+$y8jk%wM8j@z3BJ_PpEj97k&!`qNg!HtLx{TxRy2JW3K}E|Y8ql3BP^12@J|s{PC$t$sVO-C6+(!L zQ$&=UfbgEZmV=U!o_6Q|X2&*u3};(PvBZ(Om|}|s(-&|6)PWjM%E_w83Y0NU%A--} zye|V|pbK<>Ho*Lc-|3X64P;@I_@n*1Q29Nm@}hlfm9H0giF|7NdS#E3D1}F4qJ7QE z=Uu1jh81hmqSs8ltG=6HE|G8G4R`{^Ko>X$+CUvR(zzSBYo1=H;>1;noNM1AKG}TF z%P{QA{d~RHzfFDR`SQG=-q4L=AHCnTC}qhJsuK}NSs=5wP|`KpJ_n3#n7;XE^3 z2X34{kCgkiWe0KG2scitVgDkhOw@wa=C&f%$s>!{dTu|1zl=Y`-7v?UmSb7^?FR0s zRWprQOQ3SgmbIy4RNC)fcwvecF21upoTTuI8QF^*C_D)p6a43Gj45Vp<{Y_q yR^+$Bdb4qbN8|&&aBtAh*Wh2kJiZ6Hu;#8lOFe_m*oKaUx>V$w;72Q^Vf+Df5|~E- literal 0 HcmV?d00001 diff --git a/fuzz/src/test/resources/io/github/dfa1/vortex/fuzz/VortexReaderFuzzTestInputs/parseMetadataBlobs/seed-vortex-sparse b/fuzz/src/test/resources/io/github/dfa1/vortex/fuzz/VortexReaderFuzzTestInputs/parseMetadataBlobs/seed-vortex-sparse new file mode 100644 index 0000000000000000000000000000000000000000..a9d831dad788e3ce85105319762643a7a390f354 GIT binary patch literal 836 zcmZuvyH3ME5L^d?En|WS6d+NdlnC(yNGT{OsA-6CjIeO*;Flt*d;=Xl6(2xJi6|pJ zf`X4A%-CnyC|PNAx4Uz@b9<-FHD^z1v&0cyOtHm6XbzkLI?w@XJAktRXP|=5TIDmp z4$OcdFaY`h&%gaH7F=zhh?C5po;=2pKY}Y?D!19>MuC^fkBV)$2An z?=Fw-S@B5aTJ>&2HxmQgPv8ZZ0Yl&%=mR=%rOs_*uQYmblxA*}NuhG9v^*{MyNTma zUYF~o{zL9Zo-ZFN=QO=0NiYtwpzslr?-l(fZj$<#ya~0OKQ*`XjNP=nzSQq&%RYFV zIA4_3!TX9_@Srk0T^XymPRXpedKte&zUH4{ubHz!&#^2upRqgbbxf@t38FHxYwajb zl!xK97w4E_^^cCRQo=j#$X?Vy<4V}L!GGOaslV1iPIbp_snLYbiu?<#*XC6jygPbf kZ!phk@R#!(=Rqy3d8pRfFW{4FC@$Q!>bwbkjM5D44)5O>WSH|vgNskiI7 z!@6#2^OhjYEnC*6iV3rB_uPvzym0ZI?cpSa7tF|B% zy|W^}71o=ND?B0}=!JWge!d3(0_O2O$b~g`)miET=!~tZSg0#Sz6pM`QYzXXsWh1Q literal 0 HcmV?d00001 diff --git a/fuzz/src/test/resources/io/github/dfa1/vortex/fuzz/VortexReaderFuzzTestInputs/parseMetadataBlobs/seed-vortex-zstd b/fuzz/src/test/resources/io/github/dfa1/vortex/fuzz/VortexReaderFuzzTestInputs/parseMetadataBlobs/seed-vortex-zstd new file mode 100644 index 0000000000000000000000000000000000000000..650a30375e649d7d13c2b2582a4563fbd93e3cdc GIT binary patch literal 900 zcmZuw%TB^j5S?Oq1S=xAFp;=oNeEwHSh(~DbOWU=kyu(nFNCmS;o7BN;M#>B;K~*8 zI|yGPp3~lj8kpp8nK?6a=W%Onjn1W{QkJTiQW8tbF#P~6zyxwYD%U3>1t?vdlnzMh zd0++ffiBPl82k3S+wru3c{o+xqP+|Q&xIUF4oYS`3`DS<--SHTFz=O<|r1koA9nZj@Ht+D*BpXEz9E zf$O=wvAw93y5K3t_)(DI>*Wg_AIyBe=O77p&iB^>SmGD@XDzIe!jrI2+5f(bxWBYc z`Xn=`#J%xNFeBd)^JV?W(8dWB@tgQcF}3k4p^Y^ZpEjXRSVQydl+7FHsM z_!k^j$6;kD7KhbgVQnRc$YJk~_aXV7#L@{)05RWQI0O+`zC}QXoK%dMEl?kcmW)^1$v+jJ|7`^ z54OQ$5P@~j1#R#R6Yhf@@DwCq9dtnx9ALgZunnGo2wVqUfaz)NXPM_I^d--!bx}VU z#j(yU1)_WFV4bF7KkoUNjj7te_m*g#g*H~XQK=)^7476dT42(#(GOw!Sd9x`=+F+U znsxaoL|AJ5D%auVV}~lY8rmh%PM|MqwEDqFr=g~`s$V`zb;@)_ziP2FwTrQ zENzO;XJD>}8P+fMGoD@Ln9!~Mt;5^uyjeO_CWp@_(Y%_cqa>ljs?R{#`-bRK5Y3An-W#PwVS}n4=M`jGLYty_^(*XKVs9_R`XF{bzp=6| zQK`18dBO+n%YU^v%V`<+P95*^U*>;!;Tsq?R3R?{*FbSUp`mBziEypZS^g-V4(_tK zUMH>r=Y7C~@USmGd{^us4>S$d`R?Nzu!;U1z&frYx(Z6YLk-bkI?6We6fBJ*UwqU! ghBs!Z^lL(AYcamYF?nA(K7${gk3-QnX>l~@54dS+_y7O^ literal 0 HcmV?d00001 diff --git a/fuzz/src/test/resources/io/github/dfa1/vortex/fuzz/VortexReaderFuzzTestInputs/scanMalformedFile/seed-booleans b/fuzz/src/test/resources/io/github/dfa1/vortex/fuzz/VortexReaderFuzzTestInputs/scanMalformedFile/seed-booleans new file mode 100644 index 0000000000000000000000000000000000000000..72060d1ea529b4bd093cce09abb363741698109b GIT binary patch literal 2964 zcmbtWKZqPv82{$7*_&)O=LOdg<%nB^aK#a>k`y9G2y#UTDMCmg*_+*Y8z;`qZf0iJ zbE_0rh=qlSSOz&n#4-en6e5R+99LUdNeB`wEu-u2_x@zxWjE1~Z}vCuz3=BTgCV^;OD^kj+||_thZ~aQ*}zV zediz6tt#(fd<9qs=x=pQzVWw$P7`1rj-0>e`Jx?1Uq8domoMT6h0?jexoa7Rt0{dg zXnR$8fPM;W0A`JrvE$}Zj}Unrk(wMu{hF?ZD65E{C+(B`wuyoBt6Ru9&w=@VYW;7Z z@2}r&mnZoDYyFVaEMMduqy^+qMOJm>){q4)tjZSJJ`e*Y|CRsD|7GIRZoStZh@bW7 zo4n`i==*2>bBrnH-YMcTP7wyNu+W_k)a>6^S z)Fg%A8e7gjx!g}p;F7Lo5S&Jzd7Ijy19CcS8H;I8U|xt=%&DRtZwUI}wXuvhW0}NG zzO{t_v<<}W+Wi^+DI>9M;^lO83r|bW@&C*`W}K_Zy;eiK{3_LPKbSqhwP~P)2vx7) zsmJ3nD|!?d_VQ9~oyD@2f{&Z*!S)WoZ%_=Z1Is`i_!B|z0p9`JKn$D%mH;O9D;(Sh z?f|!e4d4Q>3{-#zP}>D|fNdZK)_^8Z0q#x5H7d$j_AQ@#(#AVVZjAJvd}L+EC$eFb zx07s`s1L!b;hRI--drOG@65EU!2a*>ET5kt>qoghE7z^;73kS`3UzCsvW}9`^t^nK zsf@XPVP)J;c3wYEHj^?LDzN09m2=&kywfw^Q|92Ump;Z1r+sZk#RrPM=J2iJ952Q+ zDN0?HQLqfFkK=64J5FIhg%g^jD5Y^k_aBqP<*Qzf@LUhHZMh5TgNW}KkHUs?X^ z6PvN>B%4v%YJXI!7HVAs3so_jMRfkS5E6OM< z<&2eiwma_+GF`y))AJZtyVpzQx|MnL)BK_3^WL%XL}@?ga>!>^=G6x>k#8-Z`?DJr zWg2CwIGd;35I@b7b$dBvzgrpCcQP+lX&HfY@Q!`;fKEFp`%#BU;z;72F!UGBH-1m< z+&b=*PwvPGp`kn#%&^ije$Q`+g|KtAxMo753 literal 0 HcmV?d00001 diff --git a/fuzz/src/test/resources/io/github/dfa1/vortex/fuzz/VortexReaderFuzzTestInputs/scanMalformedFile/seed-chunked b/fuzz/src/test/resources/io/github/dfa1/vortex/fuzz/VortexReaderFuzzTestInputs/scanMalformedFile/seed-chunked new file mode 100644 index 0000000000000000000000000000000000000000..91a8ed5ebec0cbb127d8f22454619397fcc4c7fc GIT binary patch literal 24500 zcmbu{1$dPA)~@m3?(RW~7bxytpcJ=4(EtGgAwUw`-QC^Y-QC^YrC6a*pcLo3*Ua-j zO?SJ`xvuZL*Z%E0E0dK;CNP<|8B(WK-Eux*jyK`F!;MJZUW)wJg?3SRL_kE?o7lxs z5Eao79Wf9Su@D<^5Et8KuMHBX_P@(d=Ip)s1k4^7bw&Ebz0Xo*$`Kx?!?Alf1b z?a&?_5R4Fnq9Zz?GrFKFx}iIIpeK5vH~OG2`k_AtU?2uzFos|#hG95HU?fIiG{#^o z#$h}rU?L`AGNxcEreQi}U?yf^Hs)Y1=3zb-U?CP^F_vH{mSH(oU?o;zHP&D))?qz1 zU?VnRGqzwWwqZMVU?+BAH}+sJ_F+E`;2;j+Fpl6Tj^Q{?;3Q7rG|u2G&fz>R;36*J zGOpk%uHiav;3jV2Htygq?%_Tj;1@i^BRs|vJjJhghUfSVFYpq-;}u@x4gSDe{E2sX zj}Q2WPxy>4_zItJ{14#~0TB@ikr4$^5e?B112GW`u@MJx5fAZ^011%@iID_JkqpU^ z0x6LSsgVZ0NQ-nxj|}(*8IcK@kp)?i4cYN6av&#i;XC9;9^^$nfFDp1KcW&UqYA2`8mglPYN8fuqYi#TUDU(RsE-C{h(>6P zCh$X3G(&UvqXk-`6#~#2Z4ijI2tqrwM+XEW1fl4NPUws-=!$OWjvnZVUg(WJ=!<^n zj{z8nK^Tl77>Z#Sju9A%Q5cOe7>jWjj|rHFNtlc&n2Kqbjv1JVS(uGEn2ULsj|EtW zMOcg_Sc+v>julvmRalKRSc`R7j}6#}P1uYr*otk~jvd&EUD%C1*o%GGj{`V}LpY2h zIErI9juSYEQ#g$?IE!;Qj|;enOSp_HxQc7IjvKg%TeyuoxQlzZj|cb#5Ag_(@dQut zE1uywe!~mA#P4{8*KkJ!y%1TArp8p`sPR>oq7P!F5l{fTxuRQzgkEw zs=Acc3*W01)F0IqnRI0C?yK#v`)NDuK055NShL+P+hO;~cG&%~9d=)A zhusf{J?#G14!iHQ!|r$Ou=`v)?EcmcyRQv<*nO-WcK>RJ-M89d_p5f;eX1RHe;W3% z`%ydWKGY7o|FpyIJMFOhO*{Tjy%Ud>G=EXUh4)?(QH`QTSKVG*%?Z@RYBDvYnnrbd z-)PRPW>a&h->G?3w^vwmF}0*xMlGlQpt`-Pnro=F)w*hZwUO%fnrm*UwpQD!?bQ&~ z?RC}ML+!2hQwOR;RJS)u^H_C)I$52j&Q#sre9ep0rRoZGwYpAqds{SbS9hs<)dT8b z)$N_sd`3O5UQ(~BH&nNGPxCM8WA#_{H}!Yb?ft3wgZkNBu5bJi)X1vai>Wz|8edJM zCRJ0YZZECo3~DAdtNN{)OLcqsH5XEgswLFY>i4SK`%!ZhwYpkM{Ym{HyX44bwbQ9ixs{C#h3aw>Mk!JawVEL|v|~Qr+GL&70M2 z>P~f!x?gpBM>U^NPpjwDi|Q5C?cLIRSAC#9QlF~NRk!zA^IP@3`bqt&hL7kryp5_k zh8kOqrzTXBsBSxzW?wbEno-T7W>?)_Zq51Bf@%@9xLQhed*wA(R4c31)S7A?)$KLV z+*oa@`m3$fHmcj}pgC0Staej-s(nTcET9n^e8J+7Wo&#D(xw|7nRP4$j?Uwx=PQQh7P&9Bry)OYGf^^5BE zB1ZDQ6h)1$#!};|2~@Y2Omj*#jhasVM$N3cy&RgqQ}e0?)WT{p)$NthTu%K#t)x~} zYp8CouIBn`Bh^oBuC`R&UR%xW)eyCl+EwkLy1jmy2dYEV;p!-Ltm^hAYo4agROhJk z)kUh?TcLTix=!7wZc(?ZZf~#V1L|S*n0it@qq@CIny;$8cHe&S=f}_g%f#||!{bqV ztojyz;vL@O1MD&DCwzuIZv6`T5Bv7mH9R7~9=}FHWY}ZasE7u892)~M5eu;q2ljY2 z9^%6u(H|yxH+B z>~U{S~V22>9*F-JYW92&d3HEro9)5;BW^RCnu*c1f(FA^Iie_jIf3!eL zv_b&vadaC5qAh~Z4(-ta!3aSpI-(OgqYJvC8@i(hdZHKXF?JvHML+b%01U(+48{-) z#V`!V2#mxijK&y@#W;+|1Wd#vOvV&U#WYOE49vtV%*Gtd#XQW%0xZNLEXEQn#WF0% z3arE`ti~Fw#X79V25iJ8Y{nLB#Wrlm4(!A(?8YAK#XjuE0UX339L5nG#W5Vm37o_! zoW>cP#W|eE1zf}>T*eh##Wh^V4cx>n+{PW;#Xa1|1N?%Ac!bAzf~WWu&+r_-;RRme zcf7)DyulxMi$C!W@9_a2@d=;t1z+LALiz^MnLQ2s>9uj3lu0g=9z$J7-9VR7j09@P(Z_q(gex`NKEJ2s?+!j4ZJ8h-}CX zJD13ToUrqW?~ogLkQe!2=M@D|5O!`+7)4;`7sXHa4-^OBZm zg#ffh8w8>)g3u1_(E-5-K`1(+6FQ>{x}qEGoTUePq8ECj5Bj1X`eOhFVh{#n2!>)9 zhGPUqViZPW48~#{#$y5|ViG1}3Z`Njreg+XVism&4(4JW=3@aCVi6W&36^3RmSY80 zVii_n4c1~E)?))UViPuF3$|h#wqpl&Vi$H}5B6do_TvB!;t&qw2#(?yj^hMQ;uKEf z49?;l&f@|u;u0?73a;WBuHy!7;udb>4({R}?&AS|!9zU4V?4oA{EBCIj^FSCFY!BG z;Wggi54^>nc!&4+fRFfu&-j9`@Zs@mIM{hs1Vlt6L`D?Y`BpSUhn;i9L@d~OR~*EJ zoqNSc0@(RiA|!^LgC#{WBu5IQgq@3}MjH4cEz-fx$ui&@*m+qdWQLuaWkojF`PsL~ zft<*N?_lR?d5{-&u9hDKVCQRvP#AX3Rusix=WQiW5_axZ8f8!x-=iGt9IgU>fSt$v zh)S??xhkj%JD;nL8nAP^TBr>>ulosgQ4c?(KJ5IiAsWHX@tVL7cAnP^&Ebz0Xo*$` zKx?!?Alf1b?a&?_5R4Fnq9Z!N&H=liE4raOdY~tIp*Q-VFZ!WB24EltVK9bZD28D; zMqngHVKl~IEXH9xCSW2aVKSy*DyCsNW?&{}VK(MqF6LoA7GNP3VKJ6qDVAY5R$wJo zVKvrZE!JT@Hee$*VKcU1E4E=fc3>xVVK??*FZN+S4&WdT;V_QiD30McPT(X?;WW16Tl|T4c#jYGh)?*8FZc={&O5>(JR-o(Un3zh>>M^KqQTB%V<0B%TsAi1z|Lpm zAwKM!HX#xrF_IuD?A$gvQozn{Qz13%9M>0VVduH&kpXtDn-Q5{=et>u71@v--@?v& zb0Qb)+&4G!z|Md3AwTRKxF8C_&V!4fDC}IgI7*--N})9DoVYB$hn*LfM+MlqaYg(H zJ3p?BDzJ0pYN!r7Pp*ktsEs=K33k3*4?n}snH!)X?7X=#n!pcD(G1Puj}~Z&RtP|A zv_T-+A_(o!9vu)2JEsmsM|47GbU{~iLwDHubx-s{Z}dT5^h19Pz(5SbU<|=f48w4Y zz(|b3XpF&FjKg?Lz(h>KWK6+SOv7}{z)Z}-Y|O!2%)@*vz(Op-Vl2T@EW>iFz)Gyb zYOKLptiyV2z(#DsW^BP$Y{Pc!z)tMKZtTHc?8AN>z(E|sVI09x9K&&(z)76KX`I1X zoWprsz(ribWn95kT*GzTfE~Bhd+IN$_fr0@YyM6BU45heseVx1o{xT@A3=?*MpI*| zaa6aLNOMv(g_>GTt7cH$URKTDs=3rWYJRnl>h?-#F0FpAR#1ObtEg_Tmgb+-pVfwH z6SbM@_5w5qs_oQZwWHcab$h)u_f-d|gVkZ`NY(9)*E~s`s?JbntMgR1w?y-Db(OkS z-Jot(-QG^kd({2vA@!(wLUnuRG+$J&sMpn7>Rr|CJ<|MCeXhP#U#o9bxA#f&S2cVT z?$=?ZwlaP)(vHS5v9Js@uz`Ig6TI&8g;A^QmsHi00yIDYdLxUahFQy=t0k zs&&+QY6G>g>h}CKw^G}vL23szRCRmZH1|~dsQuMJ>QL3~jn+I)ov2Pxr>nD6x3@s^ zVs)9iQeC62SKZ!L%{$cH>OS?LdPH@5r!=2cFQ}K*YwAta?cLY>P<^64Q(vgBRJZp| z^GEfI8ZN5$Eh4H>RJRvPb6honnpjPyrc~WtI?dmxnbmA+4)r_L?G?~mSS_ZORLiL4 zRJT`2b5*s5T3fBF)>qx0pXTOjOSQGyR&B4ky-u3Dsy)=+YCmh^|f9;J>|C#aLv zX{y_sqj|o%NL{L~P*cd>UMRPx>r4*y1iqXPpW6s^Xet_s_OP`Yrd!cqCQrC zRew|6-W$z-svp$P?(OtMz6kCKeOipBIi?y%jjtwBldA6eQfp4DW>7P!S=DbP~f!>h=z4KB}HjPpjwDi>lkZ zuKAXFSAC#9QlF}B@1^F~>Ra`_`bqt&y1huzy)Q*oW2mv!cxpn`?IqWoO7&IKs~OcS zs@uz{Ik%cmEvObzi>q$0tmg7+MYXb8O|7ZAy?UA(sEyU8s=wMwb$daYJE);*XSJK! zQ+0d&H4jpUsw33V>NwTyP0>7Eou$rI7pRL>x3^OB8g;$8N!_aMP~F}>%?H&Z>T&gy zdRBFNmo;BgZ>o3H`|3m0?LE`{LVczFp}tc;s%|e_4DU-3)hKFoHI^Dzb$f|5CsR|Z zY1DM;H>%sqra6cDotjrIpcYo$UP;Ym)N<+%Y9+O*>h@}DuB+Bp8>xP3bJgv&*4$QY zuZF0d)UK-A>#ez;I#3;=4p&F1Zf}C-$?7z9raDKRue!aZnpdc+)phDdb&Kluc4^+L z9#9Xf$JCRm+dHrMl6qCWq25;Ssc!GF=3mv{)Zf)N>Yu9H`|M^tfj@#8S&gQ~RNY>D z&56{cY6>;AnpSmtnKWlrzg2UodDQ%>+bgQMgj!ntUag@1sJgxCnro>)sXwa?)h4Rj zYoR$n4OH8y!D>gJO+0~qCZq@A-)LcX@u9i~Es^wL;S6Op4wWeA} zt*16n-Ck48{%R|=jT)qOP~Bc<&E3?VY9F<~I!JYUBQ%dz$Eg$5De839?akG^KwYdZ zQ&*~MRJXTD^Hz0-x?A0+9#q}lam}aHv+4!)vU*K*dv`S7S0AcR)Mx4o)$RSE`JMVv z{i248?S0FLs@sdMIhGn%O`s-Llc{bmjplUfH)>`zo0>y)dwDe%Pz$TY)RJl$)$RSD zxsqB{t)bRd>#A%Wx<%ct?o#)vZtt+>W9muujCx+Zq`JKuns2N3)L+!c>aVKX z`(5)J^-uMK`dRh-qkFeRBkLdXMN?y{an$%~BGs*?(41OLt7cF$saaLGmrHXVHNRR& zEvlAK-QM?_E2uxJRn+QgE!FM)thu4uL~W+FPyYz7$D~s>V=btMOE~mqc@NHI?eCrdKnnZZEs$ zoN8`0pIT5YqPo3On#-!?)rx9mwVLYo>S(T~Hc%U@O;vx@?X}Syq;^n4)y`@+)$R4s z++Q7}4pm2}qgA&zQS%gax;jgpt1eL8-ZITA)ivsRb(6YPb$h!t?^6$|N7Uo$Db?*= z(0p0FrruQVsP|R3_eAqE^@aLM{X>1Hy1g%&!^QKyCZZZejjqO0-ChFCiPdCkN;Qp| zPIY^kHD^-F)%I$L z+DUbLJv8@L`>6xfA?k3|?Tyv^|MBGb|Lv3N|Jvh&v7g`H#4Xs*@bBO*?C1FR@c{O- z{D*ji$9RILu%GEa!*kfr^}UJ0@EZ2>{Xg&)_A~x>cn|wI|3`enXMDj|*w6dJ zAw2A7{t*!g_H+Lzhzk4Je{{rv{ro=`V#A&T5Et$q8N(9o?&yJ@=!M?sgTCm8{uqFP7=*zXf}t3O;TVCD7=_UogRvNg@tA;# zn1sogf~lB>>6n3;n1$JxgSnW8`B;F3ScJt`f~8o7LhilIzx4P^E5A1m#E9tRq9&R?QPb)P2H*PQTMBdRJV6R^J(>*dQrWiURT}T zUCj^FN9t4cx%yIddv7(rS3jv=)$j?tZy8B-doeV}R^zD&)g)?i)$RFePOoNEv#8nC zoT}T)r@5e7L@lnCQp>7tucGG4YBjZ{T1Ty?y1mAlo2ve5E47Upq`JLO&7IY5YEQL~ z+Fx~hLp6_3N2}x1iRu*9?ak6WS6!ekR+p(ORkycZ^CoqxxWY+FWg^wpQI6@*Pp4@Bdy2gGN5K(%e0iBi+CFDI&k2j*Ge`d| zqXe%F$Lqaw1fP;=Ja6dvJ&F-{z5N>=9-V8C$IMI*KYZm^97Xj!-#Ih0z0R&{&L?~Q zU&n(Qf4yv%+w=W1@czc|Z+C6Z@zLHsx##z`g?&Gd|Kt5!aX%kq$Nt&h9oT-5XFsv$ z_deBPuAtYS@$QG?YCOMF)0f&!nI3k$)brx(eI&j0rnh~HNxxL*v!qMPaeLTjXSa*@ zdMge>&`a`|^u@?BkEiJNA^9?K=MB z=_EeoJRg7Je_UTd_jz%R|8{-euNCi*!#+3P=MR#4uJ>Xr&*$dL4{+`J^y%B1d5%q< z_nX}F`|-kl9om2S``5E!PPgIx_#ZRc^}N-`dx4kj&s{aWUSCJg?UR?++55KJXEf~C zsXuG>vYiely>-(5xvTfn>(P^4cX)1> ztC9cp>%l|lza5u6@Ascyk0b8-{>#_H{T;J+=J|S@=u^ydz1L%SzaIR9H}^S(y~1*y z+daAG_W_pVf`Ukiq<{O<6^UCQ3V%-H{_VBAna}DZ_W$0)ZqIkH>!0O$zlnVw@NyuU z!25nPbI%cl_Y{@CC(*dih{5~xb??{n^~=uN`ojJmRe&A0bsu8yJL#t}ne1~*x-N<5 zHciTnXrG&1zxy0hfBpBbTU^iU|NGai1n2D_ z-ml#T|Nj0m*xSH9UU%5XLKf|F zwf)rGzgq3vV5fG$q5fTRvRJk z683>y;R@_`*3Cg5G=cpNz|4q<2fWzbDwJOGt92{a$cA5pO=M%26t>0%4Mbh z<+ApE!6E)!)<~Dx-IcC0#J@vF|Db06KHdJg7u+$(Kgd4UNxIDbd}yzIa6sFD(11?% zZhbcCa(j&BE^pS3|30JFhq+zskG;>se_mHxzmPWmc3t7QZ??}n%*PlQ5EAOSRZ{D+ zFz?SU^U43u*S84a^;PtG`*W_3t3^N;|K^QD0=oG(_N@8@>uth(?EYQ2A&33*eKrr^ z7M`cqC;RJtHurB9(AF=oanr7${*AeW%g&^UFH|+9~#is z->v&Z=RWE`Zii;=fV>asBJyIHG_L7p41^gs9P<7(QjU7$}RT^8o{^Sr*3 zUT=S2_3`)xwhy*1htD`&7UuThGX7hjUT=RsTlj^92Koj0hvWzjv_BAs{`ImJ?Si@N zo-VVm?_Yb({R2b&c+KDM!|ZhGueWI$5Zd0anSGM{mGs;<5TESzy!TD+ek3OMYTkR1 zxSsnT`#Vw1-VXNX*Ukiza*5qHd(R1cB2?gpKHz@IEN}6YkC8;!!~aG6Kd+C@d;a@g fE-t^o^?Q%#+?W0R@;O|PSC@zLnd4K3-`D>?FQYo^ literal 0 HcmV?d00001 diff --git a/fuzz/src/test/resources/io/github/dfa1/vortex/fuzz/VortexReaderFuzzTestInputs/scanMalformedFile/seed-fastlanes-bitpacked b/fuzz/src/test/resources/io/github/dfa1/vortex/fuzz/VortexReaderFuzzTestInputs/scanMalformedFile/seed-fastlanes-bitpacked new file mode 100644 index 0000000000000000000000000000000000000000..222af1f7613bd9aafbe3da245b85382bebe0581a GIT binary patch literal 3513 zcmeH}dr(wW9LLYSEX%Sia9xNNAGq>T5D~}>(QH9LL^MR81r>K)S0&a(*o8=&W_{#a z!jK`83RBY*-W*MdJ!{bKW+AIDXd5qoo1p0)oVvPr z&^Wdx-?-*Idqq#yF!$Msdsc?^a;|;;gYS3hx*gGF-NiX_LrgMQ0RkCvPNw zsPOY;S^Yz=FBu)vak^~CPs-(^n69NrvFvQA;Me}BhSg}p7R+8-vY3TGzx?raSLz&* z+Ew#57@l4by|!tg<7)Hbe&Oqv#^^RJk6qVNyCq^rU0fu4YU|W}tKv5tWZOS+KAjMK z{F$B8PBkTMJp0`4&)S+k2ToV#5?A%OJa3f20Pd7PHFerH=H|uEUnK)uk_Sw7qX4ty~od!UCVnR_-5hJ z$nT0j8uUvE%aEBO3RKg*48dils-EQwyxFYv4J!Nb4YHbRdwM>WY*6{9T+Q(xHf6<_ z;UB*~Q^x{o(;&!n{>gKh5F}$lU^WDqBX>MF7J|%Gs2zC_WWKUqD1;#O5M=R+{KSb6 zWXY3nRTv;h<|s^>3PDZ_a#qfOARlhfG+Q7@YvbjhSrB9;1ZnTSCchejT&m^v*Y)#0 zusOl^kTTi-@EHj5T^4-o1_XIa&$aqNkUdJHxc(62PzZAL+(h3A5Ts${5dS$4q^dc! zTN4D?!e!lc5Q20b;WTX!Wc&Fdt^7yn+id4it7GG2(p5IY@miQ z77DrpvYwz92m`%=7W4t(AOb{!C=d<$0f$LcE76->os_*I}zmqiA=0AoC3 zK4OJxqWOux0Zb4}nx$d^sM9yizbLkgGubLvyNU%fxj0h5XfmE@@k?`PEsBSrHE4Z$ z5ad89Qxs^>p~64F9 zKMJ1^L`qRfQBpg348bqSmrU{^&rtWanIcbiZ7EQ%g;D_NlAg|Ic=ty|M~Dhy{5de{}y`b4)X; ztYzde%+=@NZ<@tu7q2tzrx`ea0i=USAOnT2d4Z_AeU@8ocB9qKDqVedilz4U^`?}U zTiA=Y`?HxVs?1iCnH{}@zgfmgn;C0==;{kXeVWl`w-~KvTl|z#`z)hL+@N@)Wn#W5 zZjYE|8g0@DYCi=(D10dAp6{IJ+e2C>4bgZis{xJm!LRFRy`=dHtVgY6a?Y(671N;I zy#n>|;+@MOhb|+((#t|)1<^mvpjD|o7r4jMpJGno(5VHb(sc$ijWuds!FjfnTE&(i zWjEItYn8<^*;HPIlah2Ahmt~g6*-`_CRs}I`C~k`MLB*1v#O-(7j`<4dX)T^1(R#f2}ZI GpZPm!mOb78 literal 0 HcmV?d00001 diff --git a/fuzz/src/test/resources/io/github/dfa1/vortex/fuzz/VortexReaderFuzzTestInputs/scanMalformedFile/seed-fastlanes-delta b/fuzz/src/test/resources/io/github/dfa1/vortex/fuzz/VortexReaderFuzzTestInputs/scanMalformedFile/seed-fastlanes-delta new file mode 100644 index 0000000000000000000000000000000000000000..f490a583197b43c5754b9264102d86c6714df1ca GIT binary patch literal 18457 zcmeHNQD{_06rGK0vfZwBn~LNkNDwJfLMT>5ij^QDRzymPlKNq@CM%6yH)(bgLg>dL zA|j!PA7YCiB1J4n5b1}M5P}4eQbfcLks=}@MWlq*+SaDA_r5tdSzq#eo1J}cD(;l= zzBhN~o;zn|?ww7aQo4*o79#qgwFV7~v>b;lM4?{{eJ%8mg*xbC(7y%=QHVqTIvNsM zjzgmLl@UU-pKJo}EB|@s6uMvmZkN+DiPA}HKTt8*pG3>jGe+BEW;E8O<{(tzl z8SWr<`j69QcEm3)xgO^({ztGlS8@J_{xrk+1rymlWoE~V4{67-Qmq>1nY>q8P9P2z zz<;^+?_*%cQ|)lMcKpWWLC|&Z@7Le9z76|s68kPJe;N^gRwE9Si@O1FH-wDxqnMbAN4&=SZ zhWCyv4)#sty-NA%ju%J4{}15bt*_MkHt=t1Z$I|kZR|T!dtT$8A`WZ|C=Pq!mfLlH_~L-FJ# z$$QP%cgqd)j~5@(K8gczxI%GwU%>h26!Onm!~7%jFI9FS{@g+QDRrE?-yy`^v4H>X zvkZB!CE)w-O~iq$;r;h9;=m5W`>$VoPlEro0morKKT6e4A`Y|}<{v-20{EXW{M~yR z`RCh!^G^r%-F?^Z-k%YF{N`J(el_Axqv1VdD;$`rP~P*J&n?7(Ou+ZwlNGxEUc&u% zxXk;nJKlP|-#{Gb4LA;a;V-oPBJ$5M!~Eliy9xX&!#KPb`|fVQzk63=-!&l)yo31D z5b!u&0;XNddxVyeW_b;#Y{RWD2FUkFP2zl@70{`yg6$icAPe3tJp*Z|i!1-qY z`|g_I?=CMsq`emWZ!9woi=DTz?^Xnye;&B<&##CBy8^z4jNl$}!gUXk`IjoMAr1@} z-oO0t_8{*)cuD^5vk`f(w#54{mff0=_-o678^7xSWDDQ+`~_cR*gV+6m3i96KWF*7 zA9wr{Uc!=s9~Pb)XYK!lf45&@y$k;$!@?CDTewwx&49!|k-|$@(iX1FuZsWtJ`i5Q zl837Iudt||7w0MXw-twN;R+d5e9eR{9u8LTU$Pag$OQObXDJR>^RJadB_N=v9qC$7d3{ zuI|pfx`Os4v+cQjUq5M-x(6MfS&ZrYtUvoXTb!P2Z(k;rclN;}>OVpCG}Qvt4yv1} zuA&;H`aZ1<%QvWs-JWbNpUURdF6id-rw_WKIqdD}$*7}q?d8&6^ry3(X?1oUd%dZ? zT$->iL&y6Ue)RTrAL!0^53pL@E3&&UmCI*R*>tX@E1k)wNXyYB>Z}LPrO4j{shmTR zGuNwRgVc4rCdKEFF?k&CEj|k@<$Zvjxo;i)%jb&9B0WAvc-+dRE+YFWp&F=N5Lj2^ zoXPEM5QXhCrtX_`?DGgl<@S%LO8*b&IT9&!6vyuj1D%Wmjz~-wE*iz?f)E#OH3$d_C@2)FqI77dGSbe#%(T{`Pz9wx zK~zA%qJTSB#0MZ+mNp!Uqfz3?m+eHHLrx;!M@}IRAwNKVh&+ruf}BQ@^D(xMB99?IL4Jxnj+{ZBK%PXNLe3&jBgy#; z+h>uVBfmg?i9ClqkGz2V3V9KE33(Yw&K$O{AiqI=i@b`whP;ma4!H(t`9V&qA1{Mr z`9V&qA8&wT`9V&qAMb!;`9V&qA02QkKgdb-<0Eh^Kgdb-;|w^KALOL^@ijP>ALOL^ z;nfdvy!zq!;nfdvy!zq!;nfdvy!zq!;nfdvy!zq!;nfdvy!zq!;nfdvy!zq!;nfdv zy!zq!;nfdvy!zq!;r+fMC)E${_Z2y*et5sH$Vv6X`+Y@Dsvq9(D{@l(@P1#Blj?`} z`-+@YKfK>p9zR$@?_2U@c1IrI`QvLA0&&f&k!}~rbC)JOQxNnvp)sG|KSbmU`>c=cN zmLKG#`f&*y%MWr={a6c*7x8nzAr4#_bn}l71>xlCXUU8UfHkf9It%s)4Aima&`_nQ#>Yr zhabWtGNYbl+#@Tp6r?wV{%k$+Ie+hslT*s+ojx(OPeK20(Zeb^J*blF>#yW=y6(rb zMYeTXy;ZvKQuuflZnHn;vh1eB87tXi)0y<5^w#m+&_{n&8d;O__;^{;o!zcSC6zUh zJ=&gZU9WPnw9y59I_{_cm}73ic;)}Fpij^L?n94{XW=&h_slK3ew8$RsPc5lV8;;;UbhzTc%iElt_H-?AG1cpT4ZrEY~VCv)(*>3y1( zHYjb4G##SHU-)}|UQ^F!qS@-yCRa~28?|BC3_oI@Du{i<^`TDq4gE{+_P0B&aoH2X zP3%)eam>)Z@y33gF)#N?SzYsuj8;3@yYQFv|03WeZXSoN;(sR&7Q(OEK8?P*$OD?+#o`;T!*BHx1yzWBwcSpx}kCRPYUoeiDwT1t&<4@%BryC^g0Gu838W!&5ovLZ_^41}FK1~&WyTyopc literal 0 HcmV?d00001 diff --git a/fuzz/src/test/resources/io/github/dfa1/vortex/fuzz/VortexReaderFuzzTestInputs/scanMalformedFile/seed-fastlanes-rle b/fuzz/src/test/resources/io/github/dfa1/vortex/fuzz/VortexReaderFuzzTestInputs/scanMalformedFile/seed-fastlanes-rle new file mode 100644 index 0000000000000000000000000000000000000000..f4d7aa3bd2b70442c9817bdd6b969d8171089f8a GIT binary patch literal 6680 zcmeI0%U4`Q5XQed3@{l6GC;&pqoSfBB1VmG6HzfnMU4s(<0DL9FqmP6Ff)kp5qzV@ z_>NJd;yb#^apB72apCcJT)E8BRgMc6`3uDQZg;~l+`8&QSJS^!)V*ER)!o(i&OHl3Dv)+bCiTc0r9Vtq8-YVARKjkT5Q@@txTe}eVgDrVS@5q;QhL$vDNb7RxymrzxC{ zc7-!=rovh15Y7(Z9Gt7riSrb?aK1t}E>P&fg$nC%k;28eB#`yE6qhMnjw=+d#8nDc zk4n+P2sHo-o^oicW_YQUA(98J`O2-fDaWu!o0%A_#}`|@fki>_yS+9 zz55yt zcSVkv{7W50OaS^sSd=nFrwESvl`a|OJJ&3ZaRo3_&}p*$jFOOl!;L%Rb=nE(KPST2 z`xX`!AmxH~xn@}Q(TxQ&`mGmXKMPy&XlyVg#sPm!?V>NoB*d%EUp0#YY*Am+#>V;M zk#@91+8Yy6GNCWW7+=qypV!Wx^KTL97GZ&fw%E5`ayFUr+9RDcNBuQK7T3VF{!!7# zeQ-6VD98EOmc{^W5}Ug-7T+up+vH-<+}ItNT%@|J7TXa94{oWhq?ba{YadK{{<0;! zJJNm`!Y#Twp*y;Nrkni4F_}!Asjt=f8(@0f^TgE6i@3RFBrRNrOD*SGYww5rl-2t- z?ud_CwxW;Yz6*(QsKTG~AF1f$oJ$XeIf}|Pb&3z?GRL^Irr={-vhx@jYk4!r==$a) zh^yPlgD5#OuLtX-zly9A;oR;V1vqAHJZozk*;&f%V#J8# zm~)lI8_TAP{v-Rl^#3R_FET4KEYc^^ArckYXvebt>f+{eg;#Id|mq zS?pgKuaMbY%H&2e_&OloWO}NQk-2}eHok;*9g|bz6XV74X_|4YZ0uONP|T)tnL^K0 zHX|nylb!H9xSz7MPNWNddi?&MWxSZ{c^-V~+ zB0bX*iFxHc<2k>lHQsD}a;0o``$)bdCupAIZqZ!A_mTt7#Wk=|Q_RnB`MN!S7j7N& zF?%$h6tnNB|GUFc<$iK&_h;TL_qH>)9LKf%ChOpD)ZvueNINd~!=g9a`=@YN=SYY0wbt!59vNa4?3GA%qxCh65qS17+Kt!p7~k?oKN<#(3z- zlks3Y62r-O@WjcZ@n8ri51x(TK>P`6{oc%5yHFhRb!Oi8=FK;6W*_To_wSOl6#5Y$ z4P=0Gz$h>VWP$TQ0pOOUJbbdoUc{XJc|Av%!S9`ZzfW|HreNfG8DO7zW5+-mb8+Ln z-vI3Q%=nc^W9$Q)z!WfvXn80qVCNzf#HUVSX&!r2*G*!m1Z^JoOm1+;4l<#9GUPyE)87I&9Z8 zxTQ0=mJ>z_+=g%$F%~_Mdel}yO;LAvUDys(z-N6e9KRh|*9yIcr@an`rG4S~PGi2- zBsvs(;=cIV($Vl<4JUf2c(0K^_S^}Lz87hem1*G;@tC9A!}fXv`#rJqyA@q^Z%4V6 z$m^<#SxTY!B($p?B=W^@OxHt(--`X}Af~HoUc>P#)hAk2kZaA~_9*Akt2t>85ar*tP9;U=p}AypDBMn@yiKgiGYl*gIn9 z?^*OXek(B}v7`An|f48eUDj?Xn%=PF-2uy9=* z)TApXe5#&qIW;~cK1ov>(9u&&@QI^3*_KCju~npCY7c)Cs{q#%Tl;dTU3sh@i)&+g t1&RL3wLsc2G-8;3iJ#?v_8cOgTsIHCnBa@2cwXOQv1lpUrFH!O{tG88UD5ym literal 0 HcmV?d00001 diff --git a/fuzz/src/test/resources/io/github/dfa1/vortex/fuzz/VortexReaderFuzzTestInputs/scanMalformedFile/seed-primitives b/fuzz/src/test/resources/io/github/dfa1/vortex/fuzz/VortexReaderFuzzTestInputs/scanMalformedFile/seed-primitives new file mode 100644 index 0000000000000000000000000000000000000000..145917170b2e8e79b59b029411dd56c3ff1a1420 GIT binary patch literal 4728 zcma)APiRy}7@wD{$+~XSEnT*03HDJ$mSQF95~N58A|)sB5P}CWdD-`3Ui-31ci*OI zC}s~iNbw*dr3VjEL`vyVq!bT5NRcAaqlcbKks^XpO0oY;f4^^Lb|=|2I^p+b=KE&8 zfAh_s&D7+{_Z3TPLw}_*l#Pu}Pe>SL3TKdn?iy=TTDD<^t${5vP|G+?f7F(1`fHw< z_%`Yt(*7RfTSR-TQcuvA($ZB{6?8*oRbFNA&7z+G86a2!*p)NuAG4PK7BfzkH(U7tOx(8uf!RWMfrLJ|F(%OaZEd*c` zxCtkY0Jq@41n^a#QV(H75x9VH^l|xb>MyBJ(N2MRL;v7ESv!w5)2>l~cG;qR*FKT9 z6F8um4B>Wt)dHV#X{#+yjUBXs@t*-`-|vnc$qKu{d@-;leW)-cgbW?KbHg zp95?;`;U&pZ4f4b34r?5znwVjL)Qn2fYtXB+AIjI8lEyW1V|qrj$=SrQ{H{iGoVhP z?dq9-h8_%)V)aq3j}Pkth*G0XK7Dt~6gCIf_A-cntbiLJ0v&eLY6hK8xs__Z(~2`3Q$API{;hqu&9V0OG$E+HZwPU zfOe0*fZgL5`K&yl6c#5>q%V154jJ3FwsPOwa!vBFBsm&_coN{cmY7;|*-WG@Wc?gmdAdyZ)#yG4FU_U8;plK}M=w&>jhn$?A-Ijh(*ISIk^ zM|8hz+SH3SbyB7+7mdy(5at2q>_2@B!-P zIcv)h#xMs$5UBSAK-(?|U5O6Hnf|(Z?0G;RMbB&6M`^=HS0CH9oHhDV5bXJ*eJ!_r zb$5#Et9ve3c{XQgGj#7Tb-$~hK6lCx;@7fy2NEz=Pu+6x;-^04-n!I0lq~4Dc(4-2$!x>p%#M13r)eZgktV z+DJon*UVCJbg`n|aL|uJ2qv*F|V-hz~i3;D6)lQ)Auqxd!OR4V`<; z=1aU1h0~3AHJX;Gs&2+Ownr-n?dh7jaa7*@OUTC zR{kb1y3IKJmNGo%zU2jJl*S8@ovv~?v$n*cx>#$ZK`m8#4bPEXop078fJeLM(XQFW z#YCMpJSToaUp92^dt*O z9X!58tY4jZVU(l+xO|Ir?%`KA$)1hVrJzbDu_fhfK&5`hI}cmD{g%$-G}Na^LC!Jm zi4y?N6Ma(UaK_~^e^7h^xqku%owGlPL3jWgNpC@Lw*6x}2g#e~jbhL}UP4lGuwgU* R51t+72cxI1s44u<>3{C@10?_e literal 0 HcmV?d00001 diff --git a/fuzz/src/test/resources/io/github/dfa1/vortex/fuzz/VortexReaderFuzzTestInputs/scanMalformedFile/seed-varbin b/fuzz/src/test/resources/io/github/dfa1/vortex/fuzz/VortexReaderFuzzTestInputs/scanMalformedFile/seed-varbin new file mode 100644 index 0000000000000000000000000000000000000000..26e5255107932984fafcb0b165a588b5085104f7 GIT binary patch literal 2544 zcma)8&x>1C6h4_|k}-{OYQr$vVa%eGEQT^dDKfZ_Qly}Wi&ADG&AjB@bRNEWiSON{ z)2upW79!%pf-AwQMMMQ%b>&iANa@;zDhDvo>Oz1!nkx5sUhk!Ghz^7{qT=u@ z-irWbL8?;3a{=vhKoOuXg|S5DOkB1&elFK9pnvb5$wcHu`E5?*BKofaE5NdOqG4oS zN>WCSMHkcSMZTOiO$%XU7lFy+`F|Pvlx(5TscZr^_Gz##BN|yh1763$amB|jkk@H| z^;6EJq8mjWlb4jt!d*9i?sKBvjGYI8KOc%}~@ z<^qMsLj3|Bw@mC)vH*u|=-vSi&-uuDgG>{?c^>5KrG&;t(e75}@-?TOTrB4r#+R_7 zRn(&{H5xJhT|E0h2v{y_Pp<#d*5zea50K8+g?zmDy0E{Fg0cQT*!q{NXy?z)W%Oz1 zm~()&*UYZxE``ifCuWXiJdG3>gD1oL@Le=N8;f&hT%Orv5XV6~RxPb|wbQ$1uJ$#v zpKQFMsU!0;2F1^B%B5(ychpbumCc(dnf332{^d;??s z9fYQlqxOm28#Gto&zq`h?x8hkILA7OlLPs${ymxSeH<9}Qk3RwHx9JCg#LZ3-(6rI z2!TsL6Q}}zV$tpc-vRqT2)qKU09?>tk>E$*4sa9L0xkkgpa}c{x1WH!z&;QHuK^99 z2;7_HHOh1-51gKB>-dhMYop*A`j4FL#MCwn(sq;#BXu3R3hrIT&W*Kp=+4aAGMx@O z+%p%Bu=Rs9Q?Oliw&&1xd@}XkKqVa|quFukAW;eD`iZk~J>9r|8tp_n8ZxZ;MvyDN zAKu>Si9B%Qa_hw{E)QpO?F8Aoin-=+FEdt-V;p7LRAtrKa`9-B^RxPH2K`%3&pqvU zb)#Jswz6oXT4t!E&L_v+9KH``V}=nb{DsrMFpU|iPP7xmt@aJATBxT#?I=H7D!7LZ)uGc%G94sZ&Ny2xc4vE#m;zpy9YHjTLIH?WM5&?rdD&>9|nE zI)H}Xhv|KF@_lC5+c~I4Cli$@ixHw#NJ00+B-&^1*TI&^L4T`XT$LuzyH$kb>NiEBLSXA73h! AMgRZ+ literal 0 HcmV?d00001 diff --git a/fuzz/src/test/resources/io/github/dfa1/vortex/fuzz/VortexReaderFuzzTestInputs/scanMalformedFile/seed-vortex-alp b/fuzz/src/test/resources/io/github/dfa1/vortex/fuzz/VortexReaderFuzzTestInputs/scanMalformedFile/seed-vortex-alp new file mode 100644 index 0000000000000000000000000000000000000000..14fc67ffe2aa6b2cda4e95c5a54b22a1dff08002 GIT binary patch literal 10714 zcmaLc37Ae*9{})o#$b%W7!xW&mJmWBWGOp(PaVq0EffG52C7jGDoXS#8kN0yOAK-(0h!67-KFY{?mB+b(PjDlj&!6 zuktm%&Nuib-{RYhysPpaH}iddzz_KmKjs#G!cX}bBVVX|$*=e|zu~w1j^FbK{>Y#B zGb6vK{L0_>Z~o3d_$RkAm6fSvF`JbbyHyrfVO8d`8uM74`K-a3ti=MxJcZ&qEaDEV z%X+NO9k~;CW&<{4%+pBRm`%7Vo3a_3vjul!OSWQb#yoArZP|`{usu7lBllz{?#0gR z!kDM4xEs53ANF8R_F`}D%l+7geHrug6Zhu;9>9Sd#KAm}2k~GI;ZVjr!^I;wlA}19 z#T>)2JcQ#oo)Z}JOcIxHGN*7VOF4~)ayn=5FwSJmGh1B7a?asg&f|PmZ~+(ca4urZ zvsk=@NAf5h&11Nf$MQHH&l7keW1f@6%XkV;43AwG{Qc|I@T zg}jIt^AcXl%Xm3so-4&y@oHYfYk3{7=MB7(t9TP{X3Vo%d@I-RHm>FEyn}c0F5b<1 zcrRm~`^D?{03YN-e3*~$Q9j1?e4HB?^K2A9$xVEUPxBc*%jft!U*L;;i80SB;#c_^ zU*{WqlW*~DzQcF<9yc@Q`9S<3KjO#S!cX`qKjY{8f?x70#ysDMzvXxQo4qoQrq_ z7jp@ZWXyB4_!utbu{@5)^8}vAlXx{0cD&EAKc?(zbR<2>pvsQdN@8F%hi+A%L z-pl)VKiBaA#yk&+ALb)`l#g*eALj->!Hs;9n;7#vEq;d2@;N@w7x*Gy;>&!6uktm< zJa35K03Kk;Y& zi@)$!{>FdvcmBaY8S|tnWolW>W+mpZGOMsEb6Jgftj?IHhPWncv4FK%$T}?I4y?<1 ztk0NdC-Kf~z=qs~jo6q?xGS5o8Jjca*-hM%t=O8ovklv_9rs{+c3?-wJe|aQu`|1H zZ+2xjcIQ6q!Jh2Jm}g({e(b}(+@Jl}p96RR2XYVxGv+x+d@zS_D2H)4M{p!ZaWsoL zhB40};&B|$37p7DEa7BM;Z&A#8e^X6;u$=QGdYX1S;lhC;atw+e8xNr#0z;i7x4%# z<`N#sqj)rr;Znvt$BB>U2|SS}@nkOJDLj>@aXC+C%yXvrES}ACcrI7)Jg(&Vynq+- zBE~$Ih%e=3yqs6?N?ygQc@3}Sb-bQ2&yC_$yoopS7Ov*4T*KSAmbdc`#yoe4@8&(c zm-q30uHyrIkPq=;KEjyiG4Xmn&JBEm8~G$R@hLvdXZS2*p6A6c@I}7Fm-z}`V^K)_&&OYv6=cONlq$$-9LmUotdM%!mo+Yl(~~z?{A$IQwS64V|FbpI zk1Ug+ApKIqD+#w8Zmig(L%vUwvLY#ilM>=i$qNp13-xe>@xzxY#6{_sUo~A1LxeAF z%k9pEhra0lj8VK@|M2ymwcWXpYu*1D#SwM_biTwiW? zZMe6;FQEV4cUVAStp7bS3n-oofBPL4Q0B->o_%J6gZ_~mnce#N{6NlPlVMO*j~Xqz&9LD|eOVs`SJLauGbn>MSYBE6={%Jl7|DJPldHJl=8Gj#de(Br= zrDan}Q>*@Ayg4QF=9eb9H)soc!93>7n=yMv#f*iaIkiQ5cu(3(X3YsL;d7Ofejnj} zwwdeflKJV!lfKW5$@NKd)$l&H-QOW!xGt=%u%Ci4UYOKF->S)f!~RT;N}20(lYHT9 zIP$XMddg2ebyd=DJhQIDDhlHd4g)&E_~BfBk}d4~&@;1UbJFW_deXLS|L`l9mAkZe zVL@J=`<`90s4zP>Yuht3%F<^F^9uASD=C{erF=nIMPZ%1+VR|iS+g=HlQF~kZj+QS zwFi@D6Xp=+7LH&jnGbyCjx%%2O3yXUFZ9XWd*;U~^E`6W-?h+x+kQ#tzB4;6vy(H= zJY3f%DVvjdv`hva)-v=?e&~{TQsVdi*C{vsyuyA>+LC_REs{!dWTlEzqec|>`wwZC B!yo_v literal 0 HcmV?d00001 diff --git a/fuzz/src/test/resources/io/github/dfa1/vortex/fuzz/VortexReaderFuzzTestInputs/scanMalformedFile/seed-vortex-alprd b/fuzz/src/test/resources/io/github/dfa1/vortex/fuzz/VortexReaderFuzzTestInputs/scanMalformedFile/seed-vortex-alprd new file mode 100644 index 0000000000000000000000000000000000000000..44c6ea8c7bed1d09e0171c506f97df0e81381676 GIT binary patch literal 16186 zcmeHucT^Nj)^|++h9M1kNJ<6~k(`wwiewOxsAQ3x1@w_2=Ol=rf*^xzz^6GFoX&slK&ck00_<~ zz{ri~B_djh>WC5oU92euh8JM~3aIT-T7VZ1(xY4S(F6>j78RvHfQWsE(Fl7K&;mKU zSVEtc21w>L0vHNvJ`6okm+>SGFH!dufhgF8CK3Q|G$2v{v@R_IO|VB5s|#Ajs3ZEY z>S%imA*w}Py_njG4vNDb;2A5j2*`Fyq&cs&Xf*CTH4crWBbuOHk-SWR2#`cdfCG~F zEZ(fcyy{2;q!w_C2E1q-nh6MNV|XPIw1Neu{0I$NG-bN|BZ3wnqHzdHK{Ti!07N!g1NsFT!QP6 z`3L~lB-em*)QUU1lCP8vWFrF^wks3_2p3#jh-TP8#TU$#InkTsjstwn3xv=DPanz!)i7 z;_U~=8MHU4;^{(TKglU@@O#jS$9qr@pPpCgYn}5`Q`tWGl8m8!?SgK9v_(z!^`6%? zoKt8NI2+#ZDwJ4qg?4jcu~umGHA(Nea{qSHU)<5nM+IppwOF?JP|U*z9H#s?e|^cbc1BvzdH9kwM0=~TRKz`dNM{Mt1vfF8s$>A+dCkR zzUcGE{`G8U-cxF3gW=b0(~U`f1YKwMCY`vBdq3+Z_wrqWnN5qbF@7(#@&Sbuqv%Zj zibWM&NRQy~wQk{DEmEI&(&U2U%>jD2ZlIGwa~_*1>$fX0*qe81uWr<7xoA*|sCO4meS&&^h@)uBtLa^2_?Z>cel9_wH_F&L|q@!2fT zaUVOd+IuU=J?y<%{VL@#;ea6)`Ixv|M4V(V@52Z*MkHw3@Xmm8C+q6XoGYI}k*q>U zIV1KKvb*h+d~ej3PP(|p*j-xpJIfgtb(s5>Mon6%DQ8Rs(k}Q>FcmnsVaefGNSdiu zdA;Wg><)fCm|N!vsP?-AYM5#FPIxW~mffxL>gmAYTUdPd(ayXNk{{>* zX#F;Czx^$_risMbdS-L@zjLyp1(-p5G~f;Wi0EZQK_(8sJx}p2TdKP z+VH;L(TqbL)sgYnnbcy*VFS?Lc5{!;g^dsLK6w@1TE~U?4NY(C@=7_XanMhd_SmW; z4tqegw7EpwvVO~CwB}Zs5{}M|*XO5-N#wncTB%kFf-YYwJg9gh-{(i49y*iE?q-yF zne``IMj(61QHy2fr@#DIGBpU~bmyf|;j^+Xc0@5j{I5Suy`(OuMh!gKih8}EL!PHz znxWfJrk9>8?2)ARsJpQ&`_kvA`ZLDmgKIRZFM^DyILgOTKPjx28mvA`DsfvClm4!? zkWu_ZOVzylLY#`~vvJmQj$^(mYQcG!6yMX_ApqEO(QMk$-nd~oUrV>QU3rvP!TIoZ zg!qc~)VW-tD~F>#MmWwOB0%2v({h#BGy6uZk7VbIiKdbRCt$=@Beu~`outR#+Ls*p zg4&2Z@=A!B5$#5Q>v=wBGTl#_qbJHQb#>U|oSsywnIVl7yolQ`@5=a~lwV%1yA{jH zo!Ax5npqO>p7i3@i#%dE5oGB^6SfuBn)Ek)3rs`6i;tbUiu~83uM62Ms#LH09gdL? zY=6XP6aEqt5!>t>d0{HSIP`6ql=qHWITm<44%*qDy(Gzh2U; z>%LpA_hqm_;mW-jC6)HPtQ3!Tq2~KnByIdDpQI(Ng-h95_tIX8-j~XDFp(Co^(nxt ziXh?s435V>dT{-%ZL;e0l=CeuZ*(07;)&tJ&9Fg;Y|1w+QAz6ef69xL3{jjM8rGi4 z*-x|An*HGL9a2-qYy$=k_r09$=hSao7E8ObogY5h^~jTM*~1E4qS-C4m;JhI<5fCW zJijw-m2c z%GV=}{CsWcGi!{Tnq(gOef&(nDMJ@zp41hx))jtF!-eYk9!JGAA9X#?!d)gx-L&!e z6AM2dI8)FK;HcfFUOdudv-4Hkedj5#kLRrBb2T^ARV*CeZQ-&`OZL-zyq!heW$t=F zN+e^5y4dj|3Mwb(6XO)|Jrb>VuTt&pL2U%P)1OT64RUBW+uwURyMlkZTx%bJTzla% zn_Cy=G+N>aRieSDXDAlPHk3zzqt*4hSd9;2?utFDXO(z+GsuwPCLgEr_)6gk;vi<) zn)Q^XR^^j?@1@T6gnGFzmEmXI1A_KuTl%gmckp)ERNCw9=~w+~az}CO{Jk?5%4Jo% z#oxF18=E&hUHT40p3N3GR)gX?HKEilgOc{FHyDlMhNO6WWD`K#vaz+jK8};Wt-Tsz z#+0pOX7tVZe7_VqBI*)4Jg8P=9cJrHHysh&gnY1hVbuU}I8(;Frh>H9Axe-Zb!#Z94|7b9I+k0?8W*ywka(7h-3xQb4m#OJwxpH-F*iH|BkmcqiK~m5?3>i*zUJ&4?mg=Ts|~L<&xcl? z@fg4;c_6XF8SU$8{SL{v_*Q*YF~c)T6-)3;Nzn02q^&Q}tq%;|kcH4^A~eaZKQk_Nhz^EcZaiE%`c)X=F~ z1^|v|B%U8lT8yRI;&r|f)35cOyI(0i&g#NGmuR|GJ!b1&iDij40r8(*>5jI`Yee{+ z$jckz6Z1V*J|KKJ2Vi$Km(bkn`%E=pL*J4{JdeK>qKM&H02t}k4#x-2I>TIdr;fCB zPkekh7$t`Lp(Ux7mgFbga_06l86N1aV9%5OgmwM}*5ix#`wgBiRO2`{S1yDJZpuBc z)IR0XW|ikML49m4KcCYw{#(;94PMjt5#hktK*6}5lfq=px@|Aj9qvy-xpjBSy6)}P z$@-R5h`HP<>Yl*h82{4V9a;a)TaDUx)%y10=91)Wsh1E+=gJzzOKSiMdaf#fFH(~u zLkkV`{VWyah;r^<-fw+#c%CVVSz);jNOx+TGdqwLpWTAwLx#xPmb$bYqbDmZsUEG)b}n&Kvk=kt+e6VyJ;=g^|N zkCP6c@BQmz?))Kaw}WZ0FtUSUW+p?oMS2e){u*z9)WBl?CSXt^va5?w$}Lan|o8Bigj|Bk-bxaW{L>o6htGq<<E0!l_$v)f*evrA%nN=JD1uQUkt*Q=*yS-se;GydPQ<#KEY>6@ zrGxG5ihjcYK5av!$f?NuQO@UqCV^&EjjdhHCtgR)bReRcWtxZ@euE~le4O_+w!r9WyIKEEPGk@*)3E&74?W;9zJJP4mqtEaa`b)f8?$L9F`Bzz!F0CPSR;3a>jV0BS-X%lniLMu zrL@xgq5cIA3Bp$N4!|f7G2~ z^47@f-9;nQKa9N-A2wAncc{|HC%Z;SMan*@60;W&mir6arL^K%?Dk#TY}UB^zGZrv znUGP2Q*#=N?xKl6v?v{$2{e@VFN;pb!jCF2P0FK9lARl|Z-peTj~eV{0JXWAwn{H4 z4Rvp>=QTc($ZHpmLJ>%m`zI2O?B85JjDJvZ5I+}C2%P|k~ zbC_F0uHPK}Op2x;lhtR6x7|^rF{O(AR(3DMOuei4_4cGg>bDEi`HxO)nm{aKjmt#) z_t%`RtB#Q)`C(C{^`d8u!Dy@M_})Yfp~GRPf0iH0pZXa1Ht&JN*?Nw(YTr$>LuT)+ zY*>+Zcs_>8Y$XVCq0x?EJ|i>gf$NXhMvWhvoJbiD{$aRoDqD`7IjRHC`jUo$Ynf3q zrB?L8BSYQ%3pUf%(l3rsD^9$lF*z}1-?CirB`FVp&8?rWw>F+zTIoFDK2c{gPihI^ zJ4lfG7JoEbq3OW)51-9C2F4D3k2HL!+;Ut0%A7zps|-#@{#BOj&wQb%*iRy~E=FIc z=%Nox6%}G)w^p#6O?Cmok2iR>FX~*sx3X#I;SkFZu(35&T9JSK9#8I@Me)~_FZWzL zKX0IY@+zSpf(=-+E^{=;NiPPHYZqBheUR$1AI-m6Dx960u z3xJTTKW=$FFu(NFFj@7cZ`7UmkQ(*#dW4&NcVq3eGj+jAsg8xmIL~~qb^?dc9`F>Q z+id)HZ)OjJ-1o#iNK{2(2n+a_7O2@8)4De?F8jcR)33{<3_l-#Ruq)RZu!kU^4ee7 z&0p=>-gv$ri0PG|h%vhRs_zsJ{QLls`5CW#7H_l}@?D(h1a&O5b@HK;O+d!fuK0>s z??~Y=)5#P=$`_YQW-0{^bFGhJDNLV)l(MG;ixz!%IG}?%tF0%@TBw%)a=N(R$oidS z6Ye4vZSggc`nf5(i2*sx-kaeWfWTT1>aULCS?6=}Ztza>p1W#(!PnXRWRZ**^B}GC zvprp5f=veGc%aoGwm{7Oiv5UyJ;P+UAKSyP_95C8L|HpoEt^?sdb8H;%j6ru+k2wc zd8p}QZ>wy3Yc^<%j5HqUpLM&ixBDq3)veejP3aYaYGQR0uh&U?u+O1tcy5Mj3YVW^ zoq5UQsj!-kPks(L21~B4=QW)3!g7pA*>ZbYcH|DDckN837g#z8hDylGwDa zW9Yv)x@&Ajf>Eb+f>Ey_{Y7sZV~p2VtsQ_A1$#oLJ6)4_(|gXV`2}`Hen0Rep9;zH zfYE$KP+fT!S?yLg4bR0O{l&T9NM;Yp#UKFoAvgk0WwiGp#Fwj;8ZE;Va~8uJcsE}g zMb%k`Io95dpQ?&GoEa=Uk&_(w(7ZM+KQFO-Tt{?3C3sJvZI|!f+pTV^=?r42725@& z?!%OyGsmP>>C+EIa}Dv$wjGOz2z%a`rTF+x_`!PTzT0WDL1RpPP-theuwXr8)ogl8!EU z#jnV25`HtR<(n)gWga~9ra)RykD+E%{Q|dt-1mMZ>(#S_&dEW<_V-z;Jb9~apCtY1o*v-Cn0Ho-Uy~YZJJ*OUV?AY?bkIg&pV~r@X5!mK>_(tH)!9p( zLkx~o2A(ED16eEsi4C~Gg;8Zdj6C~ZaO}{@-Vah|W@9%_z<@!-=kQW~`U~pabl>v> z7uz>eW*1LiL)Pm7fdEfT+- zTn~s6&v)I@WfPkk6J})Er&9hZ?NL*S$CO>eU2RnRy7MQnJfU$q5gR zoa++=Kiw)49LK-4hqZK`sm44wJ96&z5V4s&+YZ$E%$xNErxMuOzzUhjB$L+ z8)(|j3YWSzkJLG~dmWWq>3=1o%7GG4%Q@*Kw#N^*arE-J9Z5Fn-{U+Ua{`~!VVLNB z7kQbnlP-k~8*|3*FOFgfXq^Bcm%nia&X^w zhFG)duSE(9aye(K+HdDM7;K0uuh3G9>2c=>KEHFqN6^DFzzBJ_-#-i={jpc$PK3tz z$9_BR0RV&rGA6~8C$~WK@gK%HX9D49064R57a^`&*@@mgxooZcCHL5Dj(w2!kHirW ztj(CZx0b90nt#|#iKK4d)rsjjGP+{T2;9rBC=0&(FfQe2nIBx^#xBklHsibx;ai+< zXF1^ip#&AN>+0S1IukA51l|#W3S@$|)XID1m@GT#3Ldt%9*TtLgKdL|NHn7Of#PuM z^T8cIvH9A4{1sx4pYZm!9r-kG#)zL#sufX}%9fTgd0ohp3_Y(wR8gcdp)(@)M0mUo z^F@M)fd;z8DXTr_5#>u8)sjn<0%lczNwP zmONQ(AcplCvy+J$5sD@(NBL8lVq9CQB9C~ra$Qa|p=*pOtujue*u0GUurPb_)lV*; zsbkt2bc2VlV_NT;N?b752$yeCy&QR8d?#*@qO+CC&$U|;%{MofyuLfnHPg}a3!m0s zfiCO*)oo#CUi2Q^_8sO5m)xzS!|B0y%s>@^)jN$@6fia6JlPT{N2G49&(|@N`Nqn7 z$~;4^2sg;DHaF<#jJW>fc@gdVE>>DJSK@FufaaOL?7WI&}fq{x&ZdWs3aD|JthHwA9!t0$ezDx!P1EB}{iA^_bHHWG2F_+(T zhg!FlwE59S>b@^HcsDjVaRPai|MA)97UfLnmIP;3-l^r!l|{O@58Y2-Jm|Ae=0G^j z;h?koK94Y&eX?#pB3JA8LPC-L>I;tRZ~v-zqS~YTdLF#`dZi}HwcAOut75P_VM~Av zkUjH)JM~aDHjM%AuV!g#N)aHeJs)JQ(qKh2zLBxyjx28}G|%B3Dwz>YLS}479{fBAeck#RG?hm!$u~s$~MaHfy?U`rY zG~+B|Jb4q{L-)~PG$^4(OT_Q)XwaJph$?1)e#&JujrK#I;51YIe z1K@3X`6UERVnbS#bB$a~Vj8SB+T9+#C7KkAO?%TbrK&Iw?|f?9oQygzWqv4r+4=iZ zv4k6zw&MD6F|pV4uy^mSqcU~BcCIT?_}v)Ypr(3xR%y4nfen%O)^fT_psYz~;>9=D zjH|Xo^dFjH*=0s~(G}w>u%Ckc&;2|z3zy>G9qk!hkUa(ORc}S(8NgV*xRMdOA~*9_ zc=3qhzg^9JXD#%Bh3bo;3b!wCRj{@HP0IyOV8ELht=26r( z41izP^n>maHQ@GUn!y+&zAq&L(hj~%$J zI+rr#OpZY7TIC^~#2RWaI)0RKcn-S%(9Bs_ScdK|W-p90=92H>{#UX26W1RE{vhxN zfj9|Zm&@CSkaF$5OkW^$whseyd9W5&cHHISv;TQ0Il z0d{vq-5D>U1jh)W#fE+qpq!v%&E-W0GCymtcBB4YHx!dkXdPQ&0OPwPXBI!v0FCvj z;Kz0t;L0p2??Ge$72>p&{T?&`pH=WUZ;J-he4F$&3;-qt=#(f}Z5 zdwUz!<%P5?QK1>})j7s^OJu}EIP zgyf~81uT#aI3gTL2lgTe>HMw_`^mJpq9ZBvKN>SBfth7kCM*GIIuwH@E0dipQ!dfm|31z`&A}?%U}qXF|hN$ z=^v^%gw!uB9D)}>=_n2XyZgPaEGb*CmtL(+=*Z!4<+&?{=HqIB-yVH_;sef$GmLL|a`yj`53B1qkl!IR34%=wg4ASl`4 z!=D1}@B;$JSj<>EP`WAmMl_(AU8U z2I9+({5Q-UJbnG0;VT9A5cqn*?fk7RUJd~y2ub`ds6Hg7g6sc2{^5AwdT`%^#~Ymg zDhEU7ur3vJgvT#LC~|!aLJ9wXP@(J<|WMalL z{^A9BdXkxtO}PJyLkSM88NvpR0geqOIENE?@%ga^V;f?{zKfi*6{uWN zQ&CW)MEn3sN|b_%ih=@BB8vP1%zOKBcRDBH2avVi?aa>S&d#pI$>rUG`AAG*O;Uoq zSQRQuVv%>{+`)Mb&?@JB@ZZ;<-HN&|!r{x6TQRj7Q@3MkEvD)+Gs6q=0Gn3^h;mg0gfE-9;g9y z=M`&8zepNX1v9lcn{dj836npXPN$?uGj1NJAac>f)%JP0hIPiadekK-XAWLTL`>l!aTm4il&vQI#AX(nh}Hr=8S& zXK~Ya=qi0o&%kq=49&r9aM&a*Z|AetwVcOj4?`cLNFhGPU6VG;{tkEuF!s7C_bCHvGZmu;y;&X%(00dq5SOIhv&PVD^^DdQ~kCF9-x0xSdEqdi}cvVvl= zo(+C5Y7mlSy(Yjm&;XVI+J5{qopNR_<8T;t4{yE-gKh_`d^NHQiLBFWNAk<8Urg2y zqTvV<#6oT8kI#Vd*WMikd-QnS$Ws2E_gj(biDT#=0TW;wxDPA=F7VXYT>@==hQUG9 z3L+UB*=f1FnT@v-hJE=kYcp;QcX|Q*f1Z^MS_dk9`DtW)2uvRP!``45^~POR*+aIh zHFHkwQPqs~<=rjeYwl~wuGwdoere0z;r-!k`7vk@RZZC!Xs_dzR9rBQ?UxE3+M+sD zM&-z9 z)sM7F>wz`<*Efsasl(dSZW;K2JnC@z+WeuOfZxUXqTxMNH=ZEg9_Tf7K;+t#$B&v1 F{sJNg0ptJx literal 0 HcmV?d00001 diff --git a/fuzz/src/test/resources/io/github/dfa1/vortex/fuzz/VortexReaderFuzzTestInputs/scanMalformedFile/seed-vortex-fsst b/fuzz/src/test/resources/io/github/dfa1/vortex/fuzz/VortexReaderFuzzTestInputs/scanMalformedFile/seed-vortex-fsst new file mode 100644 index 0000000000000000000000000000000000000000..f60ef77e1888f1113f8557e3703aeccfab2213ac GIT binary patch literal 8585 zcmdT}Yjjn`75?VD2=V6T&P_x_Km2+h?b#M}!awH7VK z7h0u0@P)5DOiZFxYt^c?Xpvg9)>@_3TBO!mrIuQ?YOQqdJ#z?!!zw?zR#zu$W}lh; z?Z?dd&YavifPw=T9()AogFN&_KK4TbKmn2{L<$;d6rmXVqaO}Hf6^R?0XPT)F$jZk zFoxg|48<@EC(WT4fx|EohvNv0!e|_cF(^SPY040y9Ai;|N{mAlsxcl%;b_tvg9$hm z6LB1l#}_aOCtxzZh!aV35~ko}OvNcU6{leuzJ$~9WlSf{8JK}HF%z>e8(+a3d=+P5 zF6NP@2J^813sH+j_!{bv!Pz(m^`vP)BNn3x=b{-|e3+=QEP3u$h} zI^2faaR=_iU1&io+R%=>Nz;K&+=DK3qX+lmK70@N;{iNKn(yNw`~VN*5j=_?;xRmq zC-5UYNtz#HJ)XkT_z5=P8EnK)@hpCZO{94a&*SHK0l&bD_$6M#ukbQn!DiC@8n5Cv zcn!bB>-ZhE;0(T`l@`jpGv3#l~jc)rL;<`B2`S9{Z%}#fPN~T zSHJ-(o>xGB70)YTqKfAgFhIrg3OGo`^9mTK;&}xOQt`Y32CH~p0SBviUI9Z?JgtNz%UihD`2>a=M_DtDxO!s2o=vO;4l@>D`2FG=M`|cisuz@go@`CFiOSq z3K*^8c?H_9;(0~f*$b~|b@sw5(0&!qE6{!w&nwV=70)ZseihFv(0&!qE6{!w&nwV= z70)ZseihFv(0&!qE6{!w&nwV=70)ZseihFv(0&!qE6{!w&nwV=70)ZseihFv(0&!q zE6{!w&nqBL#q$a%R`I-oya(fWMSIm=ctua+UU&t0561Bd@*a%i734h_$1BKtFpgJ{ z_h1~aAn(CAUP0c2alC@O2jh4Jc@M_%3i2L|;}zsR7{@Efd$5;R*%Es3PzlS8%5G8ON?Q*#01EegyJAuVhpn- z<80p~QezVfLz4*M1wj7Afcg@E`jHgJG%gNE+J2chpuQwVGU{7DWs(^AiR^4Yl*DN~ ziW~K@ z!bP&mItZ;SvvQo36;_s8S!rdpm8DjWwKDR%uQIfbBYbER2+K`oJEa|`^%YH(?MHTW z|0T7tc0z<3$F8ICFtq-xB)wdPb^@h#!WC9lTK|<+&;EFbcqdC0Rv17y)=a><%@9sB z(Kzu@0`J6^n-wHKGG1*LF0>08R+~J@k9ZDkBB7mFd6b{MG1(H6AmJsU&^jC&xME4qjodMMKEy#-W&mNSl@?!S^QyFx*MU_mIh87#5BU`YNf{yk(vR{8tDLX!ZMS8SZdd?+^%b7v<}q*3@tIjX#RHHI1lzr&x1H3Y4a$P&{dXSl#g76Rc0cz zADDi%nP{0^uh2}RCN$}0!U~&bMYLYkg0$&o%B_D3%TyTmnKEORDL2#0R9Js@d!t`d zrkLFd^b5-rvup8xGN#yVQ+@~Wh}{JO+DdHx@O))o#3EWh1Gme{Mxf##R(cDyXQ_0OS)My5425ScEqqt!U z%@&%$n!%dE1q?1=Z~=o07+k>M0tOeb`vSHoV2dP!lMGHWILY87gOdzSGC0ZLq=4Cd znk~|7k!En3!D$Ak8JuQtn!#xX7c#hz!G-L;kSz+?qL9Ib3@&7FA%lw;T*Tla1{X27 zh`~kdzKAV~*rJHRDF&w)oMLc_!6^o(7@T5oioq#%pJIz*wkT$BF@uX4T+HBN1{X89 zn8C#iPINFj(ZK-{9c+}i^&Yuqi$n*TB+fJ=Y?SB_s~)ksLB@N;s)u7GdN^3(4l^#c zJz}+7MrKHRqO|8rJ6YPkLXvVtbn)XG+^Dv0BAywv5b@ zwneO3#Og{JX)xmwu|*;_%SgAx>XumDV%05H-4d%?^6VC?Zn5f?h}{ygO>9pQKWn6I z6RS3{Y7?sq&A3>#iB+dqbxOoeiP$L-7YVXctU4uDr&x8el@^|8;fWTWXyJ+O;E1|| zBWhub7Pe^Ni58w{;fWTWXyJ(#o@n8T?%{~Kha+lXix#$MVT%@?XyJ(#o@n8T7M^I~ zi58w{;fcPJXQ72HTG*n6En0Y@g(q5gqT6|PT6m&`Ct7%-g(td;*F+0jw6H}BTeR>* zcgdOvPqgqv3s1E0L<>)}@I>Dw`3PIIutf`7wD3d=Pqgqv3s1E0M7PMA2v4-|M7Kyp zVT%^FXkm*Mo@n8T7M^I~i58w{;fWTWXyJ*@NT^!V@h# z(ZUlgJkgzU$Av9g*rJ6kjSekaLe`F_P{3KInPi2GFJkw_!aU(kLhgD(?pZ=sJR$3q zkl7|A*My1GA(_eEq2a;vFS{Sc8kHOHW4Eyhw;s2*)s9lQP)93$2d^&+H zr1Rzj>2!G*oh2VhC&%OH*pe|CY)D`M#b)C9N-KJVphw4!esE5_# zYQ1_!ZBj3)&FXdarg}$hS0Aa*l;h+%NvGHu;0$p_IHR4=sd6Sblbk8eG-rl0$C>Zc zIgL)vS?;WGRynJkHO^XRozvoUI`=sbIgdH(osG`(&P&d#&KBn_XPfiB^Re@}G(hJ5?7J2nvv$xE<&|B$U?p^I&@7?0v>D}%1cn^AydQW;AyiMMV-e&K0?@jL=Z@c%A z_nGJTd4AF__6PVw{1N_WKlH2o3H~I1ia*Vt;m`5s`*nV!pYxadEBsadYJZKt)?eqh z_?`ZJ{zLv_{(66-|GfW_|Ej;mf6L$IzwdwSf9`uhevk_K1p|Xaf)T-}pfsom#s|j+ zlY%M1v|vUsCzv191&u*2SRSkhRt2krHNo0oUC66GZUj~RurnCe`9!baBbQ+F1j_DgAgEC;Ql0;zk%aKy_A%zZosd9XFKgq(` zmeQh6DWH!pr0aH4sy`oRrs#7FV-xhp#SzJQa$&nBs@!kcv4c+My`BDfjNP+6<3>{D zxZKb8@`&(cD>i}HGnR8&DP#W5G0rDJvk2@QOdhU$<}q{QRpMCu0m;M~E;i6}t zfnRNZTJebz=R1Dr-m&&qSACVa$%Njmq|`ozN~zR7#Yw3lXQULbz^)9PLw#N)o*|bg z8J;PZs2@q56_=?0?~<7ramz51xyNhE{_B_yRQQha$E;~oId+xYwLMwL%w@eN`aHR2 z7tEgiyNz+w75~A{)ZJ{DG4pmC<9HiotdFpT`&>BZ6;wx3<*~K9*|C0Ko~&q2hP}OW z;&n9a?H%K}?#bR>J+s#^nt(ftVQGo7w&O+QZ&6J)S6|aun;kPh zlWVG3zzMI3@!3?IR>e~66C(}IfX6@OIbJ;?9l#v02a#9Z8jn=uc!u;o_VhTAb8 zg(yNXN>GY2l%s+zm8e2BYEX+h+<^tS6L;ZmEM&_)xEJ@~emsB&@emeaF&@SvSi+V^ zu@sNtaV*0VcoNI;6jtDAJj0ficowVh9G=H&ynq++64u~lyuy}Ou@GQiviblA;J` zK+)8YVyF={rY00iO{p0*rxt8!Nx`}TT2ZjBfDKqm^;70{W2bp>>xU|j)m6s#+rrxdI!peqII z3g||`x&pdWu&#hpC|Fm(sT8a$;4})>70`o%bp>9pf_25(IgYNlcaEbg@Ol-jEAV<1 ztSj((6|5`pdKIiI@Ol-jEAV<1tSj((6|5`pdKIiI@Ol-jEAV<1tSj((6|5`pdKIiI z@Ol-jEAV<1tSj((6|5`pdKIiI@Ol-jEAV<1tScajf^`M7pkQ6W+XsVm#cS1Zbj7pr zIJ$zj4+iN9-aZ(lD|q{0kgnkEgF(82w+{yC3f?{#q$_y)V34lh?SnzOg0~L_=?dOH z7^EwB`(Ti+;O&F;x&kT34MjvoHHdB))2MNi*rv^zw`kewgw}1^p4jfB_8mHQ>f9yn zPWx-{dH`>03At4 z(2;=yb!5<)iDxAx4<0f!WmxL)vqzkB?s+56zu>}A7hQbG=rLn29e3H~SB$^%UlS&# zrDbr%X<3sdPtKW~lf@sW_e{@9FG!!AK07T-d~PP^=uk>by=vODf@uW>x#IH&#m8r4 z6l4^nXNk{`_y6y$0~7Ys>Haz#pu+?|?4yHTIyC9NI&F##^wSz`!1Sv#v$Eo|b8>St zh2)I9{QUU*_}=-0gcXyUIqRC)*Uq`_-`C%87Os2d?xfyglqIUa6O*^-{SRj6R(_m z^m*Bjw(tUJ4JwOlAL$rt% z)5G)#EulwgDLqDy(=vL3o}}gU6s@4A=^0u{&(bP-j-IF0^a8y|FVPx$nO>n+X)V1* z>*#e_PjAo$dXwIww`n82L+{diw23y;`}6^QNL%P5`k1!THu{754vRJ3ZSVpJp5ST#|xs;O$GnyVJ7rD~;4P_0!Pbqn7P3i%ErVYv?H z^V%)x`Fx9!u-4`e_R@7WJxrGy?uCVX50dgl+Fr;S^N|ZRQn5~#=&)3WWwxA;RH4)5 ze5(?~N}KkXRoQfsjSclmR_k<)4(IcY%x5JrA5CJ0nZ!+N2Wgg)XK*F%w_0L;9TGDw zPqgTX7CkWxJ<*~kTJ*#)^hArEXwehH(Ea|X)wR^>I%+j)tsNo5w^rlTYP4F7Rm=BJ ze@v4t<7CS?IgD|#Wt?mo%T3mNZTcYDvP{<4bsD)&gS{68ua>ED7mJ;%`B8W)8C02y;%29 zu|_Vo{EKye7i;ul%e+{#E!Jqo8mmNOm1wLIi&dhrN;Fo9#VXM(OEhMQ#w^uXr5dZ$ zVwGyFQjJw=u}U>osm3a`Sbk5IY0NT>S*DT7G*X#GD$_`18mUYpmDw>dGpvwb!OWP{ z&o#+4bHL0s$ulEj=9lD|5i#?tLbI&UxD~eL6&kZb^R2LaD>QP2=3Ak8mh&5)zf(xe z=P5CxQHlFJCFZM0Vn(A9_c=?<*QmsN&Jy!`wo=zpX^|^6Zly-8w3w9|u~OqzYP3qr z+3arRJO>Qt`gtX}hI9RFl3c^o%qGb*OwDYn(oCx~a+PIVrBSOi<0{LzN+VZkv?|Lq z$@XrNWtwCW%NfBiO|re4WO2oAI49ZOO|qPmY&~XIE9diV{n(jZO>)iHC5NG#9Zu|K z?92`)dH&dy^Lwt*$Pn9#Az}3lvFJm>(1%#=@*XVvfgqXw9-`5!HFmW|uC}<D zePw1uo&COk-uKnD7U?=f<+hJO!e!EYb!_|-p--}!s+yM903@6O_{Qu%&#KHreW@T>fl{0cvn zU*5C%rF}M!;XFPem+^b~0>1w|$gcp)_~m;QzjCkTH|>r5g1v>`tatD`^3p*(;M>d%G>;1T z22;)Vm3#SqvV`v=EBOBL65lp9@ZI8lzDsQ9+ru8dFYMzx!a=?j9OkP)B%l17@j1Vp z>a4n{9;&zMug+9M)Npm4x=4*xSEz|9T}@XxYNnc_=Bis&kt$a;YJpm)9#D(bqv{E@ zLOrWit2Jt^dPBXf-cui{t!lg4rS_<=)jsu;`c?g*4)diW!ijboJI$O{PFtsg)5YoL zoaV$keVu_$vNOy%$GO0{#2M#Y>7+U7&U7c+$#br8u5)g5<~j47Qm5Kk;N0UphD2mwTVP*j?%_cUQWr z-Iv{U?gn?GyV>31ZgY3KyWPF+x9)!Tfcu;Kr+dV8Ls6laP;970sCB4as8gtGs7I(* zs9$JMC@GW@8W9>98WkE7x*{|oG&wXalo^^4niZN8x*>$l#&!A6ySV>S*I9SP(ez{a z#})Z;PNt$v$S96aVehhUz@BFf7Dy($?n?eAWa{v*Xg@vPO9Q7fmN-cg|1-{yHRg#f zxc;PX%<+!wW7!M*?67Z#al9SpMT%{iiY5MyP!@^^uWXFRS85U7!c&~onaf_%i;f)O zcOBQ_OP;{X_foDqd#P8_(QXvAW@DoFO@#Nolod{+yu6Uj6FxEAiU{2XsUQ4fr@=Gz z$oCQ-&wd#D9_*#4viKpkuRAJFaJC+m=WyQlN99Soq-Uf} z{%A^AkIg@n@iHqcq~*rBoGU+m);no;FZL(1AJoOXB!)jT`cHlKIg`u9N!iZFFKZr8 zrwPgb+r`p9sx+2j|51&x)NxEKNxz5dTgFBDRoW`f(2_QbGieh_`^1^_{|-ghMwzkv z6SufXtLteZ26i%L3)zoiFLft)<9DovMfYNNJ$sbZhG?(#QF&79-2Vq> zC-rUc@*+KrN?M$7p37eJEqM!$`cCZsWL}y!`8uv|lk}snWSEc{hRma9$dMc>jMGmacZr}qAOFqlJe=djVAOW;3c@cUsRhlAP2 zv6t`j4j(xpq41fNlgCfPRc0AO`9x)%K|WS)AI7ux%|>& zZ8_PwX&HHJTYs!AD=pva0_@hd*!ni)=4J5j8yT}0R#s)#8#8HV7+l^iX7-fyDHF2NW;cr(%yw}>_KfT> zd-Ld~`upsenVAzNGl=L<=0gwmLU9fEwQv!R;v^LB?|))3?IMq};VEVOK4|Q<&F^d1 zTNmZ4@N%5g?e}M>KlA;*m+yqjF6PmTL8*(B=AUXgJ&nUVJ9|lHA^j#J$XMKf!j67E N@gMVy9X)RN{{YrZZ)*Sm literal 0 HcmV?d00001 diff --git a/fuzz/src/test/resources/io/github/dfa1/vortex/fuzz/VortexReaderFuzzTestInputs/scanMalformedFile/seed-vortex-pco b/fuzz/src/test/resources/io/github/dfa1/vortex/fuzz/VortexReaderFuzzTestInputs/scanMalformedFile/seed-vortex-pco new file mode 100644 index 0000000000000000000000000000000000000000..28684c3022cbcd5f5abf423ec27abd2af4fec566 GIT binary patch literal 2419 zcmaJD3rtg2^uD%0S1OdEuIf}oVn7K90{$I}5-|uULJ$=xPz469P|66pStlT<^%Fs1 zFhs%4=n$f-OnxDL2t#m+q7H(5qC|}%^D|^KbmzSLsvVl0=Dd5)|8wuTspPO35ehp7 z4FEfjY`n94h}{v}dH!^lyZ!S$<^({i6$LcX-B_dnv@Z4@#R8GRx32GUb(Mbiz43`_ z!i7brUB-MnY53&!ym!y7&Q2~X*E_sx>An-le{3c?CJQ$|-kIAsGBHTs=2#nEW*s|k zX7IxA_g2LJrE*EC12WD2R(sY-f_scfKwkLFb z?VX?9?o;eEk&wJusjv9UUk`ky5?MMGDr|%PtWzAXot>QJwdDX`Cb!qg z-b^32s5fzemiP9@zz3_RXHRoGwa3}nE+c19daI&lS^nut?SbZu#)7%JlE8?z%tIYd z=g%GT-Jz4M$QSmVt?mlgU-ag;>$VRnpAE!@J}mW{G-sJ-VUyz?Pq+D=3!W?854<|q zJl=3JUE1Khao!Hkmx`fvgTJpHsVNWWYQGtALggVa465^L)=Wu~ACeYd3xBkiA=>q= z5r=Iuax~pZ`jrl+tD9$Nx@|V+wfJ0&yLxP^-fj1lg5aUan(h-j^tuW4i}sbeZ#B%Q zpZT=j)~{+w$))hJ>KC(1;*~37Lgx^2zi{|MnC05boBZOV8yhLGhTSIq#aMz}X_XiGlM7pjD9FAQj4lZ54xOesO z+PD0q*B;pwBDsCP_dDI&WTo=gU)nOEHM;_y#MG9$J2~6^bF|gV+SO)hep9&5k;!&h zSG{zksLO1yHHDO)AD%u=Cwp}6MDOrP-Gu!`6K;3&(xU2Ye{i#^*|7V@&HXO?Lf^Ia z@gB1(FKm!Ai+fx5)y@4f=u%X_MZpd4P-{lQ&`ZgFCEQ*}#6&=Z@FQdqpnyXZd*KHl zmq%A_A(kv@MW_$7785JrA&q`|01;3>7ZCYkGM1RLniAL}@Q59JQHC)wH`GV;0Q>?F zXGtUwhR8rDh95gytPc=?mC=q5R1ZugA$-<1@)0l>5zmfDd3IyO5DDW`Pbg*H$EI~L zY+WLrfRUPs_`(*{9S>R4?9qaP0JwyNa1`}OYmjARr(-U%%M(eYvDwI!Fu;;9b|Df5 zYd7t=KhT&jt&Q%1Wcz?C`a_^Oj$W8CQj0cWBN&tec>_kD^if7{7894`bq)BG4Oz^nY`5xf#i9aAJI=;g&$pXci(+XaRHv#5#?Ahb>tM3_K@q6aW;052Ue2TEIv^ zZ$Jk?RH+^@7;wd0QdP+*iZqvWl~NG{B)QAUaF#e(Ok8vd8Q}P1KJe8k$r_MAme2?E zhqK1nM8|5BU!lhtoXo`EmIQeUu>jqxfF*!hz(~O5fDV8HzzELH8umt?Dy2F_u1q26 zoa|G%9Q`<6luDIA&VB4ptw_=+l+g;({0V>Cr0-~-3d~sd?_|#qq@sNvP!5Hs}n8<){ zjDRv8J8S;vkOTBfUkuchu>R2kqar^D(5Sx{_KZ;LEtQBwT+NB{G$~KWm~!HjEXP)4 z!|5nB2??8{RT?-UdOw|E#T4qH2N(-;K)@A|zP+Y-Gi(*cJWz+$NZ)i?w}8E8Xl1Gg zBv&7u7&?jc{Xjd+1>a(Pd-z;a@xN0f@JsPB0tnf@;Jgvi1bUq~KmZI0BcbcUeE$WX C$s$Mq literal 0 HcmV?d00001 diff --git a/fuzz/src/test/resources/io/github/dfa1/vortex/fuzz/VortexReaderFuzzTestInputs/scanMalformedFile/seed-vortex-runend b/fuzz/src/test/resources/io/github/dfa1/vortex/fuzz/VortexReaderFuzzTestInputs/scanMalformedFile/seed-vortex-runend new file mode 100644 index 0000000000000000000000000000000000000000..02668c5337356de796ba65b285aeca2ea39cbeb4 GIT binary patch literal 3800 zcmc)Ne`wTo9LMp`-EQ93>t?sLYBuZ-ckQlrbi13oyRn2~ zS(H@hpeUgQ3k*^)QpljhAf-kHEhs3+s30+ml8Ow9uIKCX`F`DZw-M1FyN^8Xd!OGw z-rw){X3iKhfC6e5!A?AeJs8IXUc!DHz!AKO<2ZpSOyd)r#8)_jA8-zTAQmyE747K2 zGF*X`=)xM@gbmn;J8>7bp$|n2;vww76L=EicoxrL5-;N*Uc*~>8}DNZpW-uogKzN@ ze#US36Mw;r8nYM|V=0#7N?easSc`Sogj=x%_n;U3$YBWE@h~32Q+OILU@!LJ6&%80 z9K|ubgAef$KF1gM4yW)7enlOo#h4`8a0xC&23O%~+C(u?H_=0oR6y2352|}uqh{Q`ty)aRe`TF9t=Y|=h-OSjG}D$eowlv|xV8~DATV+Z9xd&G} zR+_2j=bqWL16O0K>AjRMJuc&~+Rd5n^sm(I(%}Cur8`rRv`IxWJ*kK$4u^9yc^yme zt_i~T->ysS;kgl0HoO{18&rsc`$!>~L@?JV$EI}z;m}+Sc|9V52=2WPZ+|B8N5?<3awk`_)c3`yVdML5adibp4dz z%-kyU%uiz#w3Z+ad^MYR5h+e%{tIa@V+3*2IgOzY-BI*a1GUnF8ZpWlYp&3E_ZPEO zdoP@(`~)U3hCZyv3dC@;-}gek2jiAYm1?$BHCz3<`PMTsch39D<)Ya=cf3k|do5q; z&zl4Dj5m}WuH>2fJ-<%(u)mI>;lf~{S{PBaIW@QMaIKUt<%opGP81LAC$!eVY{edr zEuW%4)p5lm{2XdN_0zqjXF=B>um3NpZJbX%SELr#-(xA39A0G3Kr=s`97|mpZDmin5z9*+NIrZ8f)z}Tkh`N_z%dtDf0jT literal 0 HcmV?d00001 diff --git a/fuzz/src/test/resources/io/github/dfa1/vortex/fuzz/VortexReaderFuzzTestInputs/scanMalformedFile/seed-vortex-sparse b/fuzz/src/test/resources/io/github/dfa1/vortex/fuzz/VortexReaderFuzzTestInputs/scanMalformedFile/seed-vortex-sparse new file mode 100644 index 0000000000000000000000000000000000000000..dc64babf125f2e4648432bfeb69c47cc7cf36556 GIT binary patch literal 1913 zcma)7ziU%b6h7~V&GR*_&srpcL$DnLMI2m(4y98E2P?(JKGTQ~)1)RZp*VHu(80+` z!NEZ&IypIsLkB6Hii6L4fcjztn zJAM>o!G3TpOhwnOBII?~8a+#h!lDu`bq1zgr;c z$3SryEmdG_n3(X$moa5{jTw|O_FRGCv~dwX_Ny}GS0~CcZG0J{zN7ezIRSs1_`eOl ze}qpzqOwLJY0iP29h)|%P>XRxH2b)uf~{Q7UHO(6>9toX<0kh>3q_3Ofx2Cro0uybLGt5fE?Ba`ouci42W^r%FCvQV@EyY9n-lE zbd39SN{-hnP!!=zsHaFJ+K3hLJ*MVIyLFPKXbgoXfjU>}fkQhDF zDq`pkKsP}=r|UnqatQ+GG-s8ski*T5ecA-AfaXB6AjW_E+wYUJo?fe+*EeoFYc=bu z5ap}2@qJp`YF5kTm(@=U^qsujg#~!%BlGe7+L+qoZu1E^EyJ)U%d$%<+wW5AiUU*ze*yBOc>;cX-#yo!rBo zz9szfdxRBed;~mdxk{e9FJ-Af?+|G7&6`_fLc5rsnChFD{EQ$f^>;wd|2Fnsv^h7G z#IfzkTDmdiM_y5rHI3#%eBSyryN$+!TC0mG={>y;&JsSO2gG6xT;wGEz835D%qd(Q z^wBfwe}$eqG{{Vj9 P`Juq_@DFzH?()Jv%RANB literal 0 HcmV?d00001 diff --git a/fuzz/src/test/resources/io/github/dfa1/vortex/fuzz/VortexReaderFuzzTestInputs/scanMalformedFile/seed-vortex-zigzag b/fuzz/src/test/resources/io/github/dfa1/vortex/fuzz/VortexReaderFuzzTestInputs/scanMalformedFile/seed-vortex-zigzag new file mode 100644 index 0000000000000000000000000000000000000000..a062f51b947c1e418366b5d4b4648ac1e787f870 GIT binary patch literal 6168 zcmeH~!D}2<9LL{mx5>J0;x;z2g%&0dA%#RDw9rD84W!aS3MnL#ip5>iwOvehYqAL; zryMiU$w*3yj~-y!oZ$B(pQQ`-5+0=DpwV&HKLZ`@Y#Y z=UgC^h4Vs1ctxlQV?tfHAY2qCg@!;}7On`PFeOY2ZwfQQRbf_`6Xu1t1>(AJL%1p2 z65bJ-!h*0U+!pQ#cZDT^SQhRJE5fRI9@OONVLVI^YKrnOJ*dgk!+4k;)a2=5JZxW4leMqXJWLO2vi6nf zVLWVKQL`W{3h8~7=3)Danyh`5=3)MzChdI_?OK5YW z2!t#i=5z8Ug@%yF!+cJS`8|N1$AI z5ax69hwZC056)e95Jh=hS1vpV^LbPE!+0#KA09+e9_N({527fKP`U6R^7JqsHTzC-X^R|D?^*L!- z`zlmllE;jm;X$PRk;TLGpeF5)EFPS@@F0rvuQIyAB<-&tFD-UYY{uonRT`)b2$0fDFg9wFTdKiyu z%7X`CJ~w|XsBM@(s7d?7_Ek%Lr|ChB`80vxpl?xA|D36+Q;X!2WE7QYxm>$#=<5={pvTk{L z%qb_S2Q_JbG}V^WBmEtuChZTt=fnI#jrrXCQCHhAe^4_eoaMZt=9X}l^NJet`Dr}r zn8Sie6BYeTEbA|%IQS9koZwu^&BH3X^Ve6S%XsvmA(Ps1wJ{d;hfzP3&V4D+UenK< zktnks{UBxZFz(}B%n4A#`>7cbs^!XRLmkhKj*hUi<&sxZ7GrXU2+V~mIRrh`Qs^d1 z;h4|$JbiBPpIqMU=UI#1>oXapbo@@|-Vh6)MURiNT3`O4^|SFj8m;)+cJ2u;I6WGz zdeCUsgK)JTFtwx7Tu3nk0fFw2ry0C2gv+fuy^fgshNi@Bk#H# zKd(hHj281Ivf_8{5|f2sW!;kj@iw1h=rS||(e#Ir zz<+rS zbh{TPYxld4_uG%9L`6){13rnlZnn3h<%!yVQ@y%J(Ie?Y-bWpK3omfB=Uv^CJF36< zN^#EDSM)wwiIKqeQ%(O_j6{mt-Yi9)Tw=gD7@v3g+XbbC?D>(vW!PH$&pQdjuiC!cve@Y@RQbncQFToEAU6SXXfYP zk5HQjIi92cGIA0ZqdT!}@Cw{JI1T-c8 SPd;bFCGmqH- literal 0 HcmV?d00001 diff --git a/fuzz/src/test/resources/io/github/dfa1/vortex/fuzz/VortexReaderFuzzTestInputs/scanMalformedFile/seed-vortex-zstd b/fuzz/src/test/resources/io/github/dfa1/vortex/fuzz/VortexReaderFuzzTestInputs/scanMalformedFile/seed-vortex-zstd new file mode 100644 index 0000000000000000000000000000000000000000..6b9f196ffd75409845c1d4f0dc1a32802ceec1dc GIT binary patch literal 2384 zcma)7T}WG396#6GB-a@2Rf`q+!QGHnhfP!M>YQwoA>HVFko9A_wKF$gjWZ^hxn@on zNY{rw6dwxBlod9*4<80YECV}6X<-k}3Vjo83$tuEH!P%#;a+O?|J`$L5)x%Q;g@sH z|NH+x|8q~{_eY-kx^3rW*791dTWwq#WIg*wKbd;-o5vUYZN`(1`}LlW_I-XLed^G? zZ?FEbbocMUa7XH|mmYjL=Y4eNV(Z52bMIYwcD;P#*y8nud40FM`sn`2>C2g$sY6=k z`L$zpr+qVb9DAdQk1LNa`@~<^Bw2rLIr{PC)o3F9%-u!=)na=rvtfI6-OShRx63P+ z4zIK}zuMLH`pNXKW8p^I&;RUS|9vf+ec(BGYv|KZUpmxRws2{p>9@0k)zhx)i8p%A z`Q{Gj^^+^!)`}MGi!ZM@Z_GTg56x{}-t(jHvlV~8z%Lc=W-bneD>iPt6MpN@t!qbr z*mI$6b8%|^X?x)PBc2ZXy~x4xt;wUC(eqPry{<1tm^d7+ePx#_F_rP5FgKH#LO;g3 z2r|LWS?sEqr^Q?*W^0nnMHSmkGi231Fc<)hczP-(#DcTP)mkF!934f!dpg=|7WPuPUclf6|PIotCDs`l~CNm z#yLlOB#q!Pa0nO(6vNtE^G;`QZ?zH?5? zMf#{8Z}xniC|7B-=lN(#?uP_(gEytM&5#jXaz0#i7W}%qh_Fa#CLkZ{d}+z{QZlG# zCz7J!B_{T$*EqpP=p-}{pqt*xW-%j{RD9S7je5r7u~3k;fyhW~5Ejd*ZtzuLFC5hkv%l7f{uALV!X<)_&_!q<$b^@L?k-we zcH^<6p~npNw%~0SPpZ_XKOT>=2|=$Vdtp6kMDG2O32|a*dzuU%ULLW|sNPeM| zH>4-c6dAoR#W~^wLR$EY%H~-hpMn~gHAVe#x1IDa?z7XPSq}y7B6QJPxL=s9eTvo! zJp%>$cIb1zPSY=KB&(m?^p)VZ1YR-9%)8zVgJQrwe7eaSz6ALPLpJ*|LA*lwKH5vl zcxTN4SB*F$e|#hs3j@7Ok@GfW83)9D1Wm*@sj=z$})m3VAq< z)#~h)Dz!wa*reKGr|_s!kKA6Vww5YZs6uQMk1N!_PpME>Nfhxa>mG`Y~~9 zVx~%Mrtm3Kk3Cwd?j=!-Rc$d=yt}A=8Fo=!D^~DSAsrRFOVnG8OVmkXMGuuVR}pkU z?N?l^wi7G3Dr+r;N3r^$*V7j_LJ}0&LlxJDD9nb`tN8a-e3goSTgAVj;$Iyo3v!eN zS<8aD$b$4_L4Pi*3E|g-*wuu*sSYWt4#}=|P?$L=`1XqIzKZL13NyP56+cbIPgU_( zs`$%P{Kbuupxct5T1n6aNl=j_Xz$4K5VP`-rm~QtvXGT!A^ftg3bU>XzKJ5ci{iSm z!pwMxitnM~4^;78ReWa^zkilE=m&Ao_u?RlI7lE43T{@I=_>d-ifnDgbu9%S|84!o zHfCI%)d1NFk5w0mY3bZmwpJN2 zo|zAythDVmuCLV^v2Mu}eIHA!5Z>wsl}mT%Wf*(jk?oAuEuJzo!spWdu}|RJEj7DfT1tS0rNAJoU}C<1ii5@K zsLJ&PV^fTm>RNn=OkG>xo$}l~PbVhQcTItNN~O7BWMxN#w1R#qJI#|LQhzAuliw?4 ziTSf|-&F-YQsyq1Z9X&HU}b?x${6zxoa(U3Wd$8lY|LZAe3xpcv>55}UR&g)Y(MXQ z(0gx-yt=F7qyq0GX-VVWOnbkrJ(D6D9oJvAnY6_F&C8t?SFI<>H>(n7Hgs6C+ih!U z;)nXIvJGaF0=-ARyjHrw#J=~|Wr-~hMXT(sw#Im#Z7wYdG@R7edrh-$iQn?w9k)6p zu6|IpbhqBb@3y}7?$E4T9Qa*CVa(nwb_qFk!M&p$H(wd-f6Iw~yie^FcYkG8da$Li zLvCScjy(8up=nKKfB*eX1;?gUXFAwT%yA1o7A4BG^Ubkzv_3Yba;lC0BB$SvuJG~8 zDD=p=HDi2aReGW8_z)-2QF)n(ng0|guPLJ<_NLnP%2_m{KHPED6(fJ1Q_+!~C09E5 z_sQ~}aVM<9@YKJjgB`j5eb-RB9f zD)XzdHd(vd3cD$l8*e)Y4{#3_9#+oSROsezrz5yg6|mv9o%?v<`lvLe@g_63u2F9* z($^|8Zd2-d)&V zIk|F7?)SWgu$&9ZWw&*Mb=>uY`bxvxrcHY5CLEvpwyw)ipQq{NUfPfM{AfErE2X(ZE)bN0RAhbse52m zfQ9=Yr77=pfZAAI{B41U!g9z7X3uE7PvQN9MN4bEF#3f${zZ1b|GYh7@i*B#Zn(0BKG z{BeGLN3PMpu;$4wdz2CPK5Avxw)@8_)Xt`dFH&}c?Sj8n*GwakL+B%sDz? z9Lpt&Vpn5s-+yroI9fIw2mQ>#7%p##7B97$W8VBzcO~zz7nehWHmVD3gaI)`ji56a zldfntFb7Lwh4us+=uP^deZdi&h%?FqHyB6;qn#0c#SUBHxdL(RbwtVmDP26{su zVvqKN0pLPhQFj;w9>fbB3d7)EWHdS!e87+RqZ42fOd)~jbO?soBoqyU2#6v=bS}g| zEQv>@AcqAc5lx1Lu!t-{m%&O%C9BbN$bfYu6U~Ag$R$6bTcH4UkV14f?1la0AbJ>% z!cXJ`dJ2Alv*bK_0WLxrsX(it7A}!1=ry493qx6q5&qJ6;;oQN~Z12-5*2BV%Z1iZ-zbQFw%al{uL4*@Wd zOh%^yA7+r5=xhjuFcN`Afe_}Bd1xF+Kt>W!1th_D4v?dVR}1$)Ro^Z*=!Bjh;xGn|CeYbjf-7*1+(2)^ z9Z-?`XgxH+WAYUJ4St6|$V>DUyn*-RFZ2`q4O+UbbWlCehYqA8+8MflDd~oqgC$sz zo~RA;9q1EItIppFBy*pz(kl#0@3LZ46{fGItRib zk_gaf5J4=7MWT82*2{x0hXg+L*on#lf2ll}Mau_`ZMR0m=jCX3VMPKu|@lWBlIT&P#16ocQOd|0QSkxn~Xq5!e|&v zd{95|hY4g7Iu-aZgUm!{Lnwri2s8?WFqh0j<3J2jB1acMA|#WA=ptAOD@Y2O3acTV zWT5LH6SBxgG!M4GHd27@fI`?!_M-dYARHmb(IPkjr^qkpSvU_D$VIdqsz6EV&`WRy zu8|w)Ew}?Jav!aSMtDS?pwHkrydceJ3%r5%aPPZ6&c6F!{a~N*Fc`N6-whaXbn%mY zN5DSA)2&c5^@)D#Oy9SL6RD{-XXiIx#`pXC@4s>NYTjd;V>DYYTbHipU+a8$YsYcI zX9KqGh|g^P-kQ&()9|}5uE*mF&$>E)Vj)6;xk(Ffz$I@tW@unnwu%fPY8=W^Jx?DsE2XUxmR))-+6Of}z^ zu%+4YaRM7K1!K*g;(=z+*X%nUYO(dPb+--u@W8Osf_SN1km$T1TpA&YGtoBCw$y&# zskfF^KV3`w(ofq2?vtyp3+3wz&eT~AhGobAl+5B^i+RmWqKgY;{=3`UVLfgRh z&3so4d&|i&UuLh_HuH!2n*C<;uR70nmPMGFL$+WFu1*3#ANxOUs6#>vGAWa~PjIR+ekcFx;h zqqFflo6gQN+osm(opH^yUc(rr$HuX5z6>xHFE=)5v30bYN2hf;cD=E&Y~X6Kb+Y|q z>t|$gsSMFP7RAXmkJ&ZUI&{{Y@785oj{#ff6FTv2`~PKIbT!-3w(U`vx9yx-VGI7h z>z>WS&SxdG4Qw6luYLB~`Ld3$uZupIzwPzYc3rS_{I_eOb!kI9C%Y>caSSn^6R`bc z?*-X@v1RlAYJClL#A7b{_q0x61kE@e>@&bmkOgv>2JEw#5j?;_XP-Uy zz!G4euYJJ**!KnOy9f3i#t~rOwZwuiIDi2>!Sk_Wvj&51}53I#a55j4Hsez57BWE$V8-EyMn3?wJ1DJj`uAzt;@en z!z+G)NT9&DD_^g-^^SmPFTS3JOE4i4tK}GeYTb`^+anW)OJxE~>r2zvZJG8_CXgfu z;vxl{QC}}7O^6f3vEvj<)7Uxx&sQf#aTd{Zb`7*mkBrCu)A(-f#)nJsgmQLwYdsF9 zzkc5MIB~d0ie+xlw06flE+ICS^Wy7e=7-DX3D~w8(mlJ~w#*L~<1Yz~Z@2HUBAHxs zw1?BQcF)76ai)JgUMR!(I2zw>zl5ShLDU?XC|NK^vnVH%=4rPbf<&C4eP6FLN`y04 zLgOvI9IGfnq-cJ4?3{=sxnK^?q2|W&jOJ^1P0@U~GvQvyu4Q&_qVo_HE*Ho}^95~- zbJ*WuHcz|b5E&mQle3F*98GJt?#Q_bahd}g@%3_SyCUM_V>ydxTD#+?8J|Pr*}aXn zCp=axWv7XAjHb0aKA48jbu^w`zrt{tJT^Q|Aaj<+vg`NJf1M_bm*TEyfLAfQhqHZS zui-DtMu_C%@JO~3xD9CTPY5TyN9*5ScC6T7gnMJ_oy=JCw}QQvCIh>tvFjdJ7VeEk zSYBUqufqtPN<5jdwPvE`$1`jn8-D~p!dh?Icw~q*|NC-0jODePZu%e7H)DDhwvP>3 MoP5r7{J$sv2eLwEpa1{> literal 0 HcmV?d00001 From cd04d65e5ecbcb51c6b6cbb8abd0e955d9864ed5 Mon Sep 17 00:00:00 2001 From: Davide Angelocola Date: Sun, 18 Oct 2026 16:31:33 +0200 Subject: [PATCH 02/13] fix(reader): reject a corrupt array FlatBuffer as VortexException Found by the scanMalformedFile fuzz target: a segment's array FlatBuffer with a child-vector offset past its end escaped as a raw IndexOutOfBoundsException. The FlatBuffer runtime trusts every offset it follows (FbsTable), so each parse site must frame it, as PostscriptParser does for the footer blobs. SerializedArrayDecoder did not, nor did the two other unframed sites: the zone-stats read in ScanIterator (now degrades to no stats, as documented) and the HTTP reader's postscript parse. The catch alone cannot stop a vector length near Integer.MAX_VALUE sizing `new ArrayNode[n]` (an OutOfMemoryError), so a vector must fit in its FlatBuffer; an encoding index past the footer's spec table is rejected too. vortex-jni rejects the same file with its FlatBuffer verifier. Refs #493 Co-Authored-By: Claude Sonnet 5 Claude-Session: https://claude.ai/code/session_01WtuVGQ5h5Lk6Zn4JnkxF1K --- CHANGELOG.md | 1 + ...rash-array-flatbuffer-vector-out-of-bounds | Bin 0 -> 3800 bytes .../MalformedFileInteropIntegrationTest.java | 77 ++++++++++++++++++ ...ray-flatbuffer-vector-out-of-bounds.vortex | Bin 0 -> 3800 bytes .../dfa1/vortex/reader/ScanIterator.java | 17 ++-- .../vortex/reader/SerializedArrayDecoder.java | 62 ++++++++++---- .../dfa1/vortex/reader/VortexHttpReader.java | 44 ++++++---- .../ScanIteratorMalformedStatsTest.java | 55 +++++++++++++ .../SerializedArrayBoundsSecurityTest.java | 70 ++++++++++++++++ 9 files changed, 290 insertions(+), 36 deletions(-) create mode 100644 fuzz/src/test/resources/io/github/dfa1/vortex/fuzz/VortexReaderFuzzTestInputs/scanMalformedFile/crash-array-flatbuffer-vector-out-of-bounds create mode 100644 integration/src/test/java/io/github/dfa1/vortex/integration/MalformedFileInteropIntegrationTest.java create mode 100644 integration/src/test/resources/fixtures/malformed/array-flatbuffer-vector-out-of-bounds.vortex create mode 100644 reader/src/test/java/io/github/dfa1/vortex/reader/ScanIteratorMalformedStatsTest.java diff --git a/CHANGELOG.md b/CHANGELOG.md index d2c62968f..635591911 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -41,6 +41,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - **Breaking:** `ExtensionId` is a sealed `WellKnown`/`Custom` type like `EncodingId`, so third-party extension ids can be named, and `ExtensionId.parse` returns the id instead of an `Optional`. ([#530](https://github.com/dfa1/vortex-java/issues/530)) ### Fixed +- A file whose array FlatBuffer holds a corrupt offset fails the read with `VortexException` instead of `IndexOutOfBoundsException`. ([#493](https://github.com/dfa1/vortex-java/issues/493)) - SQL `WHERE` on an `I16`, `U8` or `I8` column no longer fails with a `ClassCastException`. ([#556](https://github.com/dfa1/vortex-java/pull/556)) - SQL `COUNT(DISTINCT col)` and aggregates with a `FILTER (WHERE ...)` clause return the right answer; the statistics push-down returned the row count and ignored the filter. ([#554](https://github.com/dfa1/vortex-java/pull/554)) - SQL `MIN`/`MAX` of a string column whose values fall outside Latin-1 answers instead of failing the query. ([#554](https://github.com/dfa1/vortex-java/pull/554)) diff --git a/fuzz/src/test/resources/io/github/dfa1/vortex/fuzz/VortexReaderFuzzTestInputs/scanMalformedFile/crash-array-flatbuffer-vector-out-of-bounds b/fuzz/src/test/resources/io/github/dfa1/vortex/fuzz/VortexReaderFuzzTestInputs/scanMalformedFile/crash-array-flatbuffer-vector-out-of-bounds new file mode 100644 index 0000000000000000000000000000000000000000..8ae602885cbf2ad6031fab1ad8dce812d53f1ec5 GIT binary patch literal 3800 zcmc)Ne`wTo9LMp`-EQ93>t^4rC0CRkB^5bJNU%U-feMC2RI_1!xV5|5(d}+-cVh{~ zvM8z0K~X{r78s;Zq@Y2GK}wAZT2N4sQ9)uBB^4PYUC-C&^ZmN-ZX=>Uc3*ki_xt(% z<9(m+&73i&9|cq~f*p7qyD^Rlyo`O=kHdHi$8a1|n8v3#fv<5IKjJL@L?UKPGg{Gx zWw-(>(1F#s8SAkDcj0bqMGuM?z{A*%C-D@<@f@DVBwoSmcmr?a9ejW(e1^~QExyCg z_yxb?FPwuHH)b&|#!_5~t8fEWVhz?}BW}ZH+>37XA%{V1!y|YUPvaT9h&|YgS8)Ib zaRf*4EsqLEBouc1Mx~o)t)6#CgVKxY#$Bj%vToc)rM)yCD0nagY+;TmU;tz~*~f3N2ClN~#Ler)BX`R5L< zR;+8LoS%C}(+XUPEvEHS9=cp2n6(=-(;nO@+o{g~-<4_iV;SSevR!^m14qNTiL|yQ zeAk5G`)~IpcJo}1z74N>(gGFY@Oz|?Od=d>lx@>I!fgH9}m+Q0C3t){!?Gg2O^#~Lfr-vh9+?po7P_CRB4jk$F8r{!|TX({~cho9ckEC6PUh6tp v>RolTw@P*38|I;RRKJ47b%t~gcP~hLv{g)1{|xmq?l+COcAL%jbZ__v10O5v literal 0 HcmV?d00001 diff --git a/integration/src/test/java/io/github/dfa1/vortex/integration/MalformedFileInteropIntegrationTest.java b/integration/src/test/java/io/github/dfa1/vortex/integration/MalformedFileInteropIntegrationTest.java new file mode 100644 index 000000000..4482ffa8f --- /dev/null +++ b/integration/src/test/java/io/github/dfa1/vortex/integration/MalformedFileInteropIntegrationTest.java @@ -0,0 +1,77 @@ +package io.github.dfa1.vortex.integration; + +import static org.assertj.core.api.Assertions.assertThatThrownBy; + +import dev.vortex.api.DataSource; +import dev.vortex.api.Partition; +import dev.vortex.api.Scan; +import dev.vortex.api.ScanOptions; +import dev.vortex.api.Session; +import dev.vortex.arrow.ArrowAllocation; +import dev.vortex.jni.NativeLoader; +import io.github.dfa1.vortex.core.error.VortexException; +import io.github.dfa1.vortex.reader.Chunk; +import io.github.dfa1.vortex.reader.ScanIterator; +import io.github.dfa1.vortex.reader.VortexReader; +import org.apache.arrow.vector.FieldVector; +import org.apache.arrow.vector.VectorSchemaRoot; +import org.apache.arrow.vector.ipc.ArrowReader; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.ValueSource; + +import java.net.URISyntaxException; +import java.nio.file.Path; +import java.util.Objects; + +/// Files the Jazzer `scanMalformedFile` target produced (#493) that once escaped vortex-java as a +/// raw JDK exception. Both readers must reject them: Rust is the oracle for what is malformed, and +/// vortex-java must throw only [VortexException] doing so (the reader's security contract). +class MalformedFileInteropIntegrationTest { + + static { + NativeLoader.loadJni(); + } + + @ParameterizedTest + @ValueSource(strings = { + // A child-node vector offset in a segment's array FlatBuffer points past its end: + // escaped as IndexOutOfBoundsException from the FlatBuffer runtime. + "array-flatbuffer-vector-out-of-bounds"}) + void bothReadersReject(String name) throws URISyntaxException { + // Given + Path file = Path.of(Objects.requireNonNull( + getClass().getResource("/fixtures/malformed/" + name + ".vortex")).toURI()); + + // When / Then + assertThatThrownBy(() -> scanJni(file)).as("vortex-jni").isInstanceOf(Exception.class); + assertThatThrownBy(() -> scanJava(file)).as("vortex-java").isInstanceOf(VortexException.class); + } + + private static void scanJava(Path file) throws Exception { + try (VortexReader reader = VortexReader.open(file); + ScanIterator scan = reader.scan(io.github.dfa1.vortex.reader.ScanOptions.all())) { + while (scan.hasNext()) { + try (Chunk _ = scan.next()) { + // decoding the chunk is enough: every file here fails before a value is read + } + } + } + } + + private static void scanJni(Path file) throws Exception { + Scan scan = DataSource.open(Session.create(), file.toUri().toString()).scan(ScanOptions.builder().build()); + while (scan.hasNext()) { + Partition partition = scan.next(); + try (ArrowReader reader = partition.scanArrow(ArrowAllocation.rootAllocator())) { + while (reader.loadNextBatch()) { + VectorSchemaRoot root = reader.getVectorSchemaRoot(); + for (FieldVector vector : root.getFieldVectors()) { + for (int i = 0; i < root.getRowCount(); i++) { + vector.getObject(i); + } + } + } + } + } + } +} diff --git a/integration/src/test/resources/fixtures/malformed/array-flatbuffer-vector-out-of-bounds.vortex b/integration/src/test/resources/fixtures/malformed/array-flatbuffer-vector-out-of-bounds.vortex new file mode 100644 index 0000000000000000000000000000000000000000..8ae602885cbf2ad6031fab1ad8dce812d53f1ec5 GIT binary patch literal 3800 zcmc)Ne`wTo9LMp`-EQ93>t^4rC0CRkB^5bJNU%U-feMC2RI_1!xV5|5(d}+-cVh{~ zvM8z0K~X{r78s;Zq@Y2GK}wAZT2N4sQ9)uBB^4PYUC-C&^ZmN-ZX=>Uc3*ki_xt(% z<9(m+&73i&9|cq~f*p7qyD^Rlyo`O=kHdHi$8a1|n8v3#fv<5IKjJL@L?UKPGg{Gx zWw-(>(1F#s8SAkDcj0bqMGuM?z{A*%C-D@<@f@DVBwoSmcmr?a9ejW(e1^~QExyCg z_yxb?FPwuHH)b&|#!_5~t8fEWVhz?}BW}ZH+>37XA%{V1!y|YUPvaT9h&|YgS8)Ib zaRf*4EsqLEBouc1Mx~o)t)6#CgVKxY#$Bj%vToc)rM)yCD0nagY+;TmU;tz~*~f3N2ClN~#Ler)BX`R5L< zR;+8LoS%C}(+XUPEvEHS9=cp2n6(=-(;nO@+o{g~-<4_iV;SSevR!^m14qNTiL|yQ zeAk5G`)~IpcJo}1z74N>(gGFY@Oz|?Od=d>lx@>I!fgH9}m+Q0C3t){!?Gg2O^#~Lfr-vh9+?po7P_CRB4jk$F8r{!|TX({~cho9ckEC6PUh6tp v>RolTw@P*38|I;RRKJ47b%t~gcP~hLv{g)1{|xmq?l+COcAL%jbZ__v10O5v literal 0 HcmV?d00001 diff --git a/reader/src/main/java/io/github/dfa1/vortex/reader/ScanIterator.java b/reader/src/main/java/io/github/dfa1/vortex/reader/ScanIterator.java index b21b09584..6bbc9ffc4 100644 --- a/reader/src/main/java/io/github/dfa1/vortex/reader/ScanIterator.java +++ b/reader/src/main/java/io/github/dfa1/vortex/reader/ScanIterator.java @@ -1192,8 +1192,7 @@ private ArrayStats readFlatStats(Layout flat) { SegmentSpec spec = file.footer().segmentSpecs().get(segIdx); long segLen = spec.length(); // Stats are an optional zone-map pruning optimization: a malformed stats segment - // degrades to "no stats" (empty) and never aborts the scan. This mirrors - // VortexReader.readFlatStats — both stats readers swallow bounds errors here. + // degrades to "no stats" (empty) and never aborts the scan. // The trailing 4-byte fbLen lives in the segment's last bytes; reading the whole // segment as a ByteBuffer would fail for segments larger than 2 GB (ByteBuffer cap). if (segLen < 4) { @@ -1205,13 +1204,17 @@ private ArrayStats readFlatStats(Layout flat) { return ArrayStats.empty(); } long fbStart = segLen - 4L - fbLen; - var fbArray = io.github.dfa1.vortex.core.fbs.FbsArray.getRootAsFbsArray(IoBounds.slice(seg, fbStart, fbLen)); - - io.github.dfa1.vortex.core.fbs.FbsArrayNode root = fbArray.root(); - if (root == null) { + try { + var fbArray = io.github.dfa1.vortex.core.fbs.FbsArray.getRootAsFbsArray(IoBounds.slice(seg, fbStart, fbLen)); + io.github.dfa1.vortex.core.fbs.FbsArrayNode root = fbArray.root(); + if (root == null) { + return ArrayStats.empty(); + } + return ArrayStats.fromFbs(root.stats()); + } catch (RuntimeException _) { + // The FlatBuffer runtime trusts the offsets it follows: a crafted one fails inside it. return ArrayStats.empty(); } - return ArrayStats.fromFbs(root.stats()); } // ── Layout decode context ───────────────────────────────────────────────── diff --git a/reader/src/main/java/io/github/dfa1/vortex/reader/SerializedArrayDecoder.java b/reader/src/main/java/io/github/dfa1/vortex/reader/SerializedArrayDecoder.java index aebb46436..1fcad3f78 100644 --- a/reader/src/main/java/io/github/dfa1/vortex/reader/SerializedArrayDecoder.java +++ b/reader/src/main/java/io/github/dfa1/vortex/reader/SerializedArrayDecoder.java @@ -37,6 +37,10 @@ final class SerializedArrayDecoder { /// contract. 64 is well past any real encoding's nesting. static final int MAX_ARRAY_TREE_DEPTH = 64; + /// Wire size of the `Buffer` struct in `array.fbs` (u16 padding, u8 alignment, u8 compression, + /// u32 length). + private static final int BUFFER_STRUCT_BYTES = 8; + private final ReadRegistry registry; /// Creates a decoder backed by the given registry. @@ -63,19 +67,30 @@ public Array decode(MemorySegment seg, List encodingSpecs, int fbLen = seg.get(LE_INT, segLen - 4L); long fbStart = segLen - 4L - fbLen; IoBounds.checkRange(fbStart, fbLen, segLen); - var fbArray = io.github.dfa1.vortex.core.fbs.FbsArray.getRootAsFbsArray(seg.asSlice(fbStart, fbLen)); - - int numBuffers = IoBounds.checkCount(fbArray.buffersLength()); - MemorySegment[] bufs = new MemorySegment[numBuffers]; - long dataOffset = 0; - for (int i = 0; i < numBuffers; i++) { - FbsBuffer bufDesc = fbArray.buffers(i); - dataOffset += bufDesc.padding(); - bufs[i] = IoBounds.slice(seg, dataOffset, bufDesc.length()); - dataOffset += bufDesc.length(); + + MemorySegment[] bufs; + ArrayNode rootNode; + // The FlatBuffer runtime trusts the offsets it follows (see FbsTable): a crafted vtable or + // vector offset fails inside it as a raw IndexOutOfBoundsException, so the whole parse is + // framed here, as PostscriptParser frames the footer blobs. + try { + var fbArray = io.github.dfa1.vortex.core.fbs.FbsArray.getRootAsFbsArray(seg.asSlice(fbStart, fbLen)); + int numBuffers = vectorCount(fbArray.buffersLength(), BUFFER_STRUCT_BYTES, fbLen); + bufs = new MemorySegment[numBuffers]; + long dataOffset = 0; + for (int i = 0; i < numBuffers; i++) { + FbsBuffer bufDesc = fbArray.buffers(i); + dataOffset += bufDesc.padding(); + bufs[i] = IoBounds.slice(seg, dataOffset, bufDesc.length()); + dataOffset += bufDesc.length(); + } + rootNode = convertArrayNode(fbArray.root(), encodingSpecs, fbLen, 0); + } catch (VortexException e) { + throw e; + } catch (RuntimeException e) { + throw new VortexException("malformed array flatbuffer", e); } - ArrayNode rootNode = convertArrayNode(fbArray.root(), encodingSpecs, 0); var ctx = new DecodeContext(rootNode, dtype, rowCount, bufs, registry, arena); return registry.decode(ctx); } @@ -83,21 +98,27 @@ public Array decode(MemorySegment seg, List encodingSpecs, private static ArrayNode convertArrayNode( io.github.dfa1.vortex.core.fbs.FbsArrayNode fbs, List encodingSpecs, + int fbLen, int depth ) { if (depth > MAX_ARRAY_TREE_DEPTH) { throw new VortexException( "array tree depth exceeds limit (" + MAX_ARRAY_TREE_DEPTH + ")"); } + int encoding = fbs.encoding(); + if (encoding >= encodingSpecs.size()) { + throw new VortexException("array encoding index " + encoding + " out of range [0, " + + encodingSpecs.size() + ")"); + } // Already parsed (and blank-guarded) at the footer boundary. - EncodingId encodingId = encodingSpecs.get(fbs.encoding()); + EncodingId encodingId = encodingSpecs.get(encoding); - ArrayNode[] children = new ArrayNode[fbs.childrenLength()]; + ArrayNode[] children = new ArrayNode[vectorCount(fbs.childrenLength(), Integer.BYTES, fbLen)]; for (int i = 0; i < children.length; i++) { - children[i] = convertArrayNode(fbs.children(i), encodingSpecs, depth + 1); + children[i] = convertArrayNode(fbs.children(i), encodingSpecs, fbLen, depth + 1); } - int[] bufferIndices = new int[fbs.buffersLength()]; + int[] bufferIndices = new int[vectorCount(fbs.buffersLength(), Short.BYTES, fbLen)]; for (int i = 0; i < bufferIndices.length; i++) { bufferIndices[i] = fbs.buffers(i); } @@ -105,4 +126,15 @@ private static ArrayNode convertArrayNode( MemorySegment meta = fbs.metadataAsSegment(); return new ArrayNode(encodingId, meta, children, bufferIndices); } + + /// Checks a FlatBuffer vector's claimed length before it sizes a Java array: the elements must + /// fit in the FlatBuffer, else a crafted length near [Integer#MAX_VALUE] is an + /// [OutOfMemoryError] (an `Error`, past the [VortexException] framing) before any element is read. + private static int vectorCount(int count, int elementBytes, int fbLen) { + if (count < 0 || (long) count * elementBytes > fbLen) { + throw new VortexException("array flatbuffer vector of " + count + " elements exceeds its " + + fbLen + "-byte flatbuffer"); + } + return count; + } } diff --git a/reader/src/main/java/io/github/dfa1/vortex/reader/VortexHttpReader.java b/reader/src/main/java/io/github/dfa1/vortex/reader/VortexHttpReader.java index 865de7f7a..3b396d1f5 100644 --- a/reader/src/main/java/io/github/dfa1/vortex/reader/VortexHttpReader.java +++ b/reader/src/main/java/io/github/dfa1/vortex/reader/VortexHttpReader.java @@ -124,23 +124,39 @@ public static VortexHttpReader open(URI uri, ReadRegistry registry, LayoutRegist MemorySegment postscriptSeg = IoBounds.slice(tailSeg, psOffInTail, trailer.postscriptLen()); - var ps = FbsPostscript.getRootAsFbsPostscript(postscriptSeg); - - var footerSpec = ps.footer(); - if (footerSpec == null) { - throw new VortexException("postscript missing footer segment"); + record Blob(long offset, long length) { } - var layoutSpec = ps.layout(); - if (layoutSpec == null) { - throw new VortexException("postscript missing layout segment"); + Blob footerBlob; + Blob layoutBlob; + Blob dtypeBlob; + // The FlatBuffer runtime trusts the offsets it follows (see FbsTable): a crafted one fails + // inside it as a raw IndexOutOfBoundsException, framed here as PostscriptParser.parse + // frames the local-file path. + try { + var ps = FbsPostscript.getRootAsFbsPostscript(postscriptSeg); + var footerSpec = ps.footer(); + if (footerSpec == null) { + throw new VortexException("postscript missing footer segment"); + } + var layoutSpec = ps.layout(); + if (layoutSpec == null) { + throw new VortexException("postscript missing layout segment"); + } + var dtypeSpec = ps.dtype(); + footerBlob = new Blob(footerSpec.offset(), footerSpec.length()); + layoutBlob = new Blob(layoutSpec.offset(), layoutSpec.length()); + dtypeBlob = dtypeSpec != null ? new Blob(dtypeSpec.offset(), dtypeSpec.length()) : new Blob(0, 0); + } catch (VortexException e) { + throw e; + } catch (RuntimeException e) { + throw new VortexException("malformed postscript", e); } - var dtypeSpec = ps.dtype(); - MemorySegment footerBuf = fetchBlob(footerSpec.offset(), footerSpec.length(), tailStart, tail, uri, client); - MemorySegment layoutBuf = fetchBlob(layoutSpec.offset(), layoutSpec.length(), tailStart, tail, uri, client); - MemorySegment dtypeBuf = (dtypeSpec != null && dtypeSpec.length() > 0) - ? fetchBlob(dtypeSpec.offset(), dtypeSpec.length(), tailStart, tail, uri, client) - : null; + MemorySegment footerBuf = fetchBlob(footerBlob.offset(), footerBlob.length(), tailStart, tail, uri, client); + MemorySegment layoutBuf = fetchBlob(layoutBlob.offset(), layoutBlob.length(), tailStart, tail, uri, client); + MemorySegment dtypeBuf = dtypeBlob.length() > 0 + ? fetchBlob(dtypeBlob.offset(), dtypeBlob.length(), tailStart, tail, uri, client) + : null; var parsed = PostscriptParser.parseBlobs(footerBuf, layoutBuf, dtypeBuf); // Reject footer segmentSpecs that fall outside the file before any rawSegment() builds a diff --git a/reader/src/test/java/io/github/dfa1/vortex/reader/ScanIteratorMalformedStatsTest.java b/reader/src/test/java/io/github/dfa1/vortex/reader/ScanIteratorMalformedStatsTest.java new file mode 100644 index 000000000..c65baee4e --- /dev/null +++ b/reader/src/test/java/io/github/dfa1/vortex/reader/ScanIteratorMalformedStatsTest.java @@ -0,0 +1,55 @@ +package io.github.dfa1.vortex.reader; + +import io.github.dfa1.vortex.core.io.VortexFormat; +import io.github.dfa1.vortex.core.model.ColumnName; +import io.github.dfa1.vortex.core.model.DType; +import io.github.dfa1.vortex.core.model.LayoutId; +import io.github.dfa1.vortex.reader.layout.Layout; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.mockito.Mock; +import org.mockito.junit.jupiter.MockitoExtension; + +import java.lang.foreign.MemorySegment; +import java.util.List; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.BDDMockito.given; + +/// A malformed stats segment found by the `scanMalformedFile` fuzz target (#493): it escaped the +/// scan as a raw JDK exception instead of degrading to "no stats". +@ExtendWith(MockitoExtension.class) +class ScanIteratorMalformedStatsTest { + + private static final ColumnName COLUMN = ColumnName.of("v"); + private static final DType.Struct SCHEMA = new DType.Struct(List.of(COLUMN), List.of(DType.I64), false); + + @Mock + private VortexHandle file; + + @Test + void statsSegmentWithCorruptFlatBuffer_degradesToEmptyStats() { + // Given — a flat whose segment frames a 4-byte FlatBuffer (the trailing length is sane) + // whose root offset points far outside it; FlatBuffer navigation used to escape as a raw + // IndexOutOfBoundsException although malformed stats are documented to read as "no stats" + MemorySegment seg = MemorySegment.ofArray(new byte[8]); + seg.set(VortexFormat.LE_INT, 0, 1_000_000); + seg.set(VortexFormat.LE_INT, 4, 4); + Layout flat = new Layout(LayoutId.FLAT, 5, null, List.of(), List.of(0)); + given(file.dtype()).willReturn(SCHEMA); + given(file.layout()).willReturn(new Layout(LayoutId.STRUCT, 5, null, List.of(flat), List.of())); + given(file.footer()).willReturn(new Footer(List.of(), List.of(), + List.of(new SegmentSpec(0, 8, (byte) 0, CompressionScheme.NONE)), List.of())); + given(file.rawSegment(any())).willReturn(seg); + + // When + List result; + try (ScanIterator sut = new ScanIterator(file, ScanOptions.all())) { + result = sut.chunkStats(COLUMN); + } + + // Then + assertThat(result).containsExactly(ArrayStats.empty()); + } +} diff --git a/reader/src/test/java/io/github/dfa1/vortex/reader/SerializedArrayBoundsSecurityTest.java b/reader/src/test/java/io/github/dfa1/vortex/reader/SerializedArrayBoundsSecurityTest.java index fdecfce85..7d43ce55b 100644 --- a/reader/src/test/java/io/github/dfa1/vortex/reader/SerializedArrayBoundsSecurityTest.java +++ b/reader/src/test/java/io/github/dfa1/vortex/reader/SerializedArrayBoundsSecurityTest.java @@ -85,6 +85,76 @@ void bufferDescriptorLengthPastSegment_throwsVortexException() { } } + @Test + void rootOffsetPastFlatBuffer_throwsVortexException() { + try (Arena arena = Arena.ofConfined()) { + // Given a valid FlatBuffer whose root uoffset is rewritten to land 2 bytes before its + // end: the runtime's vtable read there runs off the slice (the first fuzzer finding) + byte[] fb = arrayFlatBufferWithOneBuffer(0L); + MemorySegment seg = segmentOf(arena, fb); + seg.set(VortexFormat.LE_INT, 0, fb.length - 2); + + // When / Then + assertThatThrownBy(() -> sut.decode(seg, FLAT_SPEC, DTYPE, 1, arena)) + .isInstanceOf(VortexException.class) + .hasMessageContaining("malformed array flatbuffer"); + } + } + + @Test + void encodingIndexPastSpecTable_throwsVortexException() { + try (Arena arena = Arena.ofConfined()) { + // Given a root node naming encoding #5 while the footer declares a single encoding + FbsBuilder b = new FbsBuilder(); + int children = FbsArrayNode.createChildrenVector(b, new int[0]); + int buffers = FbsArrayNode.createBuffersVector(b, new int[0]); + int root = FbsArrayNode.createFbsArrayNode(b, 5, 0, children, buffers, 0); + MemorySegment seg = segmentOf(arena, finish(b, root)); + + // When / Then + assertThatThrownBy(() -> sut.decode(seg, FLAT_SPEC, DTYPE, 1, arena)) + .isInstanceOf(VortexException.class) + .hasMessageContaining("encoding index 5 out of range"); + } + } + + @Test + void childrenVectorClaimingMoreThanTheFlatBuffer_throwsVortexException() { + try (Arena arena = Arena.ofConfined()) { + // Given a children vector whose length prefix claims 100 000 entries it does not hold. + // At Integer.MAX_VALUE the `new ArrayNode[n]` it sizes would be an OutOfMemoryError, + // which no catch of RuntimeException stops; the count is checked before allocating. + FbsBuilder b = new FbsBuilder(); + FbsArrayNode.startChildrenVector(b, 100_000); + int children = b.endVector(); + int buffers = FbsArrayNode.createBuffersVector(b, new int[0]); + int root = FbsArrayNode.createFbsArrayNode(b, 0, 0, children, buffers, 0); + MemorySegment seg = segmentOf(arena, finish(b, root)); + + // When / Then + assertThatThrownBy(() -> sut.decode(seg, FLAT_SPEC, DTYPE, 1, arena)) + .isInstanceOf(VortexException.class) + .hasMessageContaining("vector of 100000 elements exceeds"); + } + } + + /// Lays out `fb` as a flat segment: the FlatBuffer followed by its u32 LE length. + private static MemorySegment segmentOf(Arena arena, byte[] fb) { + MemorySegment seg = arena.allocate(fb.length + 4L); + MemorySegment.copy(MemorySegment.ofArray(fb), 0, seg, 0, fb.length); + seg.set(VortexFormat.LE_INT, fb.length, fb.length); + return seg; + } + + /// Wraps `root` in an `FbsArray` with no buffer descriptors. + private static byte[] finish(FbsBuilder b, int root) { + FbsArray.startBuffersVector(b, 0); + int buffers = b.endVector(); + int array = FbsArray.createFbsArray(b, root, buffers); + FbsArray.finishFbsArrayBuffer(b, array); + return b.sizedByteArray(); + } + /// Builds a minimal valid `FbsArray` FlatBuffer with one buffer descriptor of the given length. private static byte[] arrayFlatBufferWithOneBuffer(long bufferLength) { FbsBuilder b = new FbsBuilder(); From 9d449fbab0cd742211015195f26899cadd3852a8 Mon Sep 17 00:00:00 2001 From: Davide Angelocola Date: Sun, 18 Oct 2026 16:31:45 +0200 Subject: [PATCH 03/13] fix(reader): check the dtype before the bitpacked, delta and RLE decoders cast it Found by the scanMalformedFile fuzz target: a Struct column whose segment claims fastlanes.bitpacked escaped as a raw ClassCastException, because the decoder cast its dtype to Primitive unchecked. Delta did the same, and RLE's empty-chunk shortcut cast before the check the rest of its decode has. They now reject a non-primitive dtype the way the FOR, ALP and Pco decoders already do. RLE also covers bools, so a zero-row bool chunk hit that cast in a valid file; it now reads as an empty bool array. vortex-jni rejects the same file ("Expected 1 buffer, got 0"). Refs #493 Co-Authored-By: Claude Sonnet 5 Claude-Session: https://claude.ai/code/session_01WtuVGQ5h5Lk6Zn4JnkxF1K --- CHANGELOG.md | 1 + .../crash-bitpacked-struct-dtype | Bin 0 -> 2964 bytes .../MalformedFileInteropIntegrationTest.java | 4 ++- .../malformed/bitpacked-struct-dtype.vortex | Bin 0 -> 2964 bytes .../decode/BitpackedEncodingDecoder.java | 5 +++- .../reader/decode/DeltaEncodingDecoder.java | 5 +++- .../reader/decode/RleEncodingDecoder.java | 10 +++++-- .../decode/BitpackedEncodingDecoderTest.java | 17 +++++++++++ .../decode/DeltaEncodingDecoderTest.java | 13 ++++++++ .../reader/decode/RleEncodingDecoderTest.java | 28 ++++++++++++++++++ 10 files changed, 78 insertions(+), 5 deletions(-) create mode 100644 fuzz/src/test/resources/io/github/dfa1/vortex/fuzz/VortexReaderFuzzTestInputs/scanMalformedFile/crash-bitpacked-struct-dtype create mode 100644 integration/src/test/resources/fixtures/malformed/bitpacked-struct-dtype.vortex diff --git a/CHANGELOG.md b/CHANGELOG.md index 635591911..339069247 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -42,6 +42,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Fixed - A file whose array FlatBuffer holds a corrupt offset fails the read with `VortexException` instead of `IndexOutOfBoundsException`. ([#493](https://github.com/dfa1/vortex-java/issues/493)) +- A column whose dtype does not fit its bit-packed, delta or RLE encoding fails with `VortexException` instead of `ClassCastException`, and an empty RLE boolean chunk reads. ([#493](https://github.com/dfa1/vortex-java/issues/493)) - SQL `WHERE` on an `I16`, `U8` or `I8` column no longer fails with a `ClassCastException`. ([#556](https://github.com/dfa1/vortex-java/pull/556)) - SQL `COUNT(DISTINCT col)` and aggregates with a `FILTER (WHERE ...)` clause return the right answer; the statistics push-down returned the row count and ignored the filter. ([#554](https://github.com/dfa1/vortex-java/pull/554)) - SQL `MIN`/`MAX` of a string column whose values fall outside Latin-1 answers instead of failing the query. ([#554](https://github.com/dfa1/vortex-java/pull/554)) diff --git a/fuzz/src/test/resources/io/github/dfa1/vortex/fuzz/VortexReaderFuzzTestInputs/scanMalformedFile/crash-bitpacked-struct-dtype b/fuzz/src/test/resources/io/github/dfa1/vortex/fuzz/VortexReaderFuzzTestInputs/scanMalformedFile/crash-bitpacked-struct-dtype new file mode 100644 index 0000000000000000000000000000000000000000..96f508e8a6068f5e7ffca73b7ddc97bed432a800 GIT binary patch literal 2964 zcmeHJJ!}+56n<-t^Ku-HQ7#rBi<}~aTv&(-QKYaCid;m9B0^Lgd$)72$$IbD-8-DC zQn-jxMp0@Mxrm~a2&JHiA`3;4<(h&DA%qYmCFJ}@=KSrmwOwf)T{pe$ndHhTZr7jf-@e{cSRDfBdlHfn~GqA6|9*;#%$i@MY6}+DZ7J*~< zP1-@JB1#ArkUnzps3#BC4r1^dCcO(HtH2|?li!y*MpgdA_%cA=CI+9Tg<@jVrC}lm z5;UOVoNS<{8#y-tu9Z6WjIpC*9Fpr8Uj;q~DF0o{B?vIzS-hVD{FTw)H8(eV=G*nO zsX8OCpYjDs?E{7H`-$gchI8OhMegc|(~vnWtjaq2J`e*YKg+O~g-r;nLsFB&@M}8$ zSdz%UTGLy2!F^AA20P}gKjxW`**YHndHVtH)HClTz#KC^xl;bK@tfiJ4>Jz#d$PM2 zZ+q`s7>8IrPnJDLGmOb>hmhKw2jt+}(6*v~h70$51y}~Sw~SAlC{0`0AWbnu_RM=X zN(+Ut)0g>xuW|&SZPZRxtD8nQ@XmSc(B&MGIej(^--_Nla%N86d9L2?!T+)c*oNIY z$UAHr&r?O8C+;ctY95zo3;nmir+}$M3%F;l(UL8c1w1(y{aoYyZUUF|c@Bb87<2#4 z`NaUu*|4X-=@;M^f{VG?v=c;S3@{It@FbTBcgpQtAfV>}ch~Ri6Q4RZ_DwvTt?m(N z@g4Esx{iI$?<7|>@a0#jjv8g|A@4^6DFju0k50ZCkD1ZKz_6E>YW)nZRw?+n$u{QR z1o#b#fmL7$r~@xRdK>r^*Z^YSJg^9GVGj}DK5z&42DlDf1eSmb@C0UCz$UN(#J~#B z1S-J2X%w!ZJ37$~qr9DD!$jSNtcGt6=k|P!9-zAX6FF`lHqHJXybfo~$KhGE@-BJ*(%w zIen*RzNcJ+w_m!HKAn!W78PG9#+v!ttlQ(5CPk^Ua?jRZPv(qOCs~WqR{QHxwUFzwfbK7A$M3R@ zZS6#!S{={6(?z8!lQresrG`7n+zeq* zDlH>O_SV=B_t>}@=KSrmwOwf)T{pe$ndHhTZr7jf-@e{cSRDfBdlHfn~GqA6|9*;#%$i@MY6}+DZ7J*~< zP1-@JB1#ArkUnzps3#BC4r1^dCcO(HtH2|?li!y*MpgdA_%cA=CI+9Tg<@jVrC}lm z5;UOVoNS<{8#y-tu9Z6WjIpC*9Fpr8Uj;q~DF0o{B?vIzS-hVD{FTw)H8(eV=G*nO zsX8OCpYjDs?E{7H`-$gchI8OhMegc|(~vnWtjaq2J`e*YKg+O~g-r;nLsFB&@M}8$ zSdz%UTGLy2!F^AA20P}gKjxW`**YHndHVtH)HClTz#KC^xl;bK@tfiJ4>Jz#d$PM2 zZ+q`s7>8IrPnJDLGmOb>hmhKw2jt+}(6*v~h70$51y}~Sw~SAlC{0`0AWbnu_RM=X zN(+Ut)0g>xuW|&SZPZRxtD8nQ@XmSc(B&MGIej(^--_Nla%N86d9L2?!T+)c*oNIY z$UAHr&r?O8C+;ctY95zo3;nmir+}$M3%F;l(UL8c1w1(y{aoYyZUUF|c@Bb87<2#4 z`NaUu*|4X-=@;M^f{VG?v=c;S3@{It@FbTBcgpQtAfV>}ch~Ri6Q4RZ_DwvTt?m(N z@g4Esx{iI$?<7|>@a0#jjv8g|A@4^6DFju0k50ZCkD1ZKz_6E>YW)nZRw?+n$u{QR z1o#b#fmL7$r~@xRdK>r^*Z^YSJg^9GVGj}DK5z&42DlDf1eSmb@C0UCz$UN(#J~#B z1S-J2X%w!ZJ37$~qr9DD!$jSNtcGt6=k|P!9-zAX6FF`lHqHJXybfo~$KhGE@-BJ*(%w zIen*RzNcJ+w_m!HKAn!W78PG9#+v!ttlQ(5CPk^Ua?jRZPv(qOCs~WqR{QHxwUFzwfbK7A$M3R@ zZS6#!S{={6(?z8!lQresrG`7n+zeq* zDlH>O_SV=B_t> new LazyConstantLongArray(dt, 0L, 0L); case I32, U32 -> new LazyConstantIntArray(dt, 0L, 0); case I16, U16 -> new LazyConstantShortArray(dt, 0L, (short) 0); diff --git a/reader/src/test/java/io/github/dfa1/vortex/reader/decode/BitpackedEncodingDecoderTest.java b/reader/src/test/java/io/github/dfa1/vortex/reader/decode/BitpackedEncodingDecoderTest.java index 0fbecb5a2..c4df5ef1c 100644 --- a/reader/src/test/java/io/github/dfa1/vortex/reader/decode/BitpackedEncodingDecoderTest.java +++ b/reader/src/test/java/io/github/dfa1/vortex/reader/decode/BitpackedEncodingDecoderTest.java @@ -16,6 +16,7 @@ import java.lang.foreign.Arena; import java.lang.foreign.MemorySegment; +import java.util.List; import static org.assertj.core.api.Assertions.assertThat; import static org.assertj.core.api.Assertions.assertThatThrownBy; @@ -259,6 +260,22 @@ void packedBufferTooSmallForRowCount_throws() { .hasMessageContaining("too small for 1024 rows at bit width 8"); } + @Test + void nonPrimitiveDtype_throws() { + // Given a struct column whose segment claims bit-packing (a fuzzer finding: the + // decoder cast the dtype blindly and escaped as ClassCastException) + ArrayNode node = new ArrayNode(BITPACKED, MemorySegment.ofArray(new ProtoBitPackedMetadata(8, 0, null).encode()), + new ArrayNode[0], new int[]{0}); + DType struct = new DType.Struct(List.of(), List.of(), false); + DecodeContext ctx = new DecodeContext(node, struct, 8, new MemorySegment[]{packedBytes(64)}, REGISTRY, + Arena.ofAuto()); + + // When / Then + assertThatThrownBy(() -> SUT.decode(ctx)) + .isInstanceOf(VortexException.class) + .hasMessageContaining("expected primitive dtype"); + } + private DecodeContext ctxWithMeta(ProtoBitPackedMetadata meta, MemorySegment packed, long rowCount) { ArrayNode node = new ArrayNode(BITPACKED, MemorySegment.ofArray(meta.encode()), new ArrayNode[0], new int[]{0}); diff --git a/reader/src/test/java/io/github/dfa1/vortex/reader/decode/DeltaEncodingDecoderTest.java b/reader/src/test/java/io/github/dfa1/vortex/reader/decode/DeltaEncodingDecoderTest.java index 25379a2e2..e56499ca5 100644 --- a/reader/src/test/java/io/github/dfa1/vortex/reader/decode/DeltaEncodingDecoderTest.java +++ b/reader/src/test/java/io/github/dfa1/vortex/reader/decode/DeltaEncodingDecoderTest.java @@ -304,6 +304,19 @@ void decode_absurdDeltasLenButSmallWindow_decodesWithoutAllocatingForIt() { assertThat(result.getLong(3)).isZero(); } + @Test + void decode_nonPrimitiveDtype_throws() { + // Given a Utf8 column whose segment claims delta encoding: a crafted file, which escaped + // as a raw ClassCastException before the dtype was checked + ArrayNode node = new ArrayNode(EncodingId.FASTLANES_DELTA, null, new ArrayNode[0], new int[0]); + DecodeContext ctx = new DecodeContext(node, DType.UTF8, 0, new MemorySegment[0], REGISTRY, Arena.ofAuto()); + + // When / Then + assertThatThrownBy(() -> SUT.decode(ctx)) + .isInstanceOf(VortexException.class) + .hasMessageContaining("expected primitive dtype"); + } + @Test void decode_constantBases_nonZeroOffsetAndBase() { // Given a constant base of 5 broadcast across all 16 lanes with zero deltas: diff --git a/reader/src/test/java/io/github/dfa1/vortex/reader/decode/RleEncodingDecoderTest.java b/reader/src/test/java/io/github/dfa1/vortex/reader/decode/RleEncodingDecoderTest.java index b8853cb06..7cb5c584a 100644 --- a/reader/src/test/java/io/github/dfa1/vortex/reader/decode/RleEncodingDecoderTest.java +++ b/reader/src/test/java/io/github/dfa1/vortex/reader/decode/RleEncodingDecoderTest.java @@ -341,6 +341,34 @@ void unsupportedIndicesPtype_isRejected() { } } + @Nested + class EmptyChunk { + + @Test + void boolDtype_decodesEmpty() { + // Given a zero-row bool chunk: RLE covers bools too, but the empty-array shortcut + // cast every dtype to Primitive and threw ClassCastException + MemorySegment empty = MemorySegment.ofArray(new byte[0]); + + // When + Array result = decodeRle(DType.BOOL, 0, 0, empty, empty, empty, 0); + + // Then + assertThat(result.length()).isZero(); + } + + @Test + void nonPrimitiveDtype_isRejected() { + // Given a zero-row chunk of a Utf8 column whose segment claims RLE (a crafted file) + MemorySegment empty = MemorySegment.ofArray(new byte[0]); + + // When / Then + assertThatThrownBy(() -> decodeRle(DType.UTF8, 0, 0, empty, empty, empty, 0)) + .isInstanceOf(VortexException.class) + .hasMessageContaining("expected Primitive dtype"); + } + } + // ── decode harness ───────────────────────────────────────────────────────── /// Builds an RLE node with a `u8` index table: child[0] = values, child[1] = indices, From af9a9f12477c2f988bd99d210d707e816679eaae Mon Sep 17 00:00:00 2001 From: Davide Angelocola Date: Sun, 18 Oct 2026 16:31:56 +0200 Subject: [PATCH 04/13] fix(reader): validate struct dtypes and struct layouts at open, as Rust does Found by the scanMalformedFile fuzz target: a struct dtype without a names vector opened as a 0-field struct, then the scan indexed its field list by layout child and threw a raw ArrayIndexOutOfBoundsException. Rust refuses the file at open ("failed to parse struct names from flatbuffer"). Both checks now run at open, where Rust runs them: - StructFields::from_fb requires the names and dtypes vectors; only an empty one is a 0-field struct. The generated readers could not tell absent from empty, so fbs-gen emits hasX() for every vector field. - Layout::::validate_children requires one child per field plus a leading validity child when the struct is nullable; PostscriptParser checks it for the root and directly nested struct layouts. Refs #493 Co-Authored-By: Claude Sonnet 5 Claude-Session: https://claude.ai/code/session_01WtuVGQ5h5Lk6Zn4JnkxF1K --- CHANGELOG.md | 1 + .../github/dfa1/vortex/core/fbs/FbsArray.java | 6 + .../dfa1/vortex/core/fbs/FbsArrayNode.java | 18 +++ .../dfa1/vortex/core/fbs/FbsArrayStats.java | 18 +++ .../dfa1/vortex/core/fbs/FbsExtension.java | 6 + .../vortex/core/fbs/FbsFileStatistics.java | 6 + .../dfa1/vortex/core/fbs/FbsFooter.java | 30 +++++ .../dfa1/vortex/core/fbs/FbsLayout.java | 18 +++ .../dfa1/vortex/core/fbs/FbsStruct.java | 12 ++ .../github/dfa1/vortex/core/fbs/FbsUnion.java | 18 +++ .../io/github/dfa1/vortex/fbsgen/CodeGen.java | 8 +- .../crash-struct-dtype-missing-names | Bin 0 -> 3800 bytes .../dfa1/vortex/inspect/InspectorTree.java | 20 +++- .../MalformedFileInteropIntegrationTest.java | 5 +- .../struct-dtype-missing-names.vortex | Bin 0 -> 3800 bytes .../dfa1/vortex/reader/PostscriptParser.java | 28 +++++ .../dfa1/vortex/reader/ScanIterator.java | 17 ++- .../PostscriptParserDTypeGuardsTest.java | 113 ++++++++++++++++++ .../reader/ScanIteratorNullableRootTest.java | 57 +++++++++ 19 files changed, 371 insertions(+), 10 deletions(-) create mode 100644 fuzz/src/test/resources/io/github/dfa1/vortex/fuzz/VortexReaderFuzzTestInputs/scanMalformedFile/crash-struct-dtype-missing-names create mode 100644 integration/src/test/resources/fixtures/malformed/struct-dtype-missing-names.vortex create mode 100644 reader/src/test/java/io/github/dfa1/vortex/reader/ScanIteratorNullableRootTest.java diff --git a/CHANGELOG.md b/CHANGELOG.md index 339069247..d79790d36 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -43,6 +43,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Fixed - A file whose array FlatBuffer holds a corrupt offset fails the read with `VortexException` instead of `IndexOutOfBoundsException`. ([#493](https://github.com/dfa1/vortex-java/issues/493)) - A column whose dtype does not fit its bit-packed, delta or RLE encoding fails with `VortexException` instead of `ClassCastException`, and an empty RLE boolean chunk reads. ([#493](https://github.com/dfa1/vortex-java/issues/493)) +- A struct dtype missing its field names or types, or a struct layout whose children do not match its fields, is rejected when the file opens, as Rust does, instead of failing the scan with `ArrayIndexOutOfBoundsException`. ([#493](https://github.com/dfa1/vortex-java/issues/493)) - SQL `WHERE` on an `I16`, `U8` or `I8` column no longer fails with a `ClassCastException`. ([#556](https://github.com/dfa1/vortex-java/pull/556)) - SQL `COUNT(DISTINCT col)` and aggregates with a `FILTER (WHERE ...)` clause return the right answer; the statistics push-down returned the row count and ignored the filter. ([#554](https://github.com/dfa1/vortex-java/pull/554)) - SQL `MIN`/`MAX` of a string column whose values fall outside Latin-1 answers instead of failing the query. ([#554](https://github.com/dfa1/vortex-java/pull/554)) diff --git a/core/src/main/java/io/github/dfa1/vortex/core/fbs/FbsArray.java b/core/src/main/java/io/github/dfa1/vortex/core/fbs/FbsArray.java index 22957a215..5ce550a54 100644 --- a/core/src/main/java/io/github/dfa1/vortex/core/fbs/FbsArray.java +++ b/core/src/main/java/io/github/dfa1/vortex/core/fbs/FbsArray.java @@ -37,6 +37,12 @@ public int buffersLength() { return o != 0 ? vectorLength(o) : 0; } + /// Tells an absent `buffers` vector from an empty one, which [#buffersLength()] cannot. + /// @return whether the `buffers` vector is present + public boolean hasBuffers() { + return fieldOffset(6) != 0; + } + /// @param j element index /// @return the j-th `buffers` element public FbsBuffer buffers(int j) { diff --git a/core/src/main/java/io/github/dfa1/vortex/core/fbs/FbsArrayNode.java b/core/src/main/java/io/github/dfa1/vortex/core/fbs/FbsArrayNode.java index 8e2ffeba7..87a12e74c 100644 --- a/core/src/main/java/io/github/dfa1/vortex/core/fbs/FbsArrayNode.java +++ b/core/src/main/java/io/github/dfa1/vortex/core/fbs/FbsArrayNode.java @@ -37,6 +37,12 @@ public int metadataLength() { return o != 0 ? vectorLength(o) : 0; } + /// Tells an absent `metadata` vector from an empty one, which [#metadataLength()] cannot. + /// @return whether the `metadata` vector is present + public boolean hasMetadata() { + return fieldOffset(6) != 0; + } + /// @param j element index /// @return the j-th `metadata` element public int metadata(int j) { @@ -56,6 +62,12 @@ public int childrenLength() { return o != 0 ? vectorLength(o) : 0; } + /// Tells an absent `children` vector from an empty one, which [#childrenLength()] cannot. + /// @return whether the `children` vector is present + public boolean hasChildren() { + return fieldOffset(8) != 0; + } + /// @param j element index /// @return the j-th `children` element public FbsArrayNode children(int j) { @@ -69,6 +81,12 @@ public int buffersLength() { return o != 0 ? vectorLength(o) : 0; } + /// Tells an absent `buffers` vector from an empty one, which [#buffersLength()] cannot. + /// @return whether the `buffers` vector is present + public boolean hasBuffers() { + return fieldOffset(10) != 0; + } + /// @param j element index /// @return the j-th `buffers` element public int buffers(int j) { diff --git a/core/src/main/java/io/github/dfa1/vortex/core/fbs/FbsArrayStats.java b/core/src/main/java/io/github/dfa1/vortex/core/fbs/FbsArrayStats.java index eb8da4948..969ff2588 100644 --- a/core/src/main/java/io/github/dfa1/vortex/core/fbs/FbsArrayStats.java +++ b/core/src/main/java/io/github/dfa1/vortex/core/fbs/FbsArrayStats.java @@ -31,6 +31,12 @@ public int minLength() { return o != 0 ? vectorLength(o) : 0; } + /// Tells an absent `min` vector from an empty one, which [#minLength()] cannot. + /// @return whether the `min` vector is present + public boolean hasMin() { + return fieldOffset(4) != 0; + } + /// @param j element index /// @return the j-th `min` element public int min(int j) { @@ -56,6 +62,12 @@ public int maxLength() { return o != 0 ? vectorLength(o) : 0; } + /// Tells an absent `max` vector from an empty one, which [#maxLength()] cannot. + /// @return whether the `max` vector is present + public boolean hasMax() { + return fieldOffset(8) != 0; + } + /// @param j element index /// @return the j-th `max` element public int max(int j) { @@ -81,6 +93,12 @@ public int sumLength() { return o != 0 ? vectorLength(o) : 0; } + /// Tells an absent `sum` vector from an empty one, which [#sumLength()] cannot. + /// @return whether the `sum` vector is present + public boolean hasSum() { + return fieldOffset(12) != 0; + } + /// @param j element index /// @return the j-th `sum` element public int sum(int j) { diff --git a/core/src/main/java/io/github/dfa1/vortex/core/fbs/FbsExtension.java b/core/src/main/java/io/github/dfa1/vortex/core/fbs/FbsExtension.java index 7c84a107e..ce42f2cb4 100644 --- a/core/src/main/java/io/github/dfa1/vortex/core/fbs/FbsExtension.java +++ b/core/src/main/java/io/github/dfa1/vortex/core/fbs/FbsExtension.java @@ -43,6 +43,12 @@ public int metadataLength() { return o != 0 ? vectorLength(o) : 0; } + /// Tells an absent `metadata` vector from an empty one, which [#metadataLength()] cannot. + /// @return whether the `metadata` vector is present + public boolean hasMetadata() { + return fieldOffset(8) != 0; + } + /// @param j element index /// @return the j-th `metadata` element public int metadata(int j) { diff --git a/core/src/main/java/io/github/dfa1/vortex/core/fbs/FbsFileStatistics.java b/core/src/main/java/io/github/dfa1/vortex/core/fbs/FbsFileStatistics.java index 206602aa6..2a1b8fb2d 100644 --- a/core/src/main/java/io/github/dfa1/vortex/core/fbs/FbsFileStatistics.java +++ b/core/src/main/java/io/github/dfa1/vortex/core/fbs/FbsFileStatistics.java @@ -31,6 +31,12 @@ public int fieldStatsLength() { return o != 0 ? vectorLength(o) : 0; } + /// Tells an absent `fieldStats` vector from an empty one, which [#fieldStatsLength()] cannot. + /// @return whether the `fieldStats` vector is present + public boolean hasFieldStats() { + return fieldOffset(4) != 0; + } + /// @param j element index /// @return the j-th `fieldStats` element public FbsArrayStats fieldStats(int j) { diff --git a/core/src/main/java/io/github/dfa1/vortex/core/fbs/FbsFooter.java b/core/src/main/java/io/github/dfa1/vortex/core/fbs/FbsFooter.java index f0d6fc371..909209359 100644 --- a/core/src/main/java/io/github/dfa1/vortex/core/fbs/FbsFooter.java +++ b/core/src/main/java/io/github/dfa1/vortex/core/fbs/FbsFooter.java @@ -31,6 +31,12 @@ public int arraySpecsLength() { return o != 0 ? vectorLength(o) : 0; } + /// Tells an absent `arraySpecs` vector from an empty one, which [#arraySpecsLength()] cannot. + /// @return whether the `arraySpecs` vector is present + public boolean hasArraySpecs() { + return fieldOffset(4) != 0; + } + /// @param j element index /// @return the j-th `arraySpecs` element public FbsArraySpec arraySpecs(int j) { @@ -44,6 +50,12 @@ public int layoutSpecsLength() { return o != 0 ? vectorLength(o) : 0; } + /// Tells an absent `layoutSpecs` vector from an empty one, which [#layoutSpecsLength()] cannot. + /// @return whether the `layoutSpecs` vector is present + public boolean hasLayoutSpecs() { + return fieldOffset(6) != 0; + } + /// @param j element index /// @return the j-th `layoutSpecs` element public FbsLayoutSpec layoutSpecs(int j) { @@ -57,6 +69,12 @@ public int segmentSpecsLength() { return o != 0 ? vectorLength(o) : 0; } + /// Tells an absent `segmentSpecs` vector from an empty one, which [#segmentSpecsLength()] cannot. + /// @return whether the `segmentSpecs` vector is present + public boolean hasSegmentSpecs() { + return fieldOffset(8) != 0; + } + /// @param j element index /// @return the j-th `segmentSpecs` element public FbsSegmentSpec segmentSpecs(int j) { @@ -70,6 +88,12 @@ public int compressionSpecsLength() { return o != 0 ? vectorLength(o) : 0; } + /// Tells an absent `compressionSpecs` vector from an empty one, which [#compressionSpecsLength()] cannot. + /// @return whether the `compressionSpecs` vector is present + public boolean hasCompressionSpecs() { + return fieldOffset(10) != 0; + } + /// @param j element index /// @return the j-th `compressionSpecs` element public FbsCompressionSpec compressionSpecs(int j) { @@ -83,6 +107,12 @@ public int encryptionSpecsLength() { return o != 0 ? vectorLength(o) : 0; } + /// Tells an absent `encryptionSpecs` vector from an empty one, which [#encryptionSpecsLength()] cannot. + /// @return whether the `encryptionSpecs` vector is present + public boolean hasEncryptionSpecs() { + return fieldOffset(12) != 0; + } + /// @param j element index /// @return the j-th `encryptionSpecs` element public FbsEncryptionSpec encryptionSpecs(int j) { diff --git a/core/src/main/java/io/github/dfa1/vortex/core/fbs/FbsLayout.java b/core/src/main/java/io/github/dfa1/vortex/core/fbs/FbsLayout.java index 94a7d86ef..a10dd0276 100644 --- a/core/src/main/java/io/github/dfa1/vortex/core/fbs/FbsLayout.java +++ b/core/src/main/java/io/github/dfa1/vortex/core/fbs/FbsLayout.java @@ -43,6 +43,12 @@ public int metadataLength() { return o != 0 ? vectorLength(o) : 0; } + /// Tells an absent `metadata` vector from an empty one, which [#metadataLength()] cannot. + /// @return whether the `metadata` vector is present + public boolean hasMetadata() { + return fieldOffset(8) != 0; + } + /// @param j element index /// @return the j-th `metadata` element public int metadata(int j) { @@ -62,6 +68,12 @@ public int childrenLength() { return o != 0 ? vectorLength(o) : 0; } + /// Tells an absent `children` vector from an empty one, which [#childrenLength()] cannot. + /// @return whether the `children` vector is present + public boolean hasChildren() { + return fieldOffset(10) != 0; + } + /// @param j element index /// @return the j-th `children` element public FbsLayout children(int j) { @@ -75,6 +87,12 @@ public int segmentsLength() { return o != 0 ? vectorLength(o) : 0; } + /// Tells an absent `segments` vector from an empty one, which [#segmentsLength()] cannot. + /// @return whether the `segments` vector is present + public boolean hasSegments() { + return fieldOffset(12) != 0; + } + /// @param j element index /// @return the j-th `segments` element public long segments(int j) { diff --git a/core/src/main/java/io/github/dfa1/vortex/core/fbs/FbsStruct.java b/core/src/main/java/io/github/dfa1/vortex/core/fbs/FbsStruct.java index 3bdb5c4d3..79df72760 100644 --- a/core/src/main/java/io/github/dfa1/vortex/core/fbs/FbsStruct.java +++ b/core/src/main/java/io/github/dfa1/vortex/core/fbs/FbsStruct.java @@ -31,6 +31,12 @@ public int namesLength() { return o != 0 ? vectorLength(o) : 0; } + /// Tells an absent `names` vector from an empty one, which [#namesLength()] cannot. + /// @return whether the `names` vector is present + public boolean hasNames() { + return fieldOffset(4) != 0; + } + /// @param j element index /// @return the j-th `names` string public String names(int j) { @@ -44,6 +50,12 @@ public int dtypesLength() { return o != 0 ? vectorLength(o) : 0; } + /// Tells an absent `dtypes` vector from an empty one, which [#dtypesLength()] cannot. + /// @return whether the `dtypes` vector is present + public boolean hasDtypes() { + return fieldOffset(6) != 0; + } + /// @param j element index /// @return the j-th `dtypes` element public FbsDType dtypes(int j) { diff --git a/core/src/main/java/io/github/dfa1/vortex/core/fbs/FbsUnion.java b/core/src/main/java/io/github/dfa1/vortex/core/fbs/FbsUnion.java index 03222243e..7d48ad077 100644 --- a/core/src/main/java/io/github/dfa1/vortex/core/fbs/FbsUnion.java +++ b/core/src/main/java/io/github/dfa1/vortex/core/fbs/FbsUnion.java @@ -31,6 +31,12 @@ public int namesLength() { return o != 0 ? vectorLength(o) : 0; } + /// Tells an absent `names` vector from an empty one, which [#namesLength()] cannot. + /// @return whether the `names` vector is present + public boolean hasNames() { + return fieldOffset(4) != 0; + } + /// @param j element index /// @return the j-th `names` string public String names(int j) { @@ -44,6 +50,12 @@ public int dtypesLength() { return o != 0 ? vectorLength(o) : 0; } + /// Tells an absent `dtypes` vector from an empty one, which [#dtypesLength()] cannot. + /// @return whether the `dtypes` vector is present + public boolean hasDtypes() { + return fieldOffset(6) != 0; + } + /// @param j element index /// @return the j-th `dtypes` element public FbsDType dtypes(int j) { @@ -57,6 +69,12 @@ public int typeIdsLength() { return o != 0 ? vectorLength(o) : 0; } + /// Tells an absent `typeIds` vector from an empty one, which [#typeIdsLength()] cannot. + /// @return whether the `typeIds` vector is present + public boolean hasTypeIds() { + return fieldOffset(8) != 0; + } + /// @param j element index /// @return the j-th `typeIds` element public int typeIds(int j) { diff --git a/fbs-gen/src/main/java/io/github/dfa1/vortex/fbsgen/CodeGen.java b/fbs-gen/src/main/java/io/github/dfa1/vortex/fbsgen/CodeGen.java index bd3e37d30..8e2e53f26 100644 --- a/fbs-gen/src/main/java/io/github/dfa1/vortex/fbsgen/CodeGen.java +++ b/fbs-gen/src/main/java/io/github/dfa1/vortex/fbsgen/CodeGen.java @@ -222,7 +222,13 @@ private void emitVectorField(StringBuilder sb, String accessor, Ast.FieldType el int o = fieldOffset(%2$d); return o != 0 ? vectorLength(o) : 0; } - """.formatted(accessor, vt)); + + /// Tells an absent `%1$s` vector from an empty one, which [#%1$sLength()] cannot. + /// @return whether the `%1$s` vector is present + public boolean has%3$s() { + return fieldOffset(%2$d) != 0; + } + """.formatted(accessor, vt, capitalize(accessor))); if (element instanceof Ast.Scalar scalar) { emitScalarVectorElements(sb, accessor, scalar, vt); diff --git a/fuzz/src/test/resources/io/github/dfa1/vortex/fuzz/VortexReaderFuzzTestInputs/scanMalformedFile/crash-struct-dtype-missing-names b/fuzz/src/test/resources/io/github/dfa1/vortex/fuzz/VortexReaderFuzzTestInputs/scanMalformedFile/crash-struct-dtype-missing-names new file mode 100644 index 0000000000000000000000000000000000000000..6d66e8f175ace725fa7722e1c2d6bd86d1e54e03 GIT binary patch literal 3800 zcmc)Ne`wTY0LSs?er)@C-R!oOTv0Yk`sFAgA%lzsDi{`F&4&HquI*|^x4XIBjU^Pz zqNG9xMF}lfV30zgLIx!UDK#o+ML|JE1&LXdRAi8Jy+64C)R=ud58+K5#W74^5})8WzQ!s1h%@*T@rW^XXhJiV z;xa5p8&=^ati?KP#9i2eF67aNhp-J#;7N?)Sv-evyo}fII^M$Dcpnq^44>m$e21U$ z3x3C6IE#QeF2aRag3ECQuEz@8jMZ3=+pr1upcCEbK|i+QVLXDT@HAe)ZtTG;IDmsV zj3amlAL1i?fiLkrPT*Jkh6+r~m?Ro;5iUk6uEbTi0V~mtTX8$?z-E}(z2*U)y(poK zN3k7`V<&duMU3H9?8O^6gm-ZiA7c_<;W$p>2b{tmID=T6eMbsSxD-oqEtca(til?s z#Ri0dt(X=TDz4?O4)Yyn8K;H+KYq4SsjzR;dYyZGk#A?N?bX$cW1-^pHRG7CctcIU z3l$G)#<5WG@O=u0GhgvU*r9(eR6JFE5hfW-V3HixfPefrK?ltpwx}7kw^huv`co5L zUguH9n8hY#pQ@cC#zhUalk}aa{0^wSXY92=TYOhJrKy<8$Vw<(2O|*=I?`nJ+d6gU z%Ifk;QmWqd!gqzZeY!Eb{u9ww(;RJWOqv#3R&`w2h`GeJ5zcXX- z&-JaeB>&vO)r=LU^7*-^H_gE1*lcPq;Y){$_))t)(=Gl<-VQZBw~%g0Mbah}Y3)cw z)N%E8ZXm8@3GbS4`~KT=v0XgZV#;o>TG9ly#o>FTEg47H*D%YbafI8UzO>|uciJUw zu-`RfEqA^qP2~>%OB-V~<<94KXnuiI8EWP6C3gFa73)#vr(x=ZE*pWZl?K3Ezklf&PMK2Jy4*Xiyp7amlL zQPx;}Rke42K2x&q!b#GPVH_jq!Wt|?9Jlzo=kq;mw^$e~WeO#;+2_qRo}rn2-d8N< z&CZ$a4Q98NvxV-g**8mj{h5KmEPcP{^Yjk;ar6)5`f{b*kh0B*nRN%sg>0dRod}qT z_Cxcj8f#x>&@PWnpP)Y34cd?3(`)#2LH((o-YuO4J;S*EzoawUz^Bd?A!h3ntGB62 zYKVmIQ_`eKcO~?xw@B^R{_U$YS2ktacV*+Y?44ERSMxm*jI=kW5(&R1eVO6rXb`DR z$rWtM1&JnKrclo3w{#cF?4Wy(+jweDM_2>(r8z3-Defv(PIz@j>@31@tBjkY`&n^o z5x4h_YE_pbYUiPAT_;PetBlrGu{M0eICMw#YO@HZiFfe~d_178_EhChQ!ed((->=~ K*>rd3x_ colNames = (dtype instanceof DType.Struct s) - ? s.fieldNames().stream().map(ColumnName::value).toList() : List.of(); + List colNames = rootChildLabels(dtype); Node root = shallowNode(layout, Optional.empty()); if (layout.isStruct()) { List named = new ArrayList<>(root.children().size()); @@ -183,8 +182,7 @@ public static InspectorTree build(VortexHandle handle, Progress progress) { int total = countPeekableSegments(layout, footer); int[] counter = {0}; - List colNames = (dtype instanceof DType.Struct s) - ? s.fieldNames().stream().map(ColumnName::value).toList() : List.of(); + List colNames = rootChildLabels(dtype); Set overallUsed = new LinkedHashSet<>(); Map segmentEncodings = new LinkedHashMap<>(); Node root = buildNode(layout, Optional.empty(), handle, footer.arraySpecs(), @@ -384,6 +382,20 @@ public interface Progress { void update(int current, int total); } + /// Labels for the root struct layout's children: the column names, after a `` child + /// when the struct is nullable (its layout leads with one, as Rust's `validate_children` requires). + private static List rootChildLabels(DType dtype) { + if (!(dtype instanceof DType.Struct s)) { + return List.of(); + } + List labels = new ArrayList<>(); + if (s.nullable()) { + labels.add(""); + } + s.fieldNames().forEach(name -> labels.add(name.value())); + return labels; + } + private static Peek peekFlatRoot(MemorySegment seg, List arraySpecs) { int segLen = (int) seg.byteSize(); int fbLen = seg.get(LE_INT, segLen - 4L); diff --git a/integration/src/test/java/io/github/dfa1/vortex/integration/MalformedFileInteropIntegrationTest.java b/integration/src/test/java/io/github/dfa1/vortex/integration/MalformedFileInteropIntegrationTest.java index 27e26eac7..9d187899a 100644 --- a/integration/src/test/java/io/github/dfa1/vortex/integration/MalformedFileInteropIntegrationTest.java +++ b/integration/src/test/java/io/github/dfa1/vortex/integration/MalformedFileInteropIntegrationTest.java @@ -38,7 +38,10 @@ class MalformedFileInteropIntegrationTest { // escaped as IndexOutOfBoundsException from the FlatBuffer runtime. "array-flatbuffer-vector-out-of-bounds", // A Struct column whose segment is fastlanes.bitpacked: escaped as ClassCastException. - "bitpacked-struct-dtype"}) + "bitpacked-struct-dtype", + // A struct dtype with no names vector, which Rust refuses at open: vortex-java read it + // as a 0-field struct and failed the scan with ArrayIndexOutOfBoundsException. + "struct-dtype-missing-names"}) void bothReadersReject(String name) throws URISyntaxException { // Given Path file = Path.of(Objects.requireNonNull( diff --git a/integration/src/test/resources/fixtures/malformed/struct-dtype-missing-names.vortex b/integration/src/test/resources/fixtures/malformed/struct-dtype-missing-names.vortex new file mode 100644 index 0000000000000000000000000000000000000000..6d66e8f175ace725fa7722e1c2d6bd86d1e54e03 GIT binary patch literal 3800 zcmc)Ne`wTY0LSs?er)@C-R!oOTv0Yk`sFAgA%lzsDi{`F&4&HquI*|^x4XIBjU^Pz zqNG9xMF}lfV30zgLIx!UDK#o+ML|JE1&LXdRAi8Jy+64C)R=ud58+K5#W74^5})8WzQ!s1h%@*T@rW^XXhJiV z;xa5p8&=^ati?KP#9i2eF67aNhp-J#;7N?)Sv-evyo}fII^M$Dcpnq^44>m$e21U$ z3x3C6IE#QeF2aRag3ECQuEz@8jMZ3=+pr1upcCEbK|i+QVLXDT@HAe)ZtTG;IDmsV zj3amlAL1i?fiLkrPT*Jkh6+r~m?Ro;5iUk6uEbTi0V~mtTX8$?z-E}(z2*U)y(poK zN3k7`V<&duMU3H9?8O^6gm-ZiA7c_<;W$p>2b{tmID=T6eMbsSxD-oqEtca(til?s z#Ri0dt(X=TDz4?O4)Yyn8K;H+KYq4SsjzR;dYyZGk#A?N?bX$cW1-^pHRG7CctcIU z3l$G)#<5WG@O=u0GhgvU*r9(eR6JFE5hfW-V3HixfPefrK?ltpwx}7kw^huv`co5L zUguH9n8hY#pQ@cC#zhUalk}aa{0^wSXY92=TYOhJrKy<8$Vw<(2O|*=I?`nJ+d6gU z%Ifk;QmWqd!gqzZeY!Eb{u9ww(;RJWOqv#3R&`w2h`GeJ5zcXX- z&-JaeB>&vO)r=LU^7*-^H_gE1*lcPq;Y){$_))t)(=Gl<-VQZBw~%g0Mbah}Y3)cw z)N%E8ZXm8@3GbS4`~KT=v0XgZV#;o>TG9ly#o>FTEg47H*D%YbafI8UzO>|uciJUw zu-`RfEqA^qP2~>%OB-V~<<94KXnuiI8EWP6C3gFa73)#vr(x=ZE*pWZl?K3Ezklf&PMK2Jy4*Xiyp7amlL zQPx;}Rke42K2x&q!b#GPVH_jq!Wt|?9Jlzo=kq;mw^$e~WeO#;+2_qRo}rn2-d8N< z&CZ$a4Q98NvxV-g**8mj{h5KmEPcP{^Yjk;ar6)5`f{b*kh0B*nRN%sg>0dRod}qT z_Cxcj8f#x>&@PWnpP)Y34cd?3(`)#2LH((o-YuO4J;S*EzoawUz^Bd?A!h3ntGB62 zYKVmIQ_`eKcO~?xw@B^R{_U$YS2ktacV*+Y?44ERSMxm*jI=kW5(&R1eVO6rXb`DR z$rWtM1&JnKrclo3w{#cF?4Wy(+jweDM_2>(r8z3-Defv(PIz@j>@31@tBjkY`&n^o z5x4h_YE_pbYUiPAT_;PetBlrGu{M0eICMw#YO@HZiFfe~d_178_EhChQ!ed((->=~ K*>rd3x_ 0) { dtype = convertDType(io.github.dfa1.vortex.core.fbs.FbsDType.getRootAsFbsDType(dtypeBuf), 0); + checkStructLayouts(layout, dtype); } return new ParsedFile(footer, dtype, layout); @@ -136,6 +137,26 @@ static ParsedFile parseBlobs(MemorySegment footerBuf, MemorySegment layoutBuf, M } } + /// Rejects at open a struct layout whose children do not match its struct dtype, as Rust does + /// while deserializing the layout tree (`Layout::::validate_children`): one child per + /// field, plus a leading validity child when the struct is nullable. Left to the scan, a + /// mismatch indexed the field list by layout child and escaped as a raw JDK exception. + /// Recurses only through directly nested struct layouts, whose dtype is the field's own. + private static void checkStructLayouts(Layout layout, DType dtype) { + if (!layout.isStruct() || !(dtype instanceof DType.Struct struct)) { + return; + } + int validity = struct.nullable() ? 1 : 0; + int expected = struct.fieldTypes().size() + validity; + if (layout.children().size() != expected) { + throw new VortexException("vortex.struct layout has " + layout.children().size() + + " children but its struct dtype needs " + expected); + } + for (int i = 0; i < struct.fieldTypes().size(); i++) { + checkStructLayouts(layout.children().get(validity + i), struct.fieldTypes().get(i)); + } + } + private static MemorySegment slice(MemorySegment seg, long offset, long length) { return IoBounds.slice(seg, offset, length); } @@ -284,6 +305,13 @@ private static DType convertDType(io.github.dfa1.vortex.core.fbs.FbsDType fbs, i case FbsType.FbsBinary -> new DType.Binary(fbs.type(new FbsBinary()).nullable()); case FbsType.FbsStruct -> { var s = fbs.type(new FbsStruct()); + // Rust (StructFields::from_fb) requires both vectors; only an empty one is a 0-field struct. + if (!s.hasNames()) { + throw new VortexException("failed to parse struct names from flatbuffer"); + } + if (!s.hasDtypes()) { + throw new VortexException("failed to parse struct dtypes from flatbuffer"); + } if (s.namesLength() != s.dtypesLength()) { throw new VortexException("struct names/dtypes length mismatch: " + s.namesLength() + " names, " + s.dtypesLength() + " dtypes"); diff --git a/reader/src/main/java/io/github/dfa1/vortex/reader/ScanIterator.java b/reader/src/main/java/io/github/dfa1/vortex/reader/ScanIterator.java index 6bbc9ffc4..20398c42c 100644 --- a/reader/src/main/java/io/github/dfa1/vortex/reader/ScanIterator.java +++ b/reader/src/main/java/io/github/dfa1/vortex/reader/ScanIterator.java @@ -582,10 +582,18 @@ private Layout findZonedLayout(Layout root, ColumnName column) { return null; } int idx = struct.fieldNames().indexOf(column); - if (idx < 0 || idx >= root.children().size()) { + if (idx < 0) { return null; } - return firstZoned(root.children().get(idx)); + return firstZoned(root.children().get(fieldChildOffset(struct) + idx)); + } + + /// Index of field 0 among the root struct layout's children: a nullable struct's layout leads + /// with its validity child (open checks the count, as Rust's `validate_children` does), so the + /// columns start one later. The root validity itself is not applied: [Chunk] has no row-level + /// nulls, so a null root row reads as its field values. + private static int fieldChildOffset(DType.Struct root) { + return root.nullable() ? 1 : 0; } private static Layout firstZoned(Layout layout) { @@ -711,13 +719,14 @@ private void initialize() { if (rootLayout.isStruct() && rootDtype instanceof DType.Struct structDtype) { List projection = options.columns(); - for (int i = 0; i < rootLayout.children().size(); i++) { + int offset = fieldChildOffset(structDtype); + for (int i = 0; i < structDtype.fieldNames().size(); i++) { ColumnName colName = structDtype.fieldNames().get(i); DType colDtype = structDtype.fieldTypes().get(i); if (!projection.isEmpty() && !projection.contains(colName)) { continue; } - Layout colTop = rootLayout.children().get(i); + Layout colTop = rootLayout.children().get(offset + i); var flats = new ArrayList(); collectFlats(colTop, flats); columnFlats.put(colName, flats); diff --git a/reader/src/test/java/io/github/dfa1/vortex/reader/PostscriptParserDTypeGuardsTest.java b/reader/src/test/java/io/github/dfa1/vortex/reader/PostscriptParserDTypeGuardsTest.java index 9e62e6437..179a9f673 100644 --- a/reader/src/test/java/io/github/dfa1/vortex/reader/PostscriptParserDTypeGuardsTest.java +++ b/reader/src/test/java/io/github/dfa1/vortex/reader/PostscriptParserDTypeGuardsTest.java @@ -64,6 +64,78 @@ void convertDType_structNamesDtypesArityMismatch_throwsVortexException() { .hasMessageContaining("names/dtypes length mismatch"); } + @Test + void convertDType_structWithoutNamesVector_throwsVortexException() { + // Given — a struct with no names vector at all, which Rust refuses (StructFields::from_fb) + // while it accepts an empty one; read as 0 fields, it failed the scan as a raw + // ArrayIndexOutOfBoundsException (a fuzzer finding) + MemorySegment dtype = structDType(false, true); + + // When / Then + MemorySegment footer = minimalFooter(); + MemorySegment layout = flatLayout(); + assertThatThrownBy(() -> PostscriptParser.parseBlobs(footer, layout, dtype)) + .isInstanceOf(VortexException.class) + .hasMessageContaining("failed to parse struct names"); + } + + @Test + void convertDType_structWithoutDtypesVector_throwsVortexException() { + // Given — the same for the dtypes vector + MemorySegment dtype = structDType(true, false); + + // When / Then + MemorySegment footer = minimalFooter(); + MemorySegment layout = flatLayout(); + assertThatThrownBy(() -> PostscriptParser.parseBlobs(footer, layout, dtype)) + .isInstanceOf(VortexException.class) + .hasMessageContaining("failed to parse struct dtypes"); + } + + @Test + void convertDType_structWithEmptyVectors_isAZeroFieldStruct() { + // Given — both vectors present but empty: Rust's legal 0-field struct + MemorySegment dtype = structDType(true, true); + + // When + MemorySegment footer = minimalFooter(); + MemorySegment layout = flatLayout(); + PostscriptParser.ParsedFile result = PostscriptParser.parseBlobs(footer, layout, dtype); + + // Then + assertThat(result.dtype()).isEqualTo(new DType.Struct(List.of(), List.of(), false)); + } + + @org.junit.jupiter.params.ParameterizedTest + @org.junit.jupiter.params.provider.ValueSource(ints = {0, 2}) + void parseBlobs_structLayoutChildCountNotFieldCount_throwsVortexException(int children) { + // Given — a one-field struct dtype over a struct layout with too few or too many children: + // Rust refuses it while deserializing the layout tree; the scan used to index the field + // list by layout child and throw ArrayIndexOutOfBoundsException (a fuzzer finding) + MemorySegment dtype = structDType(new String[]{"a"}, 1); + + // When / Then + MemorySegment footer = flatAndStructFooter(); + MemorySegment layout = structLayout(children); + assertThatThrownBy(() -> PostscriptParser.parseBlobs(footer, layout, dtype)) + .isInstanceOf(VortexException.class) + .hasMessageContaining("vortex.struct layout has " + children + " children but its struct dtype needs 1"); + } + + @Test + void parseBlobs_structLayoutOneChildPerField_isAccepted() { + // Given + MemorySegment dtype = structDType(new String[]{"a"}, 1); + + // When + MemorySegment footer = flatAndStructFooter(); + MemorySegment layout = structLayout(1); + PostscriptParser.ParsedFile result = PostscriptParser.parseBlobs(footer, layout, dtype); + + // Then + assertThat(result.layout().children()).hasSize(1); + } + @org.junit.jupiter.params.ParameterizedTest @org.junit.jupiter.params.provider.ValueSource(strings = {"", " ", " "}) void convertDType_blankFieldName_isAccepted(String name) { @@ -354,6 +426,18 @@ private static MemorySegment structDType(String[] names, int dtypeCount) { return fbb.dataSegment(); } + /// A 0-field struct dtype whose names/dtypes vectors are written empty or left out entirely + /// (a zero offset omits the field). + private static MemorySegment structDType(boolean withNames, boolean withDtypes) { + var fbb = new FbsBuilder(256); + int namesVec = withNames ? FbsStruct.createNamesVector(fbb, new int[0]) : 0; + int dtypesVec = withDtypes ? FbsStruct.createDtypesVector(fbb, new int[0]) : 0; + int struct = FbsStruct.createFbsStruct(fbb, namesVec, dtypesVec, false); + int root = FbsDType.createFbsDType(fbb, FbsType.FbsStruct, struct); + fbb.finish(root); + return fbb.dataSegment(); + } + private static MemorySegment minimalFooter() { var fbb = new FbsBuilder(256); int asv = FbsFooter.createArraySpecsVector(fbb, new int[]{ @@ -367,6 +451,35 @@ private static MemorySegment minimalFooter() { return fbb.dataSegment(); } + /// A footer declaring layout ids 0 = `vortex.flat`, 1 = `vortex.struct`. + private static MemorySegment flatAndStructFooter() { + var fbb = new FbsBuilder(256); + int asv = FbsFooter.createArraySpecsVector(fbb, new int[]{ + FbsArraySpec.createFbsArraySpec(fbb, fbb.createString("vortex.primitive"))}); + int lsv = FbsFooter.createLayoutSpecsVector(fbb, new int[]{ + FbsLayoutSpec.createFbsLayoutSpec(fbb, fbb.createString("vortex.flat")), + FbsLayoutSpec.createFbsLayoutSpec(fbb, fbb.createString("vortex.struct"))}); + FbsFooter.startSegmentSpecsVector(fbb, 0); + int ssv = fbb.endVector(); + int footOff = FbsFooter.createFbsFooter(fbb, asv, lsv, ssv, 0, 0); + fbb.finish(footOff); + return fbb.dataSegment(); + } + + /// A struct layout (id 1 in [#flatAndStructFooter()]) over `children` flat layouts. + private static MemorySegment structLayout(int children) { + var fbb = new FbsBuilder(256); + int[] flats = new int[children]; + for (int i = 0; i < children; i++) { + int segV = FbsLayout.createSegmentsVector(fbb, new long[]{0}); + flats[i] = FbsLayout.createFbsLayout(fbb, 0, 1L, 0, 0, segV); + } + int childV = FbsLayout.createChildrenVector(fbb, flats); + int off = FbsLayout.createFbsLayout(fbb, 1, 1L, 0, childV, 0); + FbsLayout.finishFbsLayoutBuffer(fbb, off); + return fbb.dataSegment(); + } + private static MemorySegment flatLayout() { var fbb = new FbsBuilder(128); int segV = FbsLayout.createSegmentsVector(fbb, new long[]{0}); diff --git a/reader/src/test/java/io/github/dfa1/vortex/reader/ScanIteratorNullableRootTest.java b/reader/src/test/java/io/github/dfa1/vortex/reader/ScanIteratorNullableRootTest.java new file mode 100644 index 000000000..54b713334 --- /dev/null +++ b/reader/src/test/java/io/github/dfa1/vortex/reader/ScanIteratorNullableRootTest.java @@ -0,0 +1,57 @@ +package io.github.dfa1.vortex.reader; + +import io.github.dfa1.vortex.core.model.ColumnName; +import io.github.dfa1.vortex.core.model.DType; +import io.github.dfa1.vortex.core.model.LayoutId; +import io.github.dfa1.vortex.reader.layout.Layout; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.mockito.Mock; +import org.mockito.junit.jupiter.MockitoExtension; + +import java.lang.foreign.MemorySegment; +import java.util.List; + +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.BDDMockito.given; +import static org.mockito.BDDMockito.then; + +/// A nullable root struct's layout leads with its validity child, as Rust writes it and as open +/// checks it: the scan must map field `i` to child `i + 1`. Mapping by raw index read column `a` +/// from the validity child, `b` from `a`'s data, and ran off the field list on the last child. +@ExtendWith(MockitoExtension.class) +class ScanIteratorNullableRootTest { + + private static final ColumnName A = ColumnName.of("a"); + private static final ColumnName B = ColumnName.of("b"); + + @Mock + private VortexHandle file; + + @Test + void columnsSkipTheRootValidityChild() { + // Given — layout children [validity (segment 0), a (segment 1), b (segment 2)] + DType.Struct schema = new DType.Struct(List.of(A, B), List.of(DType.I64, DType.I64), true); + List children = List.of(flat(0), flat(1), flat(2)); + SegmentSpec bSegment = new SegmentSpec(200, 8, (byte) 0, CompressionScheme.NONE); + given(file.dtype()).willReturn(schema); + given(file.layout()).willReturn(new Layout(LayoutId.STRUCT, 5, null, children, List.of())); + given(file.footer()).willReturn(new Footer(List.of(), List.of(), List.of( + new SegmentSpec(0, 8, (byte) 0, CompressionScheme.NONE), + new SegmentSpec(100, 8, (byte) 0, CompressionScheme.NONE), + bSegment), List.of())); + given(file.rawSegment(any())).willReturn(MemorySegment.ofArray(new byte[8])); + + // When + try (ScanIterator sut = new ScanIterator(file, ScanOptions.all())) { + sut.chunkStats(B); + } + + // Then — column b's stats come from b's own segment + then(file).should().rawSegment(bSegment); + } + + private static Layout flat(int segment) { + return new Layout(LayoutId.FLAT, 5, null, List.of(), List.of(segment)); + } +} From 3500150f2176d4d4757aca8d962e3d562b49f10d Mon Sep 17 00:00:00 2001 From: Davide Angelocola Date: Sun, 18 Oct 2026 16:50:29 +0200 Subject: [PATCH 05/13] feat(core): verify every FlatBuffer before reading it, as Rust does Rust reads no FlatBuffer without flatbuffers::root::(), which verifies the whole buffer first. vortex-java's FbsTable accessors trust every offset, so each parse site had to frame its own parse in a catch-all; the fuzzer found three sites that did not, each escaping as a raw JDK exception. fbs-gen now emits a package-private verify(FbsVerifier, long) per table, and every getRootAsX runs it before positioning the reader. FbsVerifier holds flatbuffers' rules with Vortex's limits: table, vtable, field, offset, vector and string bounds, NUL terminators, required fields, at most 64 nested tables and 1M tables. Alignment, UTF-8 and the apparent-size cap are not checked yet. With it, the reader drops what only guarded FlatBuffer navigation: the array, layout and dtype depth caps (the verifier's nesting limit, Rust's own, binds first; a dtype level is two tables, so dtypes now nest ~31 deep, as in Rust), the array-parse catch-all, its vector-count guard and the HTTP postscript wrapper. The footer catch-all stays: it also guards DType record validation. Fuzzed 5 min each: openMalformedFile 4.36M inputs, no finding. The inspector's own ArrayNode depth cap goes too, as in the reader; its node-count cap stays, since it bounds aliased children far below the verifier's million-table limit. Refs #493 Co-Authored-By: Claude Sonnet 5 Claude-Session: https://claude.ai/code/session_01WtuVGQ5h5Lk6Zn4JnkxF1K --- CHANGELOG.md | 1 + CLAUDE.md | 8 +- .../github/dfa1/vortex/core/fbs/FbsArray.java | 15 +- .../dfa1/vortex/core/fbs/FbsArrayNode.java | 18 +- .../dfa1/vortex/core/fbs/FbsArraySpec.java | 14 +- .../dfa1/vortex/core/fbs/FbsArrayStats.java | 24 +- .../dfa1/vortex/core/fbs/FbsBinary.java | 14 +- .../github/dfa1/vortex/core/fbs/FbsBool.java | 14 +- .../vortex/core/fbs/FbsCompressionSpec.java | 14 +- .../github/dfa1/vortex/core/fbs/FbsDType.java | 34 ++- .../dfa1/vortex/core/fbs/FbsDecimal.java | 16 +- .../vortex/core/fbs/FbsEncryptionSpec.java | 13 +- .../dfa1/vortex/core/fbs/FbsExtension.java | 16 +- .../vortex/core/fbs/FbsFileStatistics.java | 14 +- .../vortex/core/fbs/FbsFixedSizeList.java | 16 +- .../dfa1/vortex/core/fbs/FbsFooter.java | 18 +- .../dfa1/vortex/core/fbs/FbsLayout.java | 18 +- .../dfa1/vortex/core/fbs/FbsLayoutSpec.java | 14 +- .../github/dfa1/vortex/core/fbs/FbsList.java | 15 +- .../github/dfa1/vortex/core/fbs/FbsMap.java | 17 +- .../github/dfa1/vortex/core/fbs/FbsNull.java | 13 +- .../dfa1/vortex/core/fbs/FbsPostscript.java | 17 +- .../vortex/core/fbs/FbsPostscriptSegment.java | 18 +- .../dfa1/vortex/core/fbs/FbsPrimitive.java | 15 +- .../dfa1/vortex/core/fbs/FbsStruct.java | 16 +- .../github/dfa1/vortex/core/fbs/FbsTable.java | 14 +- .../github/dfa1/vortex/core/fbs/FbsUnion.java | 17 +- .../github/dfa1/vortex/core/fbs/FbsUtf8.java | 14 +- .../dfa1/vortex/core/fbs/FbsVariant.java | 14 +- .../dfa1/vortex/core/fbs/FbsVerifier.java | 281 ++++++++++++++++++ .../dfa1/vortex/core/fbs/FbsVerifierTest.java | 279 +++++++++++++++++ .../io/github/dfa1/vortex/fbsgen/CodeGen.java | 96 +++++- .../dfa1/vortex/fbsgen/CodeGenTest.java | 20 ++ .../dfa1/vortex/inspect/InspectorTree.java | 26 +- .../vortex/inspect/InspectorTreeTest.java | 14 +- .../dfa1/vortex/reader/PostscriptParser.java | 46 +-- .../dfa1/vortex/reader/ScanIterator.java | 4 +- .../vortex/reader/SerializedArrayDecoder.java | 77 ++--- .../dfa1/vortex/reader/VortexHttpReader.java | 44 +-- .../ArrayNodeDepthBombSecurityTest.java | 27 +- .../reader/DTypeDepthBombSecurityTest.java | 6 +- .../PostscriptParserParseBlobsBoundsTest.java | 29 +- .../SerializedArrayBoundsSecurityTest.java | 10 +- .../decode/ChunkedEncodingDecoderTest.java | 8 +- 44 files changed, 1157 insertions(+), 261 deletions(-) create mode 100644 core/src/main/java/io/github/dfa1/vortex/core/fbs/FbsVerifier.java create mode 100644 core/src/test/java/io/github/dfa1/vortex/core/fbs/FbsVerifierTest.java diff --git a/CHANGELOG.md b/CHANGELOG.md index d79790d36..409a48e9b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -18,6 +18,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - Custom extension types: register an `ExtensionDecoder` on `ReadRegistry` and an `ExtensionEncoder` on `WriteRegistry`, with a `validate` hook on both sides, and `Chunk.as` decodes any registered type. ([#530](https://github.com/dfa1/vortex-java/issues/530)) ### Changed +- Every FlatBuffer in a file is verified before it is read, as Rust verifies it, so corrupt file metadata fails with `VortexException`, and like Rust the reader now refuses tables nested more than 64 deep. ([#493](https://github.com/dfa1/vortex-java/issues/493)) - SQL `SUM` and `AVG` of a decimal column, grouped or not, are 4 to 6 times faster: they are totalled exactly as unscaled longs in the scan instead of by Calcite with a `BigDecimal` per row. ([#563](https://github.com/dfa1/vortex-java/pull/563)) - SQL `COUNT(DISTINCT col)` is up to 4 times faster: distinct values are counted in the scan, integers by their bits in a primitive map, instead of Calcite hashing a boxed value per row. ([#562](https://github.com/dfa1/vortex-java/pull/562)) - A SQL `WHERE` comparing a `SMALLINT` or `TINYINT` column with a literal is pushed into the scan and its aggregates, instead of being left to Calcite because of the cast Calcite adds. ([#560](https://github.com/dfa1/vortex-java/pull/560)) diff --git a/CLAUDE.md b/CLAUDE.md index 13fdb4d10..c9b9318e2 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -111,7 +111,10 @@ Both schema languages are compiled in-process to MemorySegment-native Java, with `flatc`/`protoc` and no `com.google.flatbuffers`/`protobuf-java` runtime (ADR 0017): - **`.fbs` → `fbs-gen`** (`io.github.dfa1.vortex.fbsgen`): generates readers extending `FbsTable` (vtable-based) or `FbsMemorySegment` (fixed-offset inline) and builders over - `FbsBuilder`, all in the same generated package `io.github.dfa1.vortex.core.fbs`. The + `FbsBuilder`, all in the same generated package `io.github.dfa1.vortex.core.fbs`. Every + generated `getRootAsX` first runs `FbsVerifier` over the whole buffer (Rust's + `flatbuffers::root`: bounds, vtables, required fields, ≤ 64 nested tables, ≤ 1M tables), so + reader code never guards FlatBuffer offsets itself. The runtime base classes `FbsTable`/`FbsMemorySegment` are package-private (only generated readers extend them); `FbsBuilder` is public because the writer module assembles FlatBuffers with it. Schema names with a trailing `_` (e.g. `Struct_`) have the underscore stripped in @@ -243,7 +246,8 @@ almost always this. The reader memory-maps and parses untrusted binary input. Every malformed input must throw `VortexException`, never `ArrayIndexOutOfBoundsException`, `NegativeArraySizeException`, `OutOfMemoryError`, `StackOverflowError`, a raw FlatBuffer runtime exception, or a Protobuf parser -exception. See the [`security` issues](https://github.com/dfa1/vortex-java/labels/security) for the current gap list (per-encoding adversarial +exception. Reject input where it enters, as Rust does: FlatBuffers are verified whole when parsed, +dtype and layout tree shape at open, and only segment contents at decode. See the [`security` issues](https://github.com/dfa1/vortex-java/labels/security) for the current gap list (per-encoding adversarial tests, resource caps, fuzz infra) and [ADR 0003](adr/0003-vortex-exception-sanitization.md) / [ADR 0004](adr/0004-resource-caps-read-options.md) for the design. diff --git a/core/src/main/java/io/github/dfa1/vortex/core/fbs/FbsArray.java b/core/src/main/java/io/github/dfa1/vortex/core/fbs/FbsArray.java index 5ce550a54..2dc458754 100644 --- a/core/src/main/java/io/github/dfa1/vortex/core/fbs/FbsArray.java +++ b/core/src/main/java/io/github/dfa1/vortex/core/fbs/FbsArray.java @@ -9,11 +9,22 @@ @Generated("io.github.dfa1.vortex.fbsgen.CodeGen") public final class FbsArray extends FbsTable { - /// Positions a reader at the root table of a finished buffer. + /// Verifies a finished buffer and positions a reader at its root table. /// @param seg the buffer /// @return a reader positioned at the root + /// @throws io.github.dfa1.vortex.core.error.VortexException if the buffer is malformed public static FbsArray getRootAsFbsArray(MemorySegment seg) { - return new FbsArray().assign(seg, FbsTable.rootPosition(seg, 0)); + return new FbsArray().assign(seg, FbsVerifier.verifyRoot(seg, FbsArray::verify)); + } + + /// Checks this table and everything reachable from it. + /// @param v the verifier walking the buffer + /// @param pos the table's absolute position + static void verify(FbsVerifier v, long pos) { + v.startTable(pos); + v.table(pos, 4, "root", false, FbsArrayNode::verify); + v.vector(pos, 6, 8, "buffers", false); + v.endTable(); } /// Positions this reader at a table. diff --git a/core/src/main/java/io/github/dfa1/vortex/core/fbs/FbsArrayNode.java b/core/src/main/java/io/github/dfa1/vortex/core/fbs/FbsArrayNode.java index 87a12e74c..bbbc5665a 100644 --- a/core/src/main/java/io/github/dfa1/vortex/core/fbs/FbsArrayNode.java +++ b/core/src/main/java/io/github/dfa1/vortex/core/fbs/FbsArrayNode.java @@ -9,11 +9,25 @@ @Generated("io.github.dfa1.vortex.fbsgen.CodeGen") public final class FbsArrayNode extends FbsTable { - /// Positions a reader at the root table of a finished buffer. + /// Verifies a finished buffer and positions a reader at its root table. /// @param seg the buffer /// @return a reader positioned at the root + /// @throws io.github.dfa1.vortex.core.error.VortexException if the buffer is malformed public static FbsArrayNode getRootAsFbsArrayNode(MemorySegment seg) { - return new FbsArrayNode().assign(seg, FbsTable.rootPosition(seg, 0)); + return new FbsArrayNode().assign(seg, FbsVerifier.verifyRoot(seg, FbsArrayNode::verify)); + } + + /// Checks this table and everything reachable from it. + /// @param v the verifier walking the buffer + /// @param pos the table's absolute position + static void verify(FbsVerifier v, long pos) { + v.startTable(pos); + v.inline(pos, 4, 2, "encoding", false); + v.vector(pos, 6, 1, "metadata", false); + v.tableVector(pos, 8, "children", false, FbsArrayNode::verify); + v.vector(pos, 10, 2, "buffers", false); + v.table(pos, 12, "stats", false, FbsArrayStats::verify); + v.endTable(); } /// Positions this reader at a table. diff --git a/core/src/main/java/io/github/dfa1/vortex/core/fbs/FbsArraySpec.java b/core/src/main/java/io/github/dfa1/vortex/core/fbs/FbsArraySpec.java index 3c231ed24..30e94389d 100644 --- a/core/src/main/java/io/github/dfa1/vortex/core/fbs/FbsArraySpec.java +++ b/core/src/main/java/io/github/dfa1/vortex/core/fbs/FbsArraySpec.java @@ -9,11 +9,21 @@ @Generated("io.github.dfa1.vortex.fbsgen.CodeGen") public final class FbsArraySpec extends FbsTable { - /// Positions a reader at the root table of a finished buffer. + /// Verifies a finished buffer and positions a reader at its root table. /// @param seg the buffer /// @return a reader positioned at the root + /// @throws io.github.dfa1.vortex.core.error.VortexException if the buffer is malformed public static FbsArraySpec getRootAsFbsArraySpec(MemorySegment seg) { - return new FbsArraySpec().assign(seg, FbsTable.rootPosition(seg, 0)); + return new FbsArraySpec().assign(seg, FbsVerifier.verifyRoot(seg, FbsArraySpec::verify)); + } + + /// Checks this table and everything reachable from it. + /// @param v the verifier walking the buffer + /// @param pos the table's absolute position + static void verify(FbsVerifier v, long pos) { + v.startTable(pos); + v.string(pos, 4, "id", true); + v.endTable(); } /// Positions this reader at a table. diff --git a/core/src/main/java/io/github/dfa1/vortex/core/fbs/FbsArrayStats.java b/core/src/main/java/io/github/dfa1/vortex/core/fbs/FbsArrayStats.java index 969ff2588..b97094a44 100644 --- a/core/src/main/java/io/github/dfa1/vortex/core/fbs/FbsArrayStats.java +++ b/core/src/main/java/io/github/dfa1/vortex/core/fbs/FbsArrayStats.java @@ -9,11 +9,31 @@ @Generated("io.github.dfa1.vortex.fbsgen.CodeGen") public final class FbsArrayStats extends FbsTable { - /// Positions a reader at the root table of a finished buffer. + /// Verifies a finished buffer and positions a reader at its root table. /// @param seg the buffer /// @return a reader positioned at the root + /// @throws io.github.dfa1.vortex.core.error.VortexException if the buffer is malformed public static FbsArrayStats getRootAsFbsArrayStats(MemorySegment seg) { - return new FbsArrayStats().assign(seg, FbsTable.rootPosition(seg, 0)); + return new FbsArrayStats().assign(seg, FbsVerifier.verifyRoot(seg, FbsArrayStats::verify)); + } + + /// Checks this table and everything reachable from it. + /// @param v the verifier walking the buffer + /// @param pos the table's absolute position + static void verify(FbsVerifier v, long pos) { + v.startTable(pos); + v.vector(pos, 4, 1, "min", false); + v.inline(pos, 6, 1, "min_precision", false); + v.vector(pos, 8, 1, "max", false); + v.inline(pos, 10, 1, "max_precision", false); + v.vector(pos, 12, 1, "sum", false); + v.inline(pos, 14, 1, "is_sorted", false); + v.inline(pos, 16, 1, "is_strict_sorted", false); + v.inline(pos, 18, 1, "is_constant", false); + v.inline(pos, 20, 8, "null_count", false); + v.inline(pos, 22, 8, "uncompressed_size_in_bytes", false); + v.inline(pos, 24, 8, "nan_count", false); + v.endTable(); } /// Positions this reader at a table. diff --git a/core/src/main/java/io/github/dfa1/vortex/core/fbs/FbsBinary.java b/core/src/main/java/io/github/dfa1/vortex/core/fbs/FbsBinary.java index 90bfb9b86..5eecef674 100644 --- a/core/src/main/java/io/github/dfa1/vortex/core/fbs/FbsBinary.java +++ b/core/src/main/java/io/github/dfa1/vortex/core/fbs/FbsBinary.java @@ -9,11 +9,21 @@ @Generated("io.github.dfa1.vortex.fbsgen.CodeGen") public final class FbsBinary extends FbsTable { - /// Positions a reader at the root table of a finished buffer. + /// Verifies a finished buffer and positions a reader at its root table. /// @param seg the buffer /// @return a reader positioned at the root + /// @throws io.github.dfa1.vortex.core.error.VortexException if the buffer is malformed public static FbsBinary getRootAsFbsBinary(MemorySegment seg) { - return new FbsBinary().assign(seg, FbsTable.rootPosition(seg, 0)); + return new FbsBinary().assign(seg, FbsVerifier.verifyRoot(seg, FbsBinary::verify)); + } + + /// Checks this table and everything reachable from it. + /// @param v the verifier walking the buffer + /// @param pos the table's absolute position + static void verify(FbsVerifier v, long pos) { + v.startTable(pos); + v.inline(pos, 4, 1, "nullable", false); + v.endTable(); } /// Positions this reader at a table. diff --git a/core/src/main/java/io/github/dfa1/vortex/core/fbs/FbsBool.java b/core/src/main/java/io/github/dfa1/vortex/core/fbs/FbsBool.java index 85646d6d2..cff207a9b 100644 --- a/core/src/main/java/io/github/dfa1/vortex/core/fbs/FbsBool.java +++ b/core/src/main/java/io/github/dfa1/vortex/core/fbs/FbsBool.java @@ -9,11 +9,21 @@ @Generated("io.github.dfa1.vortex.fbsgen.CodeGen") public final class FbsBool extends FbsTable { - /// Positions a reader at the root table of a finished buffer. + /// Verifies a finished buffer and positions a reader at its root table. /// @param seg the buffer /// @return a reader positioned at the root + /// @throws io.github.dfa1.vortex.core.error.VortexException if the buffer is malformed public static FbsBool getRootAsFbsBool(MemorySegment seg) { - return new FbsBool().assign(seg, FbsTable.rootPosition(seg, 0)); + return new FbsBool().assign(seg, FbsVerifier.verifyRoot(seg, FbsBool::verify)); + } + + /// Checks this table and everything reachable from it. + /// @param v the verifier walking the buffer + /// @param pos the table's absolute position + static void verify(FbsVerifier v, long pos) { + v.startTable(pos); + v.inline(pos, 4, 1, "nullable", false); + v.endTable(); } /// Positions this reader at a table. diff --git a/core/src/main/java/io/github/dfa1/vortex/core/fbs/FbsCompressionSpec.java b/core/src/main/java/io/github/dfa1/vortex/core/fbs/FbsCompressionSpec.java index b287a3572..af2d6c072 100644 --- a/core/src/main/java/io/github/dfa1/vortex/core/fbs/FbsCompressionSpec.java +++ b/core/src/main/java/io/github/dfa1/vortex/core/fbs/FbsCompressionSpec.java @@ -9,11 +9,21 @@ @Generated("io.github.dfa1.vortex.fbsgen.CodeGen") public final class FbsCompressionSpec extends FbsTable { - /// Positions a reader at the root table of a finished buffer. + /// Verifies a finished buffer and positions a reader at its root table. /// @param seg the buffer /// @return a reader positioned at the root + /// @throws io.github.dfa1.vortex.core.error.VortexException if the buffer is malformed public static FbsCompressionSpec getRootAsFbsCompressionSpec(MemorySegment seg) { - return new FbsCompressionSpec().assign(seg, FbsTable.rootPosition(seg, 0)); + return new FbsCompressionSpec().assign(seg, FbsVerifier.verifyRoot(seg, FbsCompressionSpec::verify)); + } + + /// Checks this table and everything reachable from it. + /// @param v the verifier walking the buffer + /// @param pos the table's absolute position + static void verify(FbsVerifier v, long pos) { + v.startTable(pos); + v.inline(pos, 4, 1, "scheme", false); + v.endTable(); } /// Positions this reader at a table. diff --git a/core/src/main/java/io/github/dfa1/vortex/core/fbs/FbsDType.java b/core/src/main/java/io/github/dfa1/vortex/core/fbs/FbsDType.java index 54e8bae42..5bfaf20f0 100644 --- a/core/src/main/java/io/github/dfa1/vortex/core/fbs/FbsDType.java +++ b/core/src/main/java/io/github/dfa1/vortex/core/fbs/FbsDType.java @@ -9,11 +9,41 @@ @Generated("io.github.dfa1.vortex.fbsgen.CodeGen") public final class FbsDType extends FbsTable { - /// Positions a reader at the root table of a finished buffer. + /// Verifies a finished buffer and positions a reader at its root table. /// @param seg the buffer /// @return a reader positioned at the root + /// @throws io.github.dfa1.vortex.core.error.VortexException if the buffer is malformed public static FbsDType getRootAsFbsDType(MemorySegment seg) { - return new FbsDType().assign(seg, FbsTable.rootPosition(seg, 0)); + return new FbsDType().assign(seg, FbsVerifier.verifyRoot(seg, FbsDType::verify)); + } + + /// Checks this table and everything reachable from it. + /// @param v the verifier walking the buffer + /// @param pos the table's absolute position + static void verify(FbsVerifier v, long pos) { + v.startTable(pos); + int typeType = v.unionType(pos, 4, 6, "type"); + if (typeType != 0) { + long member = v.unionValue(pos, 6, "type"); + switch (typeType) { + case 1 -> FbsNull.verify(v, member); + case 2 -> FbsBool.verify(v, member); + case 3 -> FbsPrimitive.verify(v, member); + case 4 -> FbsDecimal.verify(v, member); + case 5 -> FbsUtf8.verify(v, member); + case 6 -> FbsBinary.verify(v, member); + case 7 -> FbsStruct.verify(v, member); + case 8 -> FbsList.verify(v, member); + case 9 -> FbsExtension.verify(v, member); + case 10 -> FbsFixedSizeList.verify(v, member); + case 11 -> FbsVariant.verify(v, member); + case 12 -> FbsUnion.verify(v, member); + case 13 -> FbsMap.verify(v, member); + default -> { + } + } + } + v.endTable(); } /// Positions this reader at a table. diff --git a/core/src/main/java/io/github/dfa1/vortex/core/fbs/FbsDecimal.java b/core/src/main/java/io/github/dfa1/vortex/core/fbs/FbsDecimal.java index 2fa2d2f24..a7c60f20a 100644 --- a/core/src/main/java/io/github/dfa1/vortex/core/fbs/FbsDecimal.java +++ b/core/src/main/java/io/github/dfa1/vortex/core/fbs/FbsDecimal.java @@ -9,11 +9,23 @@ @Generated("io.github.dfa1.vortex.fbsgen.CodeGen") public final class FbsDecimal extends FbsTable { - /// Positions a reader at the root table of a finished buffer. + /// Verifies a finished buffer and positions a reader at its root table. /// @param seg the buffer /// @return a reader positioned at the root + /// @throws io.github.dfa1.vortex.core.error.VortexException if the buffer is malformed public static FbsDecimal getRootAsFbsDecimal(MemorySegment seg) { - return new FbsDecimal().assign(seg, FbsTable.rootPosition(seg, 0)); + return new FbsDecimal().assign(seg, FbsVerifier.verifyRoot(seg, FbsDecimal::verify)); + } + + /// Checks this table and everything reachable from it. + /// @param v the verifier walking the buffer + /// @param pos the table's absolute position + static void verify(FbsVerifier v, long pos) { + v.startTable(pos); + v.inline(pos, 4, 1, "precision", false); + v.inline(pos, 6, 1, "scale", false); + v.inline(pos, 8, 1, "nullable", false); + v.endTable(); } /// Positions this reader at a table. diff --git a/core/src/main/java/io/github/dfa1/vortex/core/fbs/FbsEncryptionSpec.java b/core/src/main/java/io/github/dfa1/vortex/core/fbs/FbsEncryptionSpec.java index 01cbbdd0f..039cb68d1 100644 --- a/core/src/main/java/io/github/dfa1/vortex/core/fbs/FbsEncryptionSpec.java +++ b/core/src/main/java/io/github/dfa1/vortex/core/fbs/FbsEncryptionSpec.java @@ -9,11 +9,20 @@ @Generated("io.github.dfa1.vortex.fbsgen.CodeGen") public final class FbsEncryptionSpec extends FbsTable { - /// Positions a reader at the root table of a finished buffer. + /// Verifies a finished buffer and positions a reader at its root table. /// @param seg the buffer /// @return a reader positioned at the root + /// @throws io.github.dfa1.vortex.core.error.VortexException if the buffer is malformed public static FbsEncryptionSpec getRootAsFbsEncryptionSpec(MemorySegment seg) { - return new FbsEncryptionSpec().assign(seg, FbsTable.rootPosition(seg, 0)); + return new FbsEncryptionSpec().assign(seg, FbsVerifier.verifyRoot(seg, FbsEncryptionSpec::verify)); + } + + /// Checks this table and everything reachable from it. + /// @param v the verifier walking the buffer + /// @param pos the table's absolute position + static void verify(FbsVerifier v, long pos) { + v.startTable(pos); + v.endTable(); } /// Positions this reader at a table. diff --git a/core/src/main/java/io/github/dfa1/vortex/core/fbs/FbsExtension.java b/core/src/main/java/io/github/dfa1/vortex/core/fbs/FbsExtension.java index ce42f2cb4..4d2e3eb49 100644 --- a/core/src/main/java/io/github/dfa1/vortex/core/fbs/FbsExtension.java +++ b/core/src/main/java/io/github/dfa1/vortex/core/fbs/FbsExtension.java @@ -9,11 +9,23 @@ @Generated("io.github.dfa1.vortex.fbsgen.CodeGen") public final class FbsExtension extends FbsTable { - /// Positions a reader at the root table of a finished buffer. + /// Verifies a finished buffer and positions a reader at its root table. /// @param seg the buffer /// @return a reader positioned at the root + /// @throws io.github.dfa1.vortex.core.error.VortexException if the buffer is malformed public static FbsExtension getRootAsFbsExtension(MemorySegment seg) { - return new FbsExtension().assign(seg, FbsTable.rootPosition(seg, 0)); + return new FbsExtension().assign(seg, FbsVerifier.verifyRoot(seg, FbsExtension::verify)); + } + + /// Checks this table and everything reachable from it. + /// @param v the verifier walking the buffer + /// @param pos the table's absolute position + static void verify(FbsVerifier v, long pos) { + v.startTable(pos); + v.string(pos, 4, "id", false); + v.table(pos, 6, "storage_dtype", false, FbsDType::verify); + v.vector(pos, 8, 1, "metadata", false); + v.endTable(); } /// Positions this reader at a table. diff --git a/core/src/main/java/io/github/dfa1/vortex/core/fbs/FbsFileStatistics.java b/core/src/main/java/io/github/dfa1/vortex/core/fbs/FbsFileStatistics.java index 2a1b8fb2d..4a24e0ecb 100644 --- a/core/src/main/java/io/github/dfa1/vortex/core/fbs/FbsFileStatistics.java +++ b/core/src/main/java/io/github/dfa1/vortex/core/fbs/FbsFileStatistics.java @@ -9,11 +9,21 @@ @Generated("io.github.dfa1.vortex.fbsgen.CodeGen") public final class FbsFileStatistics extends FbsTable { - /// Positions a reader at the root table of a finished buffer. + /// Verifies a finished buffer and positions a reader at its root table. /// @param seg the buffer /// @return a reader positioned at the root + /// @throws io.github.dfa1.vortex.core.error.VortexException if the buffer is malformed public static FbsFileStatistics getRootAsFbsFileStatistics(MemorySegment seg) { - return new FbsFileStatistics().assign(seg, FbsTable.rootPosition(seg, 0)); + return new FbsFileStatistics().assign(seg, FbsVerifier.verifyRoot(seg, FbsFileStatistics::verify)); + } + + /// Checks this table and everything reachable from it. + /// @param v the verifier walking the buffer + /// @param pos the table's absolute position + static void verify(FbsVerifier v, long pos) { + v.startTable(pos); + v.tableVector(pos, 4, "field_stats", false, FbsArrayStats::verify); + v.endTable(); } /// Positions this reader at a table. diff --git a/core/src/main/java/io/github/dfa1/vortex/core/fbs/FbsFixedSizeList.java b/core/src/main/java/io/github/dfa1/vortex/core/fbs/FbsFixedSizeList.java index 1c0c5c83b..608e0a4e6 100644 --- a/core/src/main/java/io/github/dfa1/vortex/core/fbs/FbsFixedSizeList.java +++ b/core/src/main/java/io/github/dfa1/vortex/core/fbs/FbsFixedSizeList.java @@ -9,11 +9,23 @@ @Generated("io.github.dfa1.vortex.fbsgen.CodeGen") public final class FbsFixedSizeList extends FbsTable { - /// Positions a reader at the root table of a finished buffer. + /// Verifies a finished buffer and positions a reader at its root table. /// @param seg the buffer /// @return a reader positioned at the root + /// @throws io.github.dfa1.vortex.core.error.VortexException if the buffer is malformed public static FbsFixedSizeList getRootAsFbsFixedSizeList(MemorySegment seg) { - return new FbsFixedSizeList().assign(seg, FbsTable.rootPosition(seg, 0)); + return new FbsFixedSizeList().assign(seg, FbsVerifier.verifyRoot(seg, FbsFixedSizeList::verify)); + } + + /// Checks this table and everything reachable from it. + /// @param v the verifier walking the buffer + /// @param pos the table's absolute position + static void verify(FbsVerifier v, long pos) { + v.startTable(pos); + v.table(pos, 4, "element_type", false, FbsDType::verify); + v.inline(pos, 6, 4, "size", false); + v.inline(pos, 8, 1, "nullable", false); + v.endTable(); } /// Positions this reader at a table. diff --git a/core/src/main/java/io/github/dfa1/vortex/core/fbs/FbsFooter.java b/core/src/main/java/io/github/dfa1/vortex/core/fbs/FbsFooter.java index 909209359..a0329e14f 100644 --- a/core/src/main/java/io/github/dfa1/vortex/core/fbs/FbsFooter.java +++ b/core/src/main/java/io/github/dfa1/vortex/core/fbs/FbsFooter.java @@ -9,11 +9,25 @@ @Generated("io.github.dfa1.vortex.fbsgen.CodeGen") public final class FbsFooter extends FbsTable { - /// Positions a reader at the root table of a finished buffer. + /// Verifies a finished buffer and positions a reader at its root table. /// @param seg the buffer /// @return a reader positioned at the root + /// @throws io.github.dfa1.vortex.core.error.VortexException if the buffer is malformed public static FbsFooter getRootAsFbsFooter(MemorySegment seg) { - return new FbsFooter().assign(seg, FbsTable.rootPosition(seg, 0)); + return new FbsFooter().assign(seg, FbsVerifier.verifyRoot(seg, FbsFooter::verify)); + } + + /// Checks this table and everything reachable from it. + /// @param v the verifier walking the buffer + /// @param pos the table's absolute position + static void verify(FbsVerifier v, long pos) { + v.startTable(pos); + v.tableVector(pos, 4, "array_specs", false, FbsArraySpec::verify); + v.tableVector(pos, 6, "layout_specs", false, FbsLayoutSpec::verify); + v.vector(pos, 8, 16, "segment_specs", false); + v.tableVector(pos, 10, "compression_specs", false, FbsCompressionSpec::verify); + v.tableVector(pos, 12, "encryption_specs", false, FbsEncryptionSpec::verify); + v.endTable(); } /// Positions this reader at a table. diff --git a/core/src/main/java/io/github/dfa1/vortex/core/fbs/FbsLayout.java b/core/src/main/java/io/github/dfa1/vortex/core/fbs/FbsLayout.java index a10dd0276..772b57c9a 100644 --- a/core/src/main/java/io/github/dfa1/vortex/core/fbs/FbsLayout.java +++ b/core/src/main/java/io/github/dfa1/vortex/core/fbs/FbsLayout.java @@ -9,11 +9,25 @@ @Generated("io.github.dfa1.vortex.fbsgen.CodeGen") public final class FbsLayout extends FbsTable { - /// Positions a reader at the root table of a finished buffer. + /// Verifies a finished buffer and positions a reader at its root table. /// @param seg the buffer /// @return a reader positioned at the root + /// @throws io.github.dfa1.vortex.core.error.VortexException if the buffer is malformed public static FbsLayout getRootAsFbsLayout(MemorySegment seg) { - return new FbsLayout().assign(seg, FbsTable.rootPosition(seg, 0)); + return new FbsLayout().assign(seg, FbsVerifier.verifyRoot(seg, FbsLayout::verify)); + } + + /// Checks this table and everything reachable from it. + /// @param v the verifier walking the buffer + /// @param pos the table's absolute position + static void verify(FbsVerifier v, long pos) { + v.startTable(pos); + v.inline(pos, 4, 2, "encoding", false); + v.inline(pos, 6, 8, "row_count", false); + v.vector(pos, 8, 1, "metadata", false); + v.tableVector(pos, 10, "children", false, FbsLayout::verify); + v.vector(pos, 12, 4, "segments", false); + v.endTable(); } /// Positions this reader at a table. diff --git a/core/src/main/java/io/github/dfa1/vortex/core/fbs/FbsLayoutSpec.java b/core/src/main/java/io/github/dfa1/vortex/core/fbs/FbsLayoutSpec.java index 696d83c23..9a7d7cd09 100644 --- a/core/src/main/java/io/github/dfa1/vortex/core/fbs/FbsLayoutSpec.java +++ b/core/src/main/java/io/github/dfa1/vortex/core/fbs/FbsLayoutSpec.java @@ -9,11 +9,21 @@ @Generated("io.github.dfa1.vortex.fbsgen.CodeGen") public final class FbsLayoutSpec extends FbsTable { - /// Positions a reader at the root table of a finished buffer. + /// Verifies a finished buffer and positions a reader at its root table. /// @param seg the buffer /// @return a reader positioned at the root + /// @throws io.github.dfa1.vortex.core.error.VortexException if the buffer is malformed public static FbsLayoutSpec getRootAsFbsLayoutSpec(MemorySegment seg) { - return new FbsLayoutSpec().assign(seg, FbsTable.rootPosition(seg, 0)); + return new FbsLayoutSpec().assign(seg, FbsVerifier.verifyRoot(seg, FbsLayoutSpec::verify)); + } + + /// Checks this table and everything reachable from it. + /// @param v the verifier walking the buffer + /// @param pos the table's absolute position + static void verify(FbsVerifier v, long pos) { + v.startTable(pos); + v.string(pos, 4, "id", true); + v.endTable(); } /// Positions this reader at a table. diff --git a/core/src/main/java/io/github/dfa1/vortex/core/fbs/FbsList.java b/core/src/main/java/io/github/dfa1/vortex/core/fbs/FbsList.java index 1824bdf86..5eabecfc9 100644 --- a/core/src/main/java/io/github/dfa1/vortex/core/fbs/FbsList.java +++ b/core/src/main/java/io/github/dfa1/vortex/core/fbs/FbsList.java @@ -9,11 +9,22 @@ @Generated("io.github.dfa1.vortex.fbsgen.CodeGen") public final class FbsList extends FbsTable { - /// Positions a reader at the root table of a finished buffer. + /// Verifies a finished buffer and positions a reader at its root table. /// @param seg the buffer /// @return a reader positioned at the root + /// @throws io.github.dfa1.vortex.core.error.VortexException if the buffer is malformed public static FbsList getRootAsFbsList(MemorySegment seg) { - return new FbsList().assign(seg, FbsTable.rootPosition(seg, 0)); + return new FbsList().assign(seg, FbsVerifier.verifyRoot(seg, FbsList::verify)); + } + + /// Checks this table and everything reachable from it. + /// @param v the verifier walking the buffer + /// @param pos the table's absolute position + static void verify(FbsVerifier v, long pos) { + v.startTable(pos); + v.table(pos, 4, "element_type", false, FbsDType::verify); + v.inline(pos, 6, 1, "nullable", false); + v.endTable(); } /// Positions this reader at a table. diff --git a/core/src/main/java/io/github/dfa1/vortex/core/fbs/FbsMap.java b/core/src/main/java/io/github/dfa1/vortex/core/fbs/FbsMap.java index 32d9e6836..81e1d88fc 100644 --- a/core/src/main/java/io/github/dfa1/vortex/core/fbs/FbsMap.java +++ b/core/src/main/java/io/github/dfa1/vortex/core/fbs/FbsMap.java @@ -9,11 +9,24 @@ @Generated("io.github.dfa1.vortex.fbsgen.CodeGen") public final class FbsMap extends FbsTable { - /// Positions a reader at the root table of a finished buffer. + /// Verifies a finished buffer and positions a reader at its root table. /// @param seg the buffer /// @return a reader positioned at the root + /// @throws io.github.dfa1.vortex.core.error.VortexException if the buffer is malformed public static FbsMap getRootAsFbsMap(MemorySegment seg) { - return new FbsMap().assign(seg, FbsTable.rootPosition(seg, 0)); + return new FbsMap().assign(seg, FbsVerifier.verifyRoot(seg, FbsMap::verify)); + } + + /// Checks this table and everything reachable from it. + /// @param v the verifier walking the buffer + /// @param pos the table's absolute position + static void verify(FbsVerifier v, long pos) { + v.startTable(pos); + v.table(pos, 4, "key_type", false, FbsDType::verify); + v.table(pos, 6, "value_type", false, FbsDType::verify); + v.inline(pos, 8, 1, "keys_sorted", false); + v.inline(pos, 10, 1, "nullable", false); + v.endTable(); } /// Positions this reader at a table. diff --git a/core/src/main/java/io/github/dfa1/vortex/core/fbs/FbsNull.java b/core/src/main/java/io/github/dfa1/vortex/core/fbs/FbsNull.java index 52c23efe7..fbfed5787 100644 --- a/core/src/main/java/io/github/dfa1/vortex/core/fbs/FbsNull.java +++ b/core/src/main/java/io/github/dfa1/vortex/core/fbs/FbsNull.java @@ -9,11 +9,20 @@ @Generated("io.github.dfa1.vortex.fbsgen.CodeGen") public final class FbsNull extends FbsTable { - /// Positions a reader at the root table of a finished buffer. + /// Verifies a finished buffer and positions a reader at its root table. /// @param seg the buffer /// @return a reader positioned at the root + /// @throws io.github.dfa1.vortex.core.error.VortexException if the buffer is malformed public static FbsNull getRootAsFbsNull(MemorySegment seg) { - return new FbsNull().assign(seg, FbsTable.rootPosition(seg, 0)); + return new FbsNull().assign(seg, FbsVerifier.verifyRoot(seg, FbsNull::verify)); + } + + /// Checks this table and everything reachable from it. + /// @param v the verifier walking the buffer + /// @param pos the table's absolute position + static void verify(FbsVerifier v, long pos) { + v.startTable(pos); + v.endTable(); } /// Positions this reader at a table. diff --git a/core/src/main/java/io/github/dfa1/vortex/core/fbs/FbsPostscript.java b/core/src/main/java/io/github/dfa1/vortex/core/fbs/FbsPostscript.java index 81bff4404..7ecea622b 100644 --- a/core/src/main/java/io/github/dfa1/vortex/core/fbs/FbsPostscript.java +++ b/core/src/main/java/io/github/dfa1/vortex/core/fbs/FbsPostscript.java @@ -9,11 +9,24 @@ @Generated("io.github.dfa1.vortex.fbsgen.CodeGen") public final class FbsPostscript extends FbsTable { - /// Positions a reader at the root table of a finished buffer. + /// Verifies a finished buffer and positions a reader at its root table. /// @param seg the buffer /// @return a reader positioned at the root + /// @throws io.github.dfa1.vortex.core.error.VortexException if the buffer is malformed public static FbsPostscript getRootAsFbsPostscript(MemorySegment seg) { - return new FbsPostscript().assign(seg, FbsTable.rootPosition(seg, 0)); + return new FbsPostscript().assign(seg, FbsVerifier.verifyRoot(seg, FbsPostscript::verify)); + } + + /// Checks this table and everything reachable from it. + /// @param v the verifier walking the buffer + /// @param pos the table's absolute position + static void verify(FbsVerifier v, long pos) { + v.startTable(pos); + v.table(pos, 4, "dtype", false, FbsPostscriptSegment::verify); + v.table(pos, 6, "layout", false, FbsPostscriptSegment::verify); + v.table(pos, 8, "statistics", false, FbsPostscriptSegment::verify); + v.table(pos, 10, "footer", false, FbsPostscriptSegment::verify); + v.endTable(); } /// Positions this reader at a table. diff --git a/core/src/main/java/io/github/dfa1/vortex/core/fbs/FbsPostscriptSegment.java b/core/src/main/java/io/github/dfa1/vortex/core/fbs/FbsPostscriptSegment.java index 28679bb02..1ea600c92 100644 --- a/core/src/main/java/io/github/dfa1/vortex/core/fbs/FbsPostscriptSegment.java +++ b/core/src/main/java/io/github/dfa1/vortex/core/fbs/FbsPostscriptSegment.java @@ -9,11 +9,25 @@ @Generated("io.github.dfa1.vortex.fbsgen.CodeGen") public final class FbsPostscriptSegment extends FbsTable { - /// Positions a reader at the root table of a finished buffer. + /// Verifies a finished buffer and positions a reader at its root table. /// @param seg the buffer /// @return a reader positioned at the root + /// @throws io.github.dfa1.vortex.core.error.VortexException if the buffer is malformed public static FbsPostscriptSegment getRootAsFbsPostscriptSegment(MemorySegment seg) { - return new FbsPostscriptSegment().assign(seg, FbsTable.rootPosition(seg, 0)); + return new FbsPostscriptSegment().assign(seg, FbsVerifier.verifyRoot(seg, FbsPostscriptSegment::verify)); + } + + /// Checks this table and everything reachable from it. + /// @param v the verifier walking the buffer + /// @param pos the table's absolute position + static void verify(FbsVerifier v, long pos) { + v.startTable(pos); + v.inline(pos, 4, 8, "offset", false); + v.inline(pos, 6, 4, "length", false); + v.inline(pos, 8, 1, "alignment_exponent", false); + v.table(pos, 10, "_compression", false, FbsCompressionSpec::verify); + v.table(pos, 12, "_encryption", false, FbsEncryptionSpec::verify); + v.endTable(); } /// Positions this reader at a table. diff --git a/core/src/main/java/io/github/dfa1/vortex/core/fbs/FbsPrimitive.java b/core/src/main/java/io/github/dfa1/vortex/core/fbs/FbsPrimitive.java index 2ff86a1c6..460b4b490 100644 --- a/core/src/main/java/io/github/dfa1/vortex/core/fbs/FbsPrimitive.java +++ b/core/src/main/java/io/github/dfa1/vortex/core/fbs/FbsPrimitive.java @@ -9,11 +9,22 @@ @Generated("io.github.dfa1.vortex.fbsgen.CodeGen") public final class FbsPrimitive extends FbsTable { - /// Positions a reader at the root table of a finished buffer. + /// Verifies a finished buffer and positions a reader at its root table. /// @param seg the buffer /// @return a reader positioned at the root + /// @throws io.github.dfa1.vortex.core.error.VortexException if the buffer is malformed public static FbsPrimitive getRootAsFbsPrimitive(MemorySegment seg) { - return new FbsPrimitive().assign(seg, FbsTable.rootPosition(seg, 0)); + return new FbsPrimitive().assign(seg, FbsVerifier.verifyRoot(seg, FbsPrimitive::verify)); + } + + /// Checks this table and everything reachable from it. + /// @param v the verifier walking the buffer + /// @param pos the table's absolute position + static void verify(FbsVerifier v, long pos) { + v.startTable(pos); + v.inline(pos, 4, 1, "ptype", false); + v.inline(pos, 6, 1, "nullable", false); + v.endTable(); } /// Positions this reader at a table. diff --git a/core/src/main/java/io/github/dfa1/vortex/core/fbs/FbsStruct.java b/core/src/main/java/io/github/dfa1/vortex/core/fbs/FbsStruct.java index 79df72760..cdaa470b8 100644 --- a/core/src/main/java/io/github/dfa1/vortex/core/fbs/FbsStruct.java +++ b/core/src/main/java/io/github/dfa1/vortex/core/fbs/FbsStruct.java @@ -9,11 +9,23 @@ @Generated("io.github.dfa1.vortex.fbsgen.CodeGen") public final class FbsStruct extends FbsTable { - /// Positions a reader at the root table of a finished buffer. + /// Verifies a finished buffer and positions a reader at its root table. /// @param seg the buffer /// @return a reader positioned at the root + /// @throws io.github.dfa1.vortex.core.error.VortexException if the buffer is malformed public static FbsStruct getRootAsFbsStruct(MemorySegment seg) { - return new FbsStruct().assign(seg, FbsTable.rootPosition(seg, 0)); + return new FbsStruct().assign(seg, FbsVerifier.verifyRoot(seg, FbsStruct::verify)); + } + + /// Checks this table and everything reachable from it. + /// @param v the verifier walking the buffer + /// @param pos the table's absolute position + static void verify(FbsVerifier v, long pos) { + v.startTable(pos); + v.stringVector(pos, 4, "names", false); + v.tableVector(pos, 6, "dtypes", false, FbsDType::verify); + v.inline(pos, 8, 1, "nullable", false); + v.endTable(); } /// Positions this reader at a table. diff --git a/core/src/main/java/io/github/dfa1/vortex/core/fbs/FbsTable.java b/core/src/main/java/io/github/dfa1/vortex/core/fbs/FbsTable.java index 544e5ce0d..efb02707d 100644 --- a/core/src/main/java/io/github/dfa1/vortex/core/fbs/FbsTable.java +++ b/core/src/main/java/io/github/dfa1/vortex/core/fbs/FbsTable.java @@ -24,10 +24,8 @@ /// /// Bounds: these accessors do not validate offsets or lengths read from the buffer /// (e.g. [#readStringAt(long)] allocates `new byte[len]` with `len` taken straight from -/// the blob). This matches the previous `com.google.flatbuffers` runtime — the caller is -/// expected to have framed the FlatBuffer from a trusted region. File-level framing -/// (segment offsets/lengths, trailer, postscript) is range-checked upstream by the -/// reader's `IoBounds`; intra-FlatBuffer fields are not. +/// the blob). They need not: a reader is only ever positioned by a generated `getRootAsX`, +/// which runs [FbsVerifier] over the whole buffer first, as Rust's `flatbuffers::root` does. /// /// Package-private: only the generated table accessors in this package extend it. class FbsTable { @@ -51,14 +49,6 @@ protected final void init(MemorySegment segment, long position) { this.pos = position; } - /// Resolves the root table position of a finished FlatBuffer. - /// - /// @param segment buffer whose first 4 bytes hold the uoffset to the root table - /// @param start byte position of the buffer start within `segment` - /// @return the absolute position of the root table - static long rootPosition(MemorySegment segment, long start) { - return start + segment.get(LE_INT, start); - } /// Looks up a field's offset relative to this table. /// diff --git a/core/src/main/java/io/github/dfa1/vortex/core/fbs/FbsUnion.java b/core/src/main/java/io/github/dfa1/vortex/core/fbs/FbsUnion.java index 7d48ad077..9f8c0cd74 100644 --- a/core/src/main/java/io/github/dfa1/vortex/core/fbs/FbsUnion.java +++ b/core/src/main/java/io/github/dfa1/vortex/core/fbs/FbsUnion.java @@ -9,11 +9,24 @@ @Generated("io.github.dfa1.vortex.fbsgen.CodeGen") public final class FbsUnion extends FbsTable { - /// Positions a reader at the root table of a finished buffer. + /// Verifies a finished buffer and positions a reader at its root table. /// @param seg the buffer /// @return a reader positioned at the root + /// @throws io.github.dfa1.vortex.core.error.VortexException if the buffer is malformed public static FbsUnion getRootAsFbsUnion(MemorySegment seg) { - return new FbsUnion().assign(seg, FbsTable.rootPosition(seg, 0)); + return new FbsUnion().assign(seg, FbsVerifier.verifyRoot(seg, FbsUnion::verify)); + } + + /// Checks this table and everything reachable from it. + /// @param v the verifier walking the buffer + /// @param pos the table's absolute position + static void verify(FbsVerifier v, long pos) { + v.startTable(pos); + v.stringVector(pos, 4, "names", false); + v.tableVector(pos, 6, "dtypes", false, FbsDType::verify); + v.vector(pos, 8, 1, "type_ids", false); + v.inline(pos, 10, 1, "nullable", false); + v.endTable(); } /// Positions this reader at a table. diff --git a/core/src/main/java/io/github/dfa1/vortex/core/fbs/FbsUtf8.java b/core/src/main/java/io/github/dfa1/vortex/core/fbs/FbsUtf8.java index 2515335b8..ec4de9712 100644 --- a/core/src/main/java/io/github/dfa1/vortex/core/fbs/FbsUtf8.java +++ b/core/src/main/java/io/github/dfa1/vortex/core/fbs/FbsUtf8.java @@ -9,11 +9,21 @@ @Generated("io.github.dfa1.vortex.fbsgen.CodeGen") public final class FbsUtf8 extends FbsTable { - /// Positions a reader at the root table of a finished buffer. + /// Verifies a finished buffer and positions a reader at its root table. /// @param seg the buffer /// @return a reader positioned at the root + /// @throws io.github.dfa1.vortex.core.error.VortexException if the buffer is malformed public static FbsUtf8 getRootAsFbsUtf8(MemorySegment seg) { - return new FbsUtf8().assign(seg, FbsTable.rootPosition(seg, 0)); + return new FbsUtf8().assign(seg, FbsVerifier.verifyRoot(seg, FbsUtf8::verify)); + } + + /// Checks this table and everything reachable from it. + /// @param v the verifier walking the buffer + /// @param pos the table's absolute position + static void verify(FbsVerifier v, long pos) { + v.startTable(pos); + v.inline(pos, 4, 1, "nullable", false); + v.endTable(); } /// Positions this reader at a table. diff --git a/core/src/main/java/io/github/dfa1/vortex/core/fbs/FbsVariant.java b/core/src/main/java/io/github/dfa1/vortex/core/fbs/FbsVariant.java index df17fe087..2ffbdb033 100644 --- a/core/src/main/java/io/github/dfa1/vortex/core/fbs/FbsVariant.java +++ b/core/src/main/java/io/github/dfa1/vortex/core/fbs/FbsVariant.java @@ -9,11 +9,21 @@ @Generated("io.github.dfa1.vortex.fbsgen.CodeGen") public final class FbsVariant extends FbsTable { - /// Positions a reader at the root table of a finished buffer. + /// Verifies a finished buffer and positions a reader at its root table. /// @param seg the buffer /// @return a reader positioned at the root + /// @throws io.github.dfa1.vortex.core.error.VortexException if the buffer is malformed public static FbsVariant getRootAsFbsVariant(MemorySegment seg) { - return new FbsVariant().assign(seg, FbsTable.rootPosition(seg, 0)); + return new FbsVariant().assign(seg, FbsVerifier.verifyRoot(seg, FbsVariant::verify)); + } + + /// Checks this table and everything reachable from it. + /// @param v the verifier walking the buffer + /// @param pos the table's absolute position + static void verify(FbsVerifier v, long pos) { + v.startTable(pos); + v.inline(pos, 4, 1, "nullable", false); + v.endTable(); } /// Positions this reader at a table. diff --git a/core/src/main/java/io/github/dfa1/vortex/core/fbs/FbsVerifier.java b/core/src/main/java/io/github/dfa1/vortex/core/fbs/FbsVerifier.java new file mode 100644 index 000000000..b6608a85f --- /dev/null +++ b/core/src/main/java/io/github/dfa1/vortex/core/fbs/FbsVerifier.java @@ -0,0 +1,281 @@ +package io.github.dfa1.vortex.core.fbs; + +import static io.github.dfa1.vortex.core.io.VortexFormat.LE_INT; +import static io.github.dfa1.vortex.core.io.VortexFormat.LE_SHORT; + +import java.lang.foreign.MemorySegment; +import java.lang.foreign.ValueLayout; + +import io.github.dfa1.vortex.core.error.VortexException; + +/// Checks a whole FlatBuffer before any accessor reads it, as the Rust reference does with +/// `flatbuffers::root::()` on every buffer it parses. +/// +/// The [FbsTable] accessors follow offsets without bounds checks, so an unverified crafted buffer +/// fails deep inside the reader as a raw JDK exception, at whichever site first touches the bad +/// offset. Every generated `getRootAsX` runs this first instead: it walks every table reachable from +/// the root through its generated `verify(FbsVerifier, long)` and rejects the buffer with a +/// [VortexException] if any offset, vtable, vector or string falls outside it. Once a buffer passes, +/// the accessors cannot read out of bounds. +/// +/// Rules (the flatbuffers verifier's, with its default limits, which Vortex keeps): +/// - every table's soffset, vtable and object lie in the buffer, and the vtable is at least 4 bytes; +/// - every present field lies in the buffer, and every uoffset points into it; +/// - every vector's elements and every string's bytes plus NUL terminator lie in it; +/// - a `(required)` field is present; +/// - tables nest at most [#MAX_DEPTH] deep, and a buffer holds at most [#MAX_TABLES] tables. +/// +/// Not yet checked, though Rust does: alignment, UTF-8 validity of strings, and the +/// total-apparent-size cap. +final class FbsVerifier { + + /// flatbuffers' default `max_depth`, which Vortex keeps (`VORTEX_MAX_LAYOUT_DEPTH`). + static final int MAX_DEPTH = 64; + + /// flatbuffers' default `max_tables`, which Vortex keeps (`VORTEX_MAX_LAYOUT_TABLES`). + static final int MAX_TABLES = 1_000_000; + + /// Verifies one table: the generated `verify` of the table's class. + @FunctionalInterface + interface TableVerifier { + + /// @param verifier the verifier walking the buffer + /// @param position the table's absolute position + void verify(FbsVerifier verifier, long position); + } + + private final MemorySegment seg; + private final long size; + private int depth; + private int tables; + + private FbsVerifier(MemorySegment seg) { + this.seg = seg; + this.size = seg.byteSize(); + } + + /// Verifies a finished buffer whose root table `root` describes. + /// + /// @param seg the buffer + /// @param root the root table's verifier + /// @return the absolute position of the root table + /// @throws VortexException if the buffer is malformed + static long verifyRoot(MemorySegment seg, TableVerifier root) { + var verifier = new FbsVerifier(seg); + long position = verifier.uoffset(0); + root.verify(verifier, position); + return position; + } + + /// Opens a table: checks its soffset, vtable and object, and the depth and table limits. + /// Every `startTable` is paired with an [#endTable()]. + /// + /// @param position the table's absolute position + void startTable(long position) { + if (++depth > MAX_DEPTH) { + throw malformed("tables nest deeper than " + MAX_DEPTH); + } + if (++tables > MAX_TABLES) { + throw malformed("more than " + MAX_TABLES + " tables"); + } + check(position, Integer.BYTES, "table"); + long vtable = position - seg.get(LE_INT, position); + check(vtable, 2 * Short.BYTES, "vtable"); + int vtableSize = Short.toUnsignedInt(seg.get(LE_SHORT, vtable)); + int objectSize = Short.toUnsignedInt(seg.get(LE_SHORT, vtable + Short.BYTES)); + if (vtableSize < 2 * Short.BYTES || vtableSize % Short.BYTES != 0) { + throw malformed("vtable size " + vtableSize + " at " + vtable); + } + check(vtable, vtableSize, "vtable"); + check(position, objectSize, "table object"); + } + + /// Closes the table the last [#startTable(long)] opened. + void endTable() { + depth--; + } + + /// Checks an inline field (a scalar, an enum, or a struct) of `size` bytes. + /// + /// @param table the table's absolute position + /// @param slot the field's vtable slot offset (`4 + fieldIndex * 2`) + /// @param size the field's inline size in bytes + /// @param name the field name, for the error message + /// @param required whether the schema marks the field `(required)` + void inline(long table, int slot, int size, String name, boolean required) { + field(table, slot, size, name, required); + } + + /// Checks a string field. + /// + /// @param table the table's absolute position + /// @param slot the field's vtable slot offset + /// @param name the field name, for the error message + /// @param required whether the schema marks the field `(required)` + void string(long table, int slot, String name, boolean required) { + long at = field(table, slot, Integer.BYTES, name, required); + if (at >= 0) { + stringAt(uoffset(at)); + } + } + + /// Checks a vector field of inline elements (scalars, enums or structs). + /// + /// @param table the table's absolute position + /// @param slot the field's vtable slot offset + /// @param elementSize the inline size of one element in bytes + /// @param name the field name, for the error message + /// @param required whether the schema marks the field `(required)` + void vector(long table, int slot, int elementSize, String name, boolean required) { + long at = field(table, slot, Integer.BYTES, name, required); + if (at >= 0) { + vectorAt(uoffset(at), elementSize); + } + } + + /// Checks a vector-of-strings field and every string in it. + /// + /// @param table the table's absolute position + /// @param slot the field's vtable slot offset + /// @param name the field name, for the error message + /// @param required whether the schema marks the field `(required)` + void stringVector(long table, int slot, String name, boolean required) { + long at = field(table, slot, Integer.BYTES, name, required); + if (at >= 0) { + long elements = uoffset(at); + long count = vectorAt(elements, Integer.BYTES); + for (long i = 0; i < count; i++) { + stringAt(uoffset(elements + Integer.BYTES + i * Integer.BYTES)); + } + } + } + + /// Checks a vector-of-tables field and every table in it. + /// + /// @param table the table's absolute position + /// @param slot the field's vtable slot offset + /// @param name the field name, for the error message + /// @param required whether the schema marks the field `(required)` + /// @param element the element table's verifier + void tableVector(long table, int slot, String name, boolean required, TableVerifier element) { + long at = field(table, slot, Integer.BYTES, name, required); + if (at >= 0) { + long elements = uoffset(at); + long count = vectorAt(elements, Integer.BYTES); + for (long i = 0; i < count; i++) { + element.verify(this, uoffset(elements + Integer.BYTES + i * Integer.BYTES)); + } + } + } + + /// Checks a table field and the table it points to. + /// + /// @param table the table's absolute position + /// @param slot the field's vtable slot offset + /// @param name the field name, for the error message + /// @param required whether the schema marks the field `(required)` + /// @param child the child table's verifier + void table(long table, int slot, String name, boolean required, TableVerifier child) { + long at = field(table, slot, Integer.BYTES, name, required); + if (at >= 0) { + child.verify(this, uoffset(at)); + } + } + + /// Checks a union's discriminator field and returns it, so the generated code can verify the + /// member table its value field points to with [#unionValue(long, int, String)]. A member type + /// without a value, or a value without a member type, is malformed, as in Rust's `visit_union` + /// (`InconsistentUnion`): the accessors would return null for the member the type names. + /// + /// @param table the table's absolute position + /// @param slot the discriminator's vtable slot offset + /// @param valueSlot the value's vtable slot offset + /// @param name the union field name, for the error message + /// @return the discriminator (0 = NONE) + int unionType(long table, int slot, int valueSlot, String name) { + long at = field(table, slot, Byte.BYTES, name + "_type", false); + int type = at >= 0 ? Byte.toUnsignedInt(seg.get(ValueLayout.JAVA_BYTE, at)) : 0; + if ((type != 0) != (field(table, valueSlot, Integer.BYTES, name, false) >= 0)) { + throw malformed("union '" + name + "' type " + type + " inconsistent with its value in table at " + table); + } + return type; + } + + /// Resolves a union's value field to the member table's position. + /// + /// @param table the table's absolute position + /// @param slot the value's vtable slot offset + /// @param name the union field name, for the error message + /// @return the member table's absolute position, or -1 when the value is absent + long unionValue(long table, int slot, String name) { + long at = field(table, slot, Integer.BYTES, name, false); + return at >= 0 ? uoffset(at) : -1; + } + + /// Returns a field's absolute position after checking it lies in the buffer, or -1 if absent. + private long field(long table, int slot, int size, String name, boolean required) { + long vtable = table - seg.get(LE_INT, table); + int vtableSize = Short.toUnsignedInt(seg.get(LE_SHORT, vtable)); + int offset = slot < vtableSize ? Short.toUnsignedInt(seg.get(LE_SHORT, vtable + slot)) : 0; + if (offset == 0) { + if (required) { + throw malformed("required field '" + name + "' missing in table at " + table); + } + return -1; + } + // The message is built only on failure: this runs for every field of every table verified. + if (!inBuffer(table + offset, size)) { + throw outside("field '" + name + "'", table + offset, size); + } + return table + offset; + } + + /// Follows the uoffset at `at` and returns the position it points to, inside the buffer. + private long uoffset(long at) { + check(at, Integer.BYTES, "offset"); + long target = at + Integer.toUnsignedLong(seg.get(LE_INT, at)); + check(target, 1, "offset target"); + return target; + } + + /// Checks a vector's length prefix and elements; returns its element count. + private long vectorAt(long at, int elementSize) { + check(at, Integer.BYTES, "vector length"); + long count = Integer.toUnsignedLong(seg.get(LE_INT, at)); + if (!inBuffer(at + Integer.BYTES, count * elementSize)) { + throw outside("vector of " + count + " elements", at + Integer.BYTES, count * elementSize); + } + return count; + } + + /// Checks a string's length prefix, bytes and NUL terminator. + private void stringAt(long at) { + long length = vectorAt(at, Byte.BYTES); + long terminator = at + Integer.BYTES + length; + check(terminator, 1, "string terminator"); + if (seg.get(ValueLayout.JAVA_BYTE, terminator) != 0) { + throw malformed("string at " + at + " is not NUL-terminated"); + } + } + + /// Checks that `[at, at + length)` lies in the buffer. + private void check(long at, long length, String what) { + if (!inBuffer(at, length)) { + throw outside(what, at, length); + } + } + + /// Whether `[at, at + length)` lies in the buffer (the arithmetic cannot overflow: both are below + /// 2^35 here, from u32 counts times element sizes of at most 8). + private boolean inBuffer(long at, long length) { + return at >= 0 && length >= 0 && at + length <= size; + } + + private VortexException outside(String what, long at, long length) { + return malformed(what + " [" + at + ", " + (at + length) + ") outside the " + size + "-byte buffer"); + } + + private static VortexException malformed(String reason) { + return new VortexException("malformed flatbuffer: " + reason); + } +} diff --git a/core/src/test/java/io/github/dfa1/vortex/core/fbs/FbsVerifierTest.java b/core/src/test/java/io/github/dfa1/vortex/core/fbs/FbsVerifierTest.java new file mode 100644 index 000000000..24724a7f4 --- /dev/null +++ b/core/src/test/java/io/github/dfa1/vortex/core/fbs/FbsVerifierTest.java @@ -0,0 +1,279 @@ +package io.github.dfa1.vortex.core.fbs; + +import static io.github.dfa1.vortex.core.io.VortexFormat.LE_INT; +import static io.github.dfa1.vortex.core.io.VortexFormat.LE_SHORT; +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatCode; +import static org.assertj.core.api.Assertions.assertThatThrownBy; + +import java.lang.foreign.MemorySegment; +import java.lang.foreign.ValueLayout; +import java.util.Arrays; + +import io.github.dfa1.vortex.core.error.VortexException; +import org.junit.jupiter.api.Nested; +import org.junit.jupiter.api.Test; + +/// Each rule of [FbsVerifier], driven through the generated `getRootAsX` entry points: a valid buffer +/// from [FbsBuilder] with one field corrupted in place, so every case fails on exactly one rule. +class FbsVerifierTest { + + @Test + void builderOutput_passes() { + // Given — a layout using every field kind it has: scalars, a byte vector, a u32 vector and a + // vector of child tables + MemorySegment seg = layout(new long[]{1, 2, 3}, 2); + + // When + FbsLayout result = FbsLayout.getRootAsFbsLayout(seg); + + // Then + assertThat(result.childrenLength()).isEqualTo(2); + assertThat(result.segments(2)).isEqualTo(3); + } + + @Nested + class Tables { + + @Test + void bufferTooShortForRootOffset_rejected() { + // Given + MemorySegment seg = MemorySegment.ofArray(new byte[3]); + + // When / Then + assertMalformed(seg, "offset"); + } + + @Test + void rootOffsetPastBuffer_rejected() { + // Given + MemorySegment seg = layout(new long[]{1}, 0); + seg.set(LE_INT, 0, (int) seg.byteSize()); + + // When / Then + assertMalformed(seg, "offset target"); + } + + @Test + void vtableOutsideBuffer_rejected() { + // Given — the root's soffset points a vtable far before the buffer start + MemorySegment seg = layout(new long[]{1}, 0); + seg.set(LE_INT, rootPosition(seg), 1_000_000); + + // When / Then + assertMalformed(seg, "vtable"); + } + + @Test + void vtableSmallerThanItsHeader_rejected() { + // Given — a vtable claiming 2 bytes, too short for its own size + object-size header + MemorySegment seg = layout(new long[]{1}, 0); + seg.set(LE_SHORT, vtable(seg), (short) 2); + + // When / Then + assertMalformed(seg, "vtable size 2"); + } + + @Test + void objectPastBuffer_rejected() { + // Given — the vtable's object size runs the table past the buffer end + MemorySegment seg = layout(new long[]{1}, 0); + seg.set(LE_SHORT, vtable(seg) + 2, (short) 0xFFFF); + + // When / Then + assertMalformed(seg, "table object"); + } + + @Test + void fieldPastBuffer_rejected() { + // Given — the row_count slot (vtable offset 6) points 8 bytes from the buffer end + // minus 4, so the u64 field straddles it + MemorySegment seg = layout(new long[]{1}, 0); + long table = rootPosition(seg); + seg.set(LE_SHORT, vtable(seg) + 6, (short) (seg.byteSize() - 4 - table)); + seg.set(LE_SHORT, vtable(seg) + 2, (short) (seg.byteSize() - table)); + + // When / Then + assertMalformed(seg, "field 'row_count'"); + } + } + + @Nested + class VectorsAndStrings { + + @Test + void vectorLengthPastBuffer_rejected() { + // Given — the segments vector's length prefix claims 100 000 u32s + MemorySegment seg = layout(new long[]{1, 2}, 0); + seg.set(LE_INT, vectorStart(seg, FbsLayout.getRootAsFbsLayout(seg), 12), 100_000); + + // When / Then + assertMalformed(seg, "vector of 100000 elements"); + } + + @Test + void stringWithoutNulTerminator_rejected() { + // Given — an ArraySpec id whose terminating NUL is overwritten + FbsBuilder b = new FbsBuilder(64); + b.finish(FbsArraySpec.createFbsArraySpec(b, b.createString("vortex.primitive"))); + MemorySegment seg = MemorySegment.ofArray(b.sizedByteArray()); + long string = vectorStart(seg, FbsArraySpec.getRootAsFbsArraySpec(seg), 4); + seg.set(ValueLayout.JAVA_BYTE, string + 4 + "vortex.primitive".length(), (byte) 'x'); + + // When / Then + assertThatThrownBy(() -> FbsArraySpec.getRootAsFbsArraySpec(seg)) + .isInstanceOf(VortexException.class) + .hasMessageContaining("not NUL-terminated"); + } + + @Test + void requiredFieldMissing_rejected() { + // Given — an ArraySpec without its `id: string (required)` + FbsBuilder b = new FbsBuilder(64); + b.finish(FbsArraySpec.createFbsArraySpec(b, 0)); + MemorySegment seg = MemorySegment.ofArray(b.sizedByteArray()); + + // When / Then + assertThatThrownBy(() -> FbsArraySpec.getRootAsFbsArraySpec(seg)) + .isInstanceOf(VortexException.class) + .hasMessageContaining("required field 'id' missing"); + } + } + + @Nested + class Unions { + + @Test + void selectedMemberIsVerified() { + // Given — a Struct_ dtype whose names vector claims far more strings than it holds: only + // the union member's own verify can see it + FbsBuilder b = new FbsBuilder(128); + int names = FbsStruct.createNamesVector(b, new int[]{b.createString("a")}); + int dtypes = FbsStruct.createDtypesVector(b, new int[0]); + int struct = FbsStruct.createFbsStruct(b, names, dtypes, false); + b.finish(FbsDType.createFbsDType(b, FbsType.FbsStruct, struct)); + MemorySegment seg = MemorySegment.ofArray(b.sizedByteArray()); + FbsStruct member = FbsDType.getRootAsFbsDType(seg).type(new FbsStruct()); + seg.set(LE_INT, vectorStart(seg, member, 4), 100_000); + + // When / Then + assertThatThrownBy(() -> FbsDType.getRootAsFbsDType(seg)) + .isInstanceOf(VortexException.class) + .hasMessageContaining("vector of 100000 elements"); + } + + @org.junit.jupiter.params.ParameterizedTest(name = "type {0}, value present {1}") + @org.junit.jupiter.params.provider.CsvSource({"7, false", "0, true"}) + void typeAndValueDisagree_rejected(byte type, boolean withValue) { + // Given — a member type with no value, or a value with no member type: Rust's + // visit_union rejects both (InconsistentUnion); the accessors would return a null member + FbsBuilder b = new FbsBuilder(64); + int value = withValue ? FbsNull.createFbsNull(b) : 0; + b.finish(FbsDType.createFbsDType(b, type, value)); + MemorySegment seg = MemorySegment.ofArray(b.sizedByteArray()); + + // When / Then + assertThatThrownBy(() -> FbsDType.getRootAsFbsDType(seg)) + .isInstanceOf(VortexException.class) + .hasMessageContaining("union 'type' type " + type + " inconsistent with its value"); + } + + @Test + void unknownMemberIsAccepted() { + // Given — a discriminator this schema does not know: like flatc's verifiers, accepted, + // since no accessor reads its value (the reader rejects the dtype itself) + FbsBuilder b = new FbsBuilder(64); + int nul = FbsNull.createFbsNull(b); + b.finish(FbsDType.createFbsDType(b, (byte) 99, nul)); + MemorySegment seg = MemorySegment.ofArray(b.sizedByteArray()); + + // When / Then + assertThatCode(() -> FbsDType.getRootAsFbsDType(seg)).doesNotThrowAnyException(); + } + } + + @Nested + class Limits { + + @Test + void nestingAtDepthLimit_passes() { + // Given + MemorySegment seg = nestedLayouts(FbsVerifier.MAX_DEPTH); + + // When / Then + assertThatCode(() -> FbsLayout.getRootAsFbsLayout(seg)).doesNotThrowAnyException(); + } + + @Test + void nestingPastDepthLimit_rejected() { + // Given + MemorySegment seg = nestedLayouts(FbsVerifier.MAX_DEPTH + 1); + + // When / Then + assertMalformed(seg, "nest deeper than 64"); + } + + @Test + void tableCountPastLimit_rejected() { + // Given — one child table referenced MAX_TABLES times: a 4 MB buffer that a walk without + // the table cap would visit a million times over (each visit counts, as in Rust) + FbsBuilder b = new FbsBuilder(8 * 1024 * 1024); + int leaf = FbsLayout.createFbsLayout(b, 0, 1L, 0, 0, 0); + int[] children = new int[FbsVerifier.MAX_TABLES]; + Arrays.fill(children, leaf); + int childrenVector = FbsLayout.createChildrenVector(b, children); + b.finish(FbsLayout.createFbsLayout(b, 0, 1L, 0, childrenVector, 0)); + MemorySegment seg = b.dataSegment(); + + // When / Then + assertMalformed(seg, "more than 1000000 tables"); + } + } + + private static void assertMalformed(MemorySegment seg, String reason) { + assertThatThrownBy(() -> FbsLayout.getRootAsFbsLayout(seg)) + .isInstanceOf(VortexException.class) + .hasMessageContaining("malformed flatbuffer: ") + .hasMessageContaining(reason); + } + + /// A layout with the given segment ids, a 3-byte metadata vector and `children` leaf children. + private static MemorySegment layout(long[] segments, int children) { + FbsBuilder b = new FbsBuilder(256); + int[] leaves = new int[children]; + for (int i = 0; i < children; i++) { + leaves[i] = FbsLayout.createFbsLayout(b, 0, 1L, 0, 0, 0); + } + int childrenVector = FbsLayout.createChildrenVector(b, leaves); + int metadata = FbsLayout.createMetadataVector(b, new byte[]{1, 2, 3}); + int segmentsVector = FbsLayout.createSegmentsVector(b, segments); + b.finish(FbsLayout.createFbsLayout(b, 0, 7L, metadata, childrenVector, segmentsVector)); + return MemorySegment.ofArray(b.sizedByteArray()); + } + + /// A chain of `tables` layouts, each the single child of the next. + private static MemorySegment nestedLayouts(int tables) { + FbsBuilder b = new FbsBuilder(tables * 32 + 64); + int current = FbsLayout.createFbsLayout(b, 0, 1L, 0, 0, 0); + for (int i = 1; i < tables; i++) { + current = FbsLayout.createFbsLayout(b, 0, 1L, 0, FbsLayout.createChildrenVector(b, new int[]{current}), 0); + } + b.finish(current); + return MemorySegment.ofArray(b.sizedByteArray()); + } + + private static long rootPosition(MemorySegment seg) { + return seg.get(LE_INT, 0); + } + + private static long vtable(MemorySegment seg) { + long table = rootPosition(seg); + return table - seg.get(LE_INT, table); + } + + /// Position of the length prefix of the vector (or string) in `table`'s `slot`. + private static long vectorStart(MemorySegment seg, FbsTable table, int slot) { + long at = table.pos + table.fieldOffset(slot); + return at + seg.get(LE_INT, at); + } +} diff --git a/fbs-gen/src/main/java/io/github/dfa1/vortex/fbsgen/CodeGen.java b/fbs-gen/src/main/java/io/github/dfa1/vortex/fbsgen/CodeGen.java index 8e2e53f26..2dddfee76 100644 --- a/fbs-gen/src/main/java/io/github/dfa1/vortex/fbsgen/CodeGen.java +++ b/fbs-gen/src/main/java/io/github/dfa1/vortex/fbsgen/CodeGen.java @@ -83,14 +83,16 @@ private String emitTable(Ast.TableDecl table) { sb.append("public final class ").append(name).append(" extends FbsTable {\n\n"); sb.append(""" - /// Positions a reader at the root table of a finished buffer. + /// Verifies a finished buffer and positions a reader at its root table. /// @param seg the buffer /// @return a reader positioned at the root + /// @throws io.github.dfa1.vortex.core.error.VortexException if the buffer is malformed public static %1$s getRootAs%1$s(MemorySegment seg) { - return new %1$s().assign(seg, FbsTable.rootPosition(seg, 0)); + return new %1$s().assign(seg, FbsVerifier.verifyRoot(seg, %1$s::verify)); } """.formatted(name)); + emitVerify(sb, table); sb.append(""" /// Positions this reader at a table. @@ -112,6 +114,96 @@ private String emitTable(Ast.TableDecl table) { return sb.toString(); } + /// Emits the table's `verify(FbsVerifier, long)`: one verifier call per field, recursing into + /// child tables, mirroring flatc's generated `Verifiable` impls. + private void emitVerify(StringBuilder sb, Ast.TableDecl table) { + sb.append(""" + /// Checks this table and everything reachable from it. + /// @param v the verifier walking the buffer + /// @param pos the table's absolute position + static void verify(FbsVerifier v, long pos) { + v.startTable(pos); + """); + int vtableIndex = 0; + for (Ast.Field field : table.fields()) { + int vt = 4 + vtableIndex * 2; + vtableIndex += emitVerifyField(sb, field, vt); + } + sb.append(""" + v.endTable(); + } + + """); + } + + /// Emits the verifier call for one field; returns the vtable slots it uses (two for a union). + private int emitVerifyField(StringBuilder sb, Ast.Field field, int vt) { + String name = field.name(); + boolean required = field.required(); + String call = switch (field.type()) { + case Ast.Scalar scalar when scalar == Ast.Scalar.STRING -> "v.string(pos, %d, \"%s\", %b);" + .formatted(vt, name, required); + case Ast.Scalar scalar -> inlineCall(vt, TypeRegistry.sizeOf(scalar), name, required); + case Ast.Vector vector -> vectorCall(vector.element(), vt, name, required); + case Ast.Ref ref -> switch (registry.resolve(ref)) { + case Ast.EnumDecl e -> inlineCall(vt, TypeRegistry.sizeOf(e.underlying()), name, required); + case Ast.StructDecl s -> inlineCall(vt, registry.layoutOf(s).size(), name, required); + case Ast.TableDecl t -> "v.table(pos, %d, \"%s\", %b, %s::verify);" + .formatted(vt, name, required, className(t.name())); + case Ast.UnionDecl u -> unionCall(u, vt, name); + }; + }; + sb.append(" ").append(call).append('\n'); + return field.type() instanceof Ast.Ref ref && registry.resolve(ref) instanceof Ast.UnionDecl ? 2 : 1; + } + + private static String inlineCall(int vt, int size, String name, boolean required) { + return "v.inline(pos, %d, %d, \"%s\", %b);".formatted(vt, size, name, required); + } + + private String vectorCall(Ast.FieldType element, int vt, String name, boolean required) { + return switch (element) { + case Ast.Scalar scalar when scalar == Ast.Scalar.STRING -> "v.stringVector(pos, %d, \"%s\", %b);" + .formatted(vt, name, required); + case Ast.Scalar scalar -> "v.vector(pos, %d, %d, \"%s\", %b);" + .formatted(vt, TypeRegistry.sizeOf(scalar), name, required); + case Ast.Vector _ -> throw new FbsParseException("nested vectors are not supported ('" + name + "')"); + case Ast.Ref ref -> switch (registry.resolve(ref)) { + case Ast.EnumDecl e -> "v.vector(pos, %d, %d, \"%s\", %b);" + .formatted(vt, TypeRegistry.sizeOf(e.underlying()), name, required); + case Ast.StructDecl s -> "v.vector(pos, %d, %d, \"%s\", %b);" + .formatted(vt, registry.layoutOf(s).size(), name, required); + case Ast.TableDecl t -> "v.tableVector(pos, %d, \"%s\", %b, %s::verify);" + .formatted(vt, name, required, className(t.name())); + case Ast.UnionDecl _ -> throw new FbsParseException("vectors of unions are not supported ('" + name + "')"); + }; + }; + } + + /// A union is a discriminator slot plus a value slot, consistent or rejected. Only a member this + /// schema knows is verified: like flatc's verifiers, an unknown discriminator is accepted, since + /// no accessor reads its value. + private static String unionCall(Ast.UnionDecl union, int vt, String name) { + StringBuilder sb = new StringBuilder(); + String type = camelCase(name) + "Type"; + sb.append("int ").append(type).append(" = v.unionType(pos, ").append(vt).append(", ").append(vt + 2) + .append(", \"").append(name).append("\");\n"); + sb.append(" if (").append(type).append(" != 0) {\n"); + sb.append(" long member = v.unionValue(pos, ").append(vt + 2).append(", \"").append(name).append("\");\n"); + sb.append(" switch (").append(type).append(") {\n"); + long next = 1; + for (Ast.UnionMember m : union.members()) { + long value = m.number().isPresent() ? m.number().getAsInt() : next; + next = value + 1; + sb.append(" case ").append(value).append(" -> ") + .append(className(simpleName(m.typeName()))).append(".verify(v, member);\n"); + } + sb.append(" default -> {\n }\n"); + sb.append(" }\n"); + sb.append(" }"); + return sb.toString(); + } + /// Emits accessors for one table field; returns the next vtable index (a union /// consumes two slots: discriminator + value). private int emitTableField(StringBuilder sb, Ast.Field field, int vtableIndex) { diff --git a/fbs-gen/src/test/java/io/github/dfa1/vortex/fbsgen/CodeGenTest.java b/fbs-gen/src/test/java/io/github/dfa1/vortex/fbsgen/CodeGenTest.java index c465d95b1..199f554dc 100644 --- a/fbs-gen/src/test/java/io/github/dfa1/vortex/fbsgen/CodeGenTest.java +++ b/fbs-gen/src/test/java/io/github/dfa1/vortex/fbsgen/CodeGenTest.java @@ -65,6 +65,26 @@ void emitsScalarVectorAndChildAccessorsForTable() throws IOException { .contains("readInt(vectorElements(o) + (long) j * 4) & 0xFFFFFFFFL"); } + @Test + void emitsVerifierRunBeforeEveryRootAccess() throws IOException { + // Given — DType is a union holder, ArraySpec has a required string, Footer has table and + // struct vectors: between them every verifier call shape + String dtype = generateOne("FbsDType.java"); + String arraySpec = generateOne("FbsArraySpec.java"); + String footer = generateOne("FbsFooter.java"); + + // Then — getRootAs verifies first, and verify walks each field with its wire shape + assertThat(dtype) + .contains("return new FbsDType().assign(seg, FbsVerifier.verifyRoot(seg, FbsDType::verify));") + .contains("int typeType = v.unionType(pos, 4, 6, \"type\");") + .contains("long member = v.unionValue(pos, 6, \"type\");") + .contains("case 7 -> FbsStruct.verify(v, member);"); + assertThat(arraySpec).contains("v.string(pos, 4, \"id\", true);"); + assertThat(footer) + .contains("v.tableVector(pos, 4, \"array_specs\", false, FbsArraySpec::verify);") + .contains("v.vector(pos, 8, 16, \"segment_specs\", false);"); + } + @Test void emitsInlineOffsetsForStruct() throws IOException { // Given — Buffer struct: padding u16 @0, alignment u8 @2, compression(enum u8) @3, diff --git a/inspector/src/main/java/io/github/dfa1/vortex/inspect/InspectorTree.java b/inspector/src/main/java/io/github/dfa1/vortex/inspect/InspectorTree.java index 77f52733a..df0074c4d 100644 --- a/inspector/src/main/java/io/github/dfa1/vortex/inspect/InspectorTree.java +++ b/inspector/src/main/java/io/github/dfa1/vortex/inspect/InspectorTree.java @@ -406,24 +406,16 @@ private static Peek peekFlatRoot(MemorySegment seg, List arraySpecs) return new Peek(null, ArrayStats.empty(), Set.of()); } Set nested = new LinkedHashSet<>(); - collectEncodings(root, arraySpecs, nested, 0, new int[]{0}); + collectEncodings(root, arraySpecs, nested, new int[]{0}); return new Peek(resolveEncodingId(arraySpecs, root.encoding()), ArrayStats.fromFbs(root.stats()), Set.copyOf(nested)); } - /// Hard cap on the `ArrayNode`-tree walk depth in [#collectEncodings(FbsArrayNode, List, Set, int, int[])], - /// mirroring `SerializedArrayDecoder.MAX_ARRAY_TREE_DEPTH` in the reader module: a crafted segment with - /// deeply nested children could otherwise drive unbounded recursion into a `StackOverflowError`, - /// an `Error` that would bypass the [VortexException] contract for malformed untrusted input. - static final int MAX_ENCODING_TREE_DEPTH = 64; - /// Hard cap on the total number of `ArrayNode`s visited by one - /// [#collectEncodings(FbsArrayNode, List, Set, int, int[])] walk. The depth cap alone is not enough: nothing - /// stops two sibling `children` vector slots from resolving to the exact same absolute child position (the - /// underlying FlatBuffers `uoffset` mechanism is not validated for aliasing), so a crafted segment with a - /// handful of levels, each fanning out to two children that both alias the next level's single node, drives - /// the walk to an exponential (up to 2^[#MAX_ENCODING_TREE_DEPTH]) number of visits while depth itself - /// never exceeds the limit. + /// [#collectEncodings(FbsArrayNode, List, Set, int[])] walk. Depth is already bounded by the FlatBuffer + /// verifier `FbsArray.getRootAsFbsArray` runs (64 nested tables, as Rust), and so is the total, but only at + /// a million visits: two sibling `children` slots may alias the same child, so a handful of levels fanning + /// out to two aliases each reaches that many. The inspector peeks every segment, so it stops far sooner. static final int MAX_ENCODING_TREE_NODES = 4096; /// Resolves a wire-supplied encoding index against the footer's array spec table. @@ -447,20 +439,16 @@ private static String resolveEncodingId(List arraySpecs, int encIdx) /// @param node array node to visit /// @param arraySpecs footer's array spec table, indexed by [FbsArrayNode#encoding()] /// @param into accumulator for resolved encoding ids - /// @param depth current recursion depth, starting at `0` for the root /// @param visited single-element counter of nodes visited so far across the whole walk private static void collectEncodings(FbsArrayNode node, List arraySpecs, Set into, - int depth, int[] visited) { - if (depth > MAX_ENCODING_TREE_DEPTH) { - throw new VortexException("array tree depth exceeds limit (" + MAX_ENCODING_TREE_DEPTH + ")"); - } + int[] visited) { if (++visited[0] > MAX_ENCODING_TREE_NODES) { throw new VortexException("array tree node count exceeds limit (" + MAX_ENCODING_TREE_NODES + ")"); } into.add(resolveEncodingId(arraySpecs, node.encoding())); int childCount = node.childrenLength(); for (int i = 0; i < childCount; i++) { - collectEncodings(node.children(i), arraySpecs, into, depth + 1, visited); + collectEncodings(node.children(i), arraySpecs, into, visited); } } diff --git a/inspector/src/test/java/io/github/dfa1/vortex/inspect/InspectorTreeTest.java b/inspector/src/test/java/io/github/dfa1/vortex/inspect/InspectorTreeTest.java index 0399ad11a..2eeaca970 100644 --- a/inspector/src/test/java/io/github/dfa1/vortex/inspect/InspectorTreeTest.java +++ b/inspector/src/test/java/io/github/dfa1/vortex/inspect/InspectorTreeTest.java @@ -299,14 +299,14 @@ void build_flatSegmentWithNestedEncoding_reportsRootAndNestedEncodings() { @Test void build_flatSegmentEncodingTreeAtDepthLimit_clearsGuard() { - // Given — deepest node at exactly MAX_ENCODING_TREE_DEPTH clears the guard - // (`depth > limit` is false there); brackets the boundary with the next test. + // Given — 64 nested tables (the Array table and 63 nodes), exactly the FlatBuffer verifier's + // depth limit, Rust's own; brackets the boundary with the next test. try (Arena arena = Arena.ofConfined()) { Layout root = new Layout(LayoutId.parse("vortex.flat"), 10, null, List.of(), List.of(0)); DType dtype = DType.I32; SegmentSpec spec = new SegmentSpec(0, 100, (byte) 0, CompressionScheme.NONE); givenHandle(dtype, root, List.of("vortex.flat"), List.of(spec)); - byte[] fb = deeplyNestedArrayFlatBuffer(InspectorTree.MAX_ENCODING_TREE_DEPTH); + byte[] fb = deeplyNestedArrayFlatBuffer(62); given(handle.rawSegment(spec)).willReturn(wrapAsSegment(fb, arena)); // When @@ -319,7 +319,7 @@ void build_flatSegmentEncodingTreeAtDepthLimit_clearsGuard() { @Test void build_flatSegmentEncodingTreeOneOverDepthLimit_throwsVortexException() { - // Given — one level deeper than MAX_ENCODING_TREE_DEPTH: a crafted file with a very + // Given — one table deeper than the verifier allows: a crafted file with a very // deep ArrayNode nesting must surface as VortexException, never StackOverflowError // (the untrusted-input contract every reader/inspector parse path must uphold). try (Arena arena = Arena.ofConfined()) { @@ -327,13 +327,13 @@ void build_flatSegmentEncodingTreeOneOverDepthLimit_throwsVortexException() { DType dtype = DType.I32; SegmentSpec spec = new SegmentSpec(0, 100, (byte) 0, CompressionScheme.NONE); givenHandle(dtype, root, List.of("vortex.flat"), List.of(spec)); - byte[] fb = deeplyNestedArrayFlatBuffer(InspectorTree.MAX_ENCODING_TREE_DEPTH + 1); + byte[] fb = deeplyNestedArrayFlatBuffer(63); given(handle.rawSegment(spec)).willReturn(wrapAsSegment(fb, arena)); // When / Then assertThatThrownBy(() -> InspectorTree.build(handle)) .isInstanceOf(VortexException.class) - .hasMessageContaining("depth"); + .hasMessageContaining("nest deeper than 64"); } } @@ -364,7 +364,7 @@ void build_flatSegmentEncodingTreeOverNodeCountLimit_throwsVortexException() { // children, so depth never exceeds 1 but total node count does. Without a separate // node-count cap, two aliased children per level (both children vector slots resolving // to the same table position) could drive collectEncodings to an exponential number of - // visits while depth stays within MAX_ENCODING_TREE_DEPTH; this is the simpler, + // visits while depth stays within the verifier's limit; this is the simpler, // non-aliased way to exercise the same cap. try (Arena arena = Arena.ofConfined()) { Layout root = new Layout(LayoutId.parse("vortex.flat"), 10, null, List.of(), List.of(0)); diff --git a/reader/src/main/java/io/github/dfa1/vortex/reader/PostscriptParser.java b/reader/src/main/java/io/github/dfa1/vortex/reader/PostscriptParser.java index 0d2160bc3..d959a05f7 100644 --- a/reader/src/main/java/io/github/dfa1/vortex/reader/PostscriptParser.java +++ b/reader/src/main/java/io/github/dfa1/vortex/reader/PostscriptParser.java @@ -29,26 +29,12 @@ final class PostscriptParser { - /// Hard cap on layout-tree recursion depth. Real-world layouts are typically four levels - /// (Struct → Zoned → Chunked → Flat); 64 is well past any expected schema and prevents - /// adversarial inputs — deeply nested trees or self-referential FlatBuffer cycles — from - /// blowing the JVM stack during [#convertLayout(io.github.dfa1.vortex.core.fbs.FbsLayout, List, int)]. - static final int MAX_LAYOUT_DEPTH = 64; - /// Hard cap on per-layout metadata size. The FlatBuffer runtime returns an unbounded slice /// from `metadataAsSegment()`; a crafted file can claim a multi-gigabyte metadata /// blob and force later allocators into pathological behavior. 4 MiB is well above any /// real encoding's metadata footprint (the largest is FSST's symbol table at ~32 KiB). static final MemorySize MAX_LAYOUT_METADATA_BYTES = MemorySize.ofMiB(4); - /// Hard cap on DType-tree recursion depth. A `DType` nests through Struct fields, List/ - /// FixedSizeList element types, and Extension storage types; like the layout tree, a crafted - /// or self-referential FlatBuffer can drive [#convertDType(io.github.dfa1.vortex.core.fbs.FbsDType, int)] - /// into unbounded recursion and a [StackOverflowError] — which, being an `Error`, would escape - /// the [VortexException] sanitization and leak the reader's memory-mapped Arena. 64 is well past - /// any real schema's nesting. - static final int MAX_DTYPE_DEPTH = 64; - private PostscriptParser() { } @@ -121,11 +107,11 @@ static ParsedFile parseBlobs(MemorySegment footerBuf, MemorySegment layoutBuf, M var fbsLayout = io.github.dfa1.vortex.core.fbs.FbsLayout.getRootAsFbsLayout(layoutBuf); Footer footer = convertFooter(fbsFooter); - Layout layout = convertLayout(fbsLayout, footer.layoutSpecs(), 0); + Layout layout = convertLayout(fbsLayout, footer.layoutSpecs()); DType dtype = null; if (dtypeBuf != null && dtypeBuf.byteSize() > 0) { - dtype = convertDType(io.github.dfa1.vortex.core.fbs.FbsDType.getRootAsFbsDType(dtypeBuf), 0); + dtype = convertDType(io.github.dfa1.vortex.core.fbs.FbsDType.getRootAsFbsDType(dtypeBuf)); checkStructLayouts(layout, dtype); } @@ -236,11 +222,7 @@ static Footer convertFooter(io.github.dfa1.vortex.core.fbs.FbsFooter f) { List.copyOf(segmentSpecs), List.copyOf(compressionSpecs)); } - private static Layout convertLayout(io.github.dfa1.vortex.core.fbs.FbsLayout l, List layoutSpecs, int depth) { - if (depth > MAX_LAYOUT_DEPTH) { - throw new VortexException( - "layout tree depth exceeds limit (" + MAX_LAYOUT_DEPTH + ")"); - } + private static Layout convertLayout(io.github.dfa1.vortex.core.fbs.FbsLayout l, List layoutSpecs) { int encIdx = l.encoding(); if (encIdx < 0 || encIdx >= layoutSpecs.size()) { throw new VortexException( @@ -259,7 +241,7 @@ private static Layout convertLayout(io.github.dfa1.vortex.core.fbs.FbsLayout l, var children = new ArrayList(l.childrenLength()); for (int i = 0; i < l.childrenLength(); i++) { - children.add(convertLayout(l.children(i), layoutSpecs, depth + 1)); + children.add(convertLayout(l.children(i), layoutSpecs)); } var segments = new ArrayList(l.segmentsLength()); @@ -270,11 +252,7 @@ private static Layout convertLayout(io.github.dfa1.vortex.core.fbs.FbsLayout l, return new Layout(layoutId, l.rowCount(), metadata, List.copyOf(children), List.copyOf(segments)); } - private static DType convertDType(io.github.dfa1.vortex.core.fbs.FbsDType fbs, int depth) { - if (depth > MAX_DTYPE_DEPTH) { - throw new VortexException( - "DType tree depth exceeds limit (" + MAX_DTYPE_DEPTH + ")"); - } + private static DType convertDType(io.github.dfa1.vortex.core.fbs.FbsDType fbs) { int typeType = fbs.typeType(); return switch (typeType) { case FbsType.FbsNull -> new DType.Null(true); @@ -339,17 +317,17 @@ private static DType convertDType(io.github.dfa1.vortex.core.fbs.FbsDType fbs, i names.add(new io.github.dfa1.vortex.core.model.ColumnName(name)); } for (int i = 0; i < s.dtypesLength(); i++) { - types.add(convertDType(s.dtypes(i), depth + 1)); + types.add(convertDType(s.dtypes(i))); } yield new DType.Struct(List.copyOf(names), List.copyOf(types), s.nullable()); } case FbsType.FbsList -> { var l = fbs.type(new io.github.dfa1.vortex.core.fbs.FbsList()); - yield new DType.List(convertDType(l.elementType(), depth + 1), l.nullable()); + yield new DType.List(convertDType(l.elementType()), l.nullable()); } case FbsType.FbsFixedSizeList -> { var fsl = fbs.type(new FbsFixedSizeList()); - yield new DType.FixedSizeList(convertDType(fsl.elementType(), depth + 1), (int) fsl.size(), fsl.nullable()); + yield new DType.FixedSizeList(convertDType(fsl.elementType()), (int) fsl.size(), fsl.nullable()); } case FbsType.FbsMap -> { var m = fbs.type(new FbsMap()); @@ -358,15 +336,15 @@ private static DType convertDType(io.github.dfa1.vortex.core.fbs.FbsDType fbs, i if (m.keyType() == null || m.valueType() == null) { throw new VortexException("map dtype missing key_type or value_type"); } - DType keyType = convertDType(m.keyType(), depth + 1); - DType valueType = convertDType(m.valueType(), depth + 1); + DType keyType = convertDType(m.keyType()); + DType valueType = convertDType(m.valueType()); // A nullable key type is rejected by DType.Map's own constructor, which raises // VortexException — the security contract holds without a second check here. yield new DType.Map(keyType, valueType, m.keysSorted(), m.nullable()); } case FbsType.FbsExtension -> { var e = fbs.type(new FbsExtension()); - DType storage = convertDType(e.storageDtype(), depth + 1); + DType storage = convertDType(e.storageDtype()); // Rust requires the field, empty when there is no metadata MemorySegment metadata = e.metadataAsSegment(); if (metadata == null) { @@ -393,7 +371,7 @@ private static DType convertDType(io.github.dfa1.vortex.core.fbs.FbsDType fbs, i } var types = new ArrayList(u.dtypesLength()); for (int i = 0; i < u.dtypesLength(); i++) { - types.add(convertDType(u.dtypes(i), depth + 1)); + types.add(convertDType(u.dtypes(i))); } var typeIds = new ArrayList(u.typeIdsLength()); for (int i = 0; i < u.typeIdsLength(); i++) { diff --git a/reader/src/main/java/io/github/dfa1/vortex/reader/ScanIterator.java b/reader/src/main/java/io/github/dfa1/vortex/reader/ScanIterator.java index 20398c42c..9abbb82ca 100644 --- a/reader/src/main/java/io/github/dfa1/vortex/reader/ScanIterator.java +++ b/reader/src/main/java/io/github/dfa1/vortex/reader/ScanIterator.java @@ -1220,8 +1220,8 @@ private ArrayStats readFlatStats(Layout flat) { return ArrayStats.empty(); } return ArrayStats.fromFbs(root.stats()); - } catch (RuntimeException _) { - // The FlatBuffer runtime trusts the offsets it follows: a crafted one fails inside it. + } catch (VortexException _) { + // The FlatBuffer verifier rejected the stats segment: malformed stats read as none. return ArrayStats.empty(); } } diff --git a/reader/src/main/java/io/github/dfa1/vortex/reader/SerializedArrayDecoder.java b/reader/src/main/java/io/github/dfa1/vortex/reader/SerializedArrayDecoder.java index 1fcad3f78..25723b5a6 100644 --- a/reader/src/main/java/io/github/dfa1/vortex/reader/SerializedArrayDecoder.java +++ b/reader/src/main/java/io/github/dfa1/vortex/reader/SerializedArrayDecoder.java @@ -30,17 +30,6 @@ /// FlatBuffer parsing, buffer-offset arithmetic, and encoding-spec lookup. final class SerializedArrayDecoder { - /// Hard cap on array-node recursion depth. The encoded array tree nests through child nodes - /// (validity, patches, run-ends, dictionary codes/values, …); a crafted or self-referential - /// FlatBuffer can drive [#convertArrayNode] into unbounded recursion and a [StackOverflowError] - /// — an `Error`, so it would bypass the [VortexException] - /// contract. 64 is well past any real encoding's nesting. - static final int MAX_ARRAY_TREE_DEPTH = 64; - - /// Wire size of the `Buffer` struct in `array.fbs` (u16 padding, u8 alignment, u8 compression, - /// u32 length). - private static final int BUFFER_STRUCT_BYTES = 8; - private final ReadRegistry registry; /// Creates a decoder backed by the given registry. @@ -68,43 +57,28 @@ public Array decode(MemorySegment seg, List encodingSpecs, long fbStart = segLen - 4L - fbLen; IoBounds.checkRange(fbStart, fbLen, segLen); - MemorySegment[] bufs; - ArrayNode rootNode; - // The FlatBuffer runtime trusts the offsets it follows (see FbsTable): a crafted vtable or - // vector offset fails inside it as a raw IndexOutOfBoundsException, so the whole parse is - // framed here, as PostscriptParser frames the footer blobs. - try { - var fbArray = io.github.dfa1.vortex.core.fbs.FbsArray.getRootAsFbsArray(seg.asSlice(fbStart, fbLen)); - int numBuffers = vectorCount(fbArray.buffersLength(), BUFFER_STRUCT_BYTES, fbLen); - bufs = new MemorySegment[numBuffers]; - long dataOffset = 0; - for (int i = 0; i < numBuffers; i++) { - FbsBuffer bufDesc = fbArray.buffers(i); - dataOffset += bufDesc.padding(); - bufs[i] = IoBounds.slice(seg, dataOffset, bufDesc.length()); - dataOffset += bufDesc.length(); - } - rootNode = convertArrayNode(fbArray.root(), encodingSpecs, fbLen, 0); - } catch (VortexException e) { - throw e; - } catch (RuntimeException e) { - throw new VortexException("malformed array flatbuffer", e); + // Verified whole on parse (FbsVerifier), so the accessors below stay inside it, every vector + // fits in it, and its tables nest at most 64 deep. + var fbArray = io.github.dfa1.vortex.core.fbs.FbsArray.getRootAsFbsArray(seg.asSlice(fbStart, fbLen)); + MemorySegment[] bufs = new MemorySegment[fbArray.buffersLength()]; + long dataOffset = 0; + for (int i = 0; i < bufs.length; i++) { + FbsBuffer bufDesc = fbArray.buffers(i); + dataOffset += bufDesc.padding(); + bufs[i] = IoBounds.slice(seg, dataOffset, bufDesc.length()); + dataOffset += bufDesc.length(); + } + var root = fbArray.root(); + if (root == null) { + throw new VortexException("array flatbuffer missing root"); } - var ctx = new DecodeContext(rootNode, dtype, rowCount, bufs, registry, arena); + var ctx = new DecodeContext(convertArrayNode(root, encodingSpecs), dtype, rowCount, bufs, registry, arena); return registry.decode(ctx); } - private static ArrayNode convertArrayNode( - io.github.dfa1.vortex.core.fbs.FbsArrayNode fbs, - List encodingSpecs, - int fbLen, - int depth - ) { - if (depth > MAX_ARRAY_TREE_DEPTH) { - throw new VortexException( - "array tree depth exceeds limit (" + MAX_ARRAY_TREE_DEPTH + ")"); - } + private static ArrayNode convertArrayNode(io.github.dfa1.vortex.core.fbs.FbsArrayNode fbs, + List encodingSpecs) { int encoding = fbs.encoding(); if (encoding >= encodingSpecs.size()) { throw new VortexException("array encoding index " + encoding + " out of range [0, " @@ -113,12 +87,12 @@ private static ArrayNode convertArrayNode( // Already parsed (and blank-guarded) at the footer boundary. EncodingId encodingId = encodingSpecs.get(encoding); - ArrayNode[] children = new ArrayNode[vectorCount(fbs.childrenLength(), Integer.BYTES, fbLen)]; + ArrayNode[] children = new ArrayNode[fbs.childrenLength()]; for (int i = 0; i < children.length; i++) { - children[i] = convertArrayNode(fbs.children(i), encodingSpecs, fbLen, depth + 1); + children[i] = convertArrayNode(fbs.children(i), encodingSpecs); } - int[] bufferIndices = new int[vectorCount(fbs.buffersLength(), Short.BYTES, fbLen)]; + int[] bufferIndices = new int[fbs.buffersLength()]; for (int i = 0; i < bufferIndices.length; i++) { bufferIndices[i] = fbs.buffers(i); } @@ -126,15 +100,4 @@ private static ArrayNode convertArrayNode( MemorySegment meta = fbs.metadataAsSegment(); return new ArrayNode(encodingId, meta, children, bufferIndices); } - - /// Checks a FlatBuffer vector's claimed length before it sizes a Java array: the elements must - /// fit in the FlatBuffer, else a crafted length near [Integer#MAX_VALUE] is an - /// [OutOfMemoryError] (an `Error`, past the [VortexException] framing) before any element is read. - private static int vectorCount(int count, int elementBytes, int fbLen) { - if (count < 0 || (long) count * elementBytes > fbLen) { - throw new VortexException("array flatbuffer vector of " + count + " elements exceeds its " - + fbLen + "-byte flatbuffer"); - } - return count; - } } diff --git a/reader/src/main/java/io/github/dfa1/vortex/reader/VortexHttpReader.java b/reader/src/main/java/io/github/dfa1/vortex/reader/VortexHttpReader.java index 3b396d1f5..865de7f7a 100644 --- a/reader/src/main/java/io/github/dfa1/vortex/reader/VortexHttpReader.java +++ b/reader/src/main/java/io/github/dfa1/vortex/reader/VortexHttpReader.java @@ -124,39 +124,23 @@ public static VortexHttpReader open(URI uri, ReadRegistry registry, LayoutRegist MemorySegment postscriptSeg = IoBounds.slice(tailSeg, psOffInTail, trailer.postscriptLen()); - record Blob(long offset, long length) { + var ps = FbsPostscript.getRootAsFbsPostscript(postscriptSeg); + + var footerSpec = ps.footer(); + if (footerSpec == null) { + throw new VortexException("postscript missing footer segment"); } - Blob footerBlob; - Blob layoutBlob; - Blob dtypeBlob; - // The FlatBuffer runtime trusts the offsets it follows (see FbsTable): a crafted one fails - // inside it as a raw IndexOutOfBoundsException, framed here as PostscriptParser.parse - // frames the local-file path. - try { - var ps = FbsPostscript.getRootAsFbsPostscript(postscriptSeg); - var footerSpec = ps.footer(); - if (footerSpec == null) { - throw new VortexException("postscript missing footer segment"); - } - var layoutSpec = ps.layout(); - if (layoutSpec == null) { - throw new VortexException("postscript missing layout segment"); - } - var dtypeSpec = ps.dtype(); - footerBlob = new Blob(footerSpec.offset(), footerSpec.length()); - layoutBlob = new Blob(layoutSpec.offset(), layoutSpec.length()); - dtypeBlob = dtypeSpec != null ? new Blob(dtypeSpec.offset(), dtypeSpec.length()) : new Blob(0, 0); - } catch (VortexException e) { - throw e; - } catch (RuntimeException e) { - throw new VortexException("malformed postscript", e); + var layoutSpec = ps.layout(); + if (layoutSpec == null) { + throw new VortexException("postscript missing layout segment"); } + var dtypeSpec = ps.dtype(); - MemorySegment footerBuf = fetchBlob(footerBlob.offset(), footerBlob.length(), tailStart, tail, uri, client); - MemorySegment layoutBuf = fetchBlob(layoutBlob.offset(), layoutBlob.length(), tailStart, tail, uri, client); - MemorySegment dtypeBuf = dtypeBlob.length() > 0 - ? fetchBlob(dtypeBlob.offset(), dtypeBlob.length(), tailStart, tail, uri, client) - : null; + MemorySegment footerBuf = fetchBlob(footerSpec.offset(), footerSpec.length(), tailStart, tail, uri, client); + MemorySegment layoutBuf = fetchBlob(layoutSpec.offset(), layoutSpec.length(), tailStart, tail, uri, client); + MemorySegment dtypeBuf = (dtypeSpec != null && dtypeSpec.length() > 0) + ? fetchBlob(dtypeSpec.offset(), dtypeSpec.length(), tailStart, tail, uri, client) + : null; var parsed = PostscriptParser.parseBlobs(footerBuf, layoutBuf, dtypeBuf); // Reject footer segmentSpecs that fall outside the file before any rawSegment() builds a diff --git a/reader/src/test/java/io/github/dfa1/vortex/reader/ArrayNodeDepthBombSecurityTest.java b/reader/src/test/java/io/github/dfa1/vortex/reader/ArrayNodeDepthBombSecurityTest.java index 252291f55..687f21aa3 100644 --- a/reader/src/test/java/io/github/dfa1/vortex/reader/ArrayNodeDepthBombSecurityTest.java +++ b/reader/src/test/java/io/github/dfa1/vortex/reader/ArrayNodeDepthBombSecurityTest.java @@ -19,13 +19,11 @@ /// [SerializedArrayDecoder]'s `convertArrayNode`. /// /// The decoder walks the array node tree recursively (validity, patches, run-ends, dictionary -/// codes/values, …). Without the [SerializedArrayDecoder#MAX_ARRAY_TREE_DEPTH] cap a crafted segment -/// with thousands of nested children produces a [StackOverflowError] — an `Error` that escapes the -/// "malformed input must surface as [VortexException]" contract (ADR 0003). -/// -/// The two cases bracket the exact `depth > MAX_ARRAY_TREE_DEPTH` boundary: a tree whose deepest -/// node sits at exactly the limit must clear the guard (and then fail later for an unrelated reason -/// — no decoder), while one node deeper must be rejected by the depth guard itself. +/// codes/values, …). A crafted segment with thousands of nested children would overflow the stack +/// with a [StackOverflowError], an `Error` that escapes the "malformed input must surface as +/// [VortexException]" contract (ADR 0003). The FlatBuffer verifier bounds it before the walk: like +/// Rust's, it rejects tables nested deeper than 64, and here the `Array` table plus `depth + 1` +/// nodes are `depth + 2` tables. class ArrayNodeDepthBombSecurityTest { private static final DType DTYPE = DType.I32; @@ -35,11 +33,9 @@ class ArrayNodeDepthBombSecurityTest { @Test void arrayTreeAtDepthLimit_clearsGuard() { try (Arena arena = Arena.ofConfined()) { - // Given — deepest node at exactly MAX_ARRAY_TREE_DEPTH: convertArrayNode is entered with - // depth == limit there, and `limit > limit` is false. The walk completes and only then - // fails because the empty registry has no decoder. Kills `depth >` relaxed to `>=`, - // which would wrongly reject this legal max-depth tree with the depth message. - byte[] fb = deeplyNestedArrayFlatBuffer(SerializedArrayDecoder.MAX_ARRAY_TREE_DEPTH); + // Given — 64 tables, exactly the verifier's limit: the walk completes and only then + // fails because the empty registry has no decoder + byte[] fb = deeplyNestedArrayFlatBuffer(62); MemorySegment seg = wrapAsSegment(fb, arena); List encodings = List.of(EncodingId.parse("vortex.flat")); @@ -53,16 +49,15 @@ void arrayTreeAtDepthLimit_clearsGuard() { @Test void arrayTreeOneOverDepthLimit_throwsVortexException() { try (Arena arena = Arena.ofConfined()) { - // Given — one level deeper: the deepest node reaches limit + 1, tripping the guard - // before any StackOverflowError can escape. - byte[] fb = deeplyNestedArrayFlatBuffer(SerializedArrayDecoder.MAX_ARRAY_TREE_DEPTH + 1); + // Given — one table deeper (65): rejected by the verifier before the walk starts + byte[] fb = deeplyNestedArrayFlatBuffer(63); MemorySegment seg = wrapAsSegment(fb, arena); List encodings = List.of(EncodingId.parse("vortex.flat")); // When / Then — must surface as VortexException, not StackOverflowError assertThatThrownBy(() -> sut.decode(seg, encodings, DTYPE, 1, arena)) .isInstanceOf(VortexException.class) - .hasMessageContaining("depth"); + .hasMessageContaining("nest deeper than 64"); } } diff --git a/reader/src/test/java/io/github/dfa1/vortex/reader/DTypeDepthBombSecurityTest.java b/reader/src/test/java/io/github/dfa1/vortex/reader/DTypeDepthBombSecurityTest.java index f63afb882..752ebddbc 100644 --- a/reader/src/test/java/io/github/dfa1/vortex/reader/DTypeDepthBombSecurityTest.java +++ b/reader/src/test/java/io/github/dfa1/vortex/reader/DTypeDepthBombSecurityTest.java @@ -22,7 +22,7 @@ /// Adversarial test for the DType-tree recursion in [PostscriptParser]'s `convertDType`. /// /// `convertDType` walks Struct fields, List/FixedSizeList element types, and Extension storage -/// types recursively. Without the [PostscriptParser#MAX_DTYPE_DEPTH] cap a crafted file with +/// types recursively. Without the FlatBuffer verifier's 64-table nesting cap a crafted file with /// thousands of nested `List` types produces a [StackOverflowError] during `VortexReader.open` — /// an `Error` that escapes the [VortexException] sanitization and leaks the reader's memory-mapped /// Arena (`open`'s `catch (Exception)` never runs `arena.close()`). This pins the contract: deeply @@ -36,8 +36,8 @@ class DTypeDepthBombSecurityTest { void deeplyNestedDtype_throwsVortexException(NestKind kind, @TempDir Path tmp) throws Exception { // Given — a file whose DType nests 65536 levels of `kind`. Real schemas nest a handful of // levels; 65536 reliably blows the JVM stack on the recursive convertDType walk. Each kind - // drives a different recursion arm (List/Struct/FixedSizeList/Extension), so all four must - // increment the depth counter for the MAX_DTYPE_DEPTH guard to bound them. + // drives a different recursion arm (List/Struct/FixedSizeList/Extension); the verifier + // walks each union member, so all four must be bounded by its table-nesting cap. Path file = buildDeeplyNestedDtypeFile(tmp, 65536, kind); // When / Then — must surface as VortexException, not StackOverflowError diff --git a/reader/src/test/java/io/github/dfa1/vortex/reader/PostscriptParserParseBlobsBoundsTest.java b/reader/src/test/java/io/github/dfa1/vortex/reader/PostscriptParserParseBlobsBoundsTest.java index 7dea5af77..f03a95725 100644 --- a/reader/src/test/java/io/github/dfa1/vortex/reader/PostscriptParserParseBlobsBoundsTest.java +++ b/reader/src/test/java/io/github/dfa1/vortex/reader/PostscriptParserParseBlobsBoundsTest.java @@ -54,15 +54,14 @@ void parseBlobs_layoutEncodingIndex_equalToSize_throws() { .hasMessageContaining("encoding index"); } - // ── FbsLayout depth bound: depth > MAX_LAYOUT_DEPTH (64) ──────────────────────── + // ── Layout nesting: the FlatBuffer verifier caps tables at 64 deep, as Rust does ── @Test void parseBlobs_layoutDepth_atLimit_parses() { - // Given — a single-child chain whose deepest node sits at exactly MAX_LAYOUT_DEPTH. - // convertLayout is called with depth == 64 there, and `64 > 64` is false. Kills the - // `depth >` relaxed to `depth >=` mutant, which would reject the legal max-depth tree. + // Given — a single-child chain of 64 layout tables (63 wraps around the leaf): exactly the + // verifier's depth limit, so it parses MemorySegment footer = footerWithLayoutSpecs("vortex.flat"); - MemorySegment layout = nestedLayout(PostscriptParser.MAX_LAYOUT_DEPTH); + MemorySegment layout = nestedLayout(63); // When / Then assertThatCode(() -> PostscriptParser.parseBlobs(footer, layout, null)) @@ -71,14 +70,14 @@ void parseBlobs_layoutDepth_atLimit_parses() { @Test void parseBlobs_layoutDepth_oneOverLimit_throws() { - // Given — one level deeper: the deepest node reaches depth 65, tripping `65 > 64` + // Given — one table deeper (65): rejected before convertLayout recurses at all MemorySegment footer = footerWithLayoutSpecs("vortex.flat"); - MemorySegment layout = nestedLayout(PostscriptParser.MAX_LAYOUT_DEPTH + 1); + MemorySegment layout = nestedLayout(64); // When / Then assertThatThrownBy(() -> PostscriptParser.parseBlobs(footer, layout, null)) .isInstanceOf(VortexException.class) - .hasMessageContaining("depth"); + .hasMessageContaining("nest deeper than 64"); } // ── FbsDecimal precision bound: precision < 1 || precision > 76 ───────────────── @@ -162,14 +161,12 @@ void parseBlobs_layoutMetadata_oneOverLimit_throws() { .hasMessageContaining("metadata size"); } - // ── FbsDType depth bound: depth > MAX_DTYPE_DEPTH (64) ────────────────────────── + // ── DType nesting: DType + List per level, so 64 tables allow 31 list wraps ── @Test void parseBlobs_dtypeDepth_atLimit_parses() { - // Given — a single-child List chain whose deepest node sits at exactly MAX_DTYPE_DEPTH. - // convertDType is entered with depth == 64 there, and `64 > 64` is false. Kills the - // `depth >` relaxed to `depth >=` mutant, which would reject the legal max-depth dtype. - MemorySegment dtype = nestedListDtype(PostscriptParser.MAX_DTYPE_DEPTH); + // Given — 31 List wraps: 64 tables, exactly the verifier's depth limit + MemorySegment dtype = nestedListDtype(31); // When DType result = parseDtype(dtype); @@ -180,13 +177,13 @@ void parseBlobs_dtypeDepth_atLimit_parses() { @Test void parseBlobs_dtypeDepth_oneOverLimit_throws() { - // Given — one level deeper: the deepest node reaches depth 65, tripping `65 > 64` - MemorySegment dtype = nestedListDtype(PostscriptParser.MAX_DTYPE_DEPTH + 1); + // Given — one wrap more: 66 tables + MemorySegment dtype = nestedListDtype(32); // When / Then assertThatThrownBy(() -> parseDtype(dtype)) .isInstanceOf(VortexException.class) - .hasMessageContaining("depth"); + .hasMessageContaining("nest deeper than 64"); } // ── parseBlobs dtype-presence guard: dtypeBuf != null && byteSize() > 0 ────────── diff --git a/reader/src/test/java/io/github/dfa1/vortex/reader/SerializedArrayBoundsSecurityTest.java b/reader/src/test/java/io/github/dfa1/vortex/reader/SerializedArrayBoundsSecurityTest.java index 7d43ce55b..962009a2a 100644 --- a/reader/src/test/java/io/github/dfa1/vortex/reader/SerializedArrayBoundsSecurityTest.java +++ b/reader/src/test/java/io/github/dfa1/vortex/reader/SerializedArrayBoundsSecurityTest.java @@ -89,7 +89,7 @@ void bufferDescriptorLengthPastSegment_throwsVortexException() { void rootOffsetPastFlatBuffer_throwsVortexException() { try (Arena arena = Arena.ofConfined()) { // Given a valid FlatBuffer whose root uoffset is rewritten to land 2 bytes before its - // end: the runtime's vtable read there runs off the slice (the first fuzzer finding) + // end: the table's soffset there runs off the slice (the first fuzzer finding) byte[] fb = arrayFlatBufferWithOneBuffer(0L); MemorySegment seg = segmentOf(arena, fb); seg.set(VortexFormat.LE_INT, 0, fb.length - 2); @@ -97,7 +97,7 @@ void rootOffsetPastFlatBuffer_throwsVortexException() { // When / Then assertThatThrownBy(() -> sut.decode(seg, FLAT_SPEC, DTYPE, 1, arena)) .isInstanceOf(VortexException.class) - .hasMessageContaining("malformed array flatbuffer"); + .hasMessageContaining("malformed flatbuffer: table"); } } @@ -122,8 +122,8 @@ void encodingIndexPastSpecTable_throwsVortexException() { void childrenVectorClaimingMoreThanTheFlatBuffer_throwsVortexException() { try (Arena arena = Arena.ofConfined()) { // Given a children vector whose length prefix claims 100 000 entries it does not hold. - // At Integer.MAX_VALUE the `new ArrayNode[n]` it sizes would be an OutOfMemoryError, - // which no catch of RuntimeException stops; the count is checked before allocating. + // At Integer.MAX_VALUE the `new ArrayNode[n]` it sizes would be an OutOfMemoryError; + // the verifier rejects the vector before anything is allocated from its length. FbsBuilder b = new FbsBuilder(); FbsArrayNode.startChildrenVector(b, 100_000); int children = b.endVector(); @@ -134,7 +134,7 @@ void childrenVectorClaimingMoreThanTheFlatBuffer_throwsVortexException() { // When / Then assertThatThrownBy(() -> sut.decode(seg, FLAT_SPEC, DTYPE, 1, arena)) .isInstanceOf(VortexException.class) - .hasMessageContaining("vector of 100000 elements exceeds"); + .hasMessageContaining("vector of 100000 elements"); } } diff --git a/reader/src/test/java/io/github/dfa1/vortex/reader/decode/ChunkedEncodingDecoderTest.java b/reader/src/test/java/io/github/dfa1/vortex/reader/decode/ChunkedEncodingDecoderTest.java index fa9b55bd4..6d8d3615c 100644 --- a/reader/src/test/java/io/github/dfa1/vortex/reader/decode/ChunkedEncodingDecoderTest.java +++ b/reader/src/test/java/io/github/dfa1/vortex/reader/decode/ChunkedEncodingDecoderTest.java @@ -54,10 +54,10 @@ void decode_twoChunks_viewsBothWithoutConcatenating() { /// comment is the raw JDK exception the reader leaked before the guards landed. /// /// The TODO's other Chunked item — a maliciously deep child tree — is guarded and tested one - /// layer up, at the point the `ArrayNode` tree is built from the file's FlatBuffer: see - /// [io.github.dfa1.vortex.reader.SerializedArrayDecoder] `MAX_ARRAY_TREE_DEPTH` and - /// `ArrayNodeDepthBombSecurityTest`. FlatBuffers cannot encode a true cycle, so the depth cap - /// is the whole defense and nothing here can reproduce it from a hand-built node. + /// layer up, when the segment's FlatBuffer is verified (`FbsVerifier`, at most 64 nested tables) + /// before the `ArrayNode` tree is built: see `ArrayNodeDepthBombSecurityTest`. FlatBuffers cannot + /// encode a true cycle, so the depth cap is the whole defense and nothing here can reproduce it + /// from a hand-built node. @Nested class AdversarialInput { From 3aa8bd94f3294b0191ad7a96836f3dba3f07a1a3 Mon Sep 17 00:00:00 2001 From: Davide Angelocola Date: Sun, 18 Oct 2026 16:50:41 +0200 Subject: [PATCH 06/13] fix(reader): reject a zstd_buffers inner encoding id EncodingId.parse refuses Found by the scanMalformedFile fuzz target: vortex.zstd_buffers names the encoding it wraps by an id string in its metadata, and one holding a control character escaped EncodingId.parse as a raw IllegalArgumentException. The footer, extension and registry paths that parse ids from a file already convert it; this one now does too. vortex-jni rejects the same file. Refs #493 Co-Authored-By: Claude Sonnet 5 Claude-Session: https://claude.ai/code/session_01WtuVGQ5h5Lk6Zn4JnkxF1K --- CHANGELOG.md | 1 + .../crash-zstd-buffers-inner-id-control-char | Bin 0 -> 8832 bytes .../MalformedFileInteropIntegrationTest.java | 5 ++++- .../zstd-buffers-inner-id-control-char.vortex | Bin 0 -> 8832 bytes .../decode/ZstdBuffersEncodingDecoder.java | 12 +++++++++++- .../decode/ZstdBuffersEncodingDecoderTest.java | 12 ++++++++++++ 6 files changed, 28 insertions(+), 2 deletions(-) create mode 100644 fuzz/src/test/resources/io/github/dfa1/vortex/fuzz/VortexReaderFuzzTestInputs/scanMalformedFile/crash-zstd-buffers-inner-id-control-char create mode 100644 integration/src/test/resources/fixtures/malformed/zstd-buffers-inner-id-control-char.vortex diff --git a/CHANGELOG.md b/CHANGELOG.md index 409a48e9b..1941d9a1f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -45,6 +45,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - A file whose array FlatBuffer holds a corrupt offset fails the read with `VortexException` instead of `IndexOutOfBoundsException`. ([#493](https://github.com/dfa1/vortex-java/issues/493)) - A column whose dtype does not fit its bit-packed, delta or RLE encoding fails with `VortexException` instead of `ClassCastException`, and an empty RLE boolean chunk reads. ([#493](https://github.com/dfa1/vortex-java/issues/493)) - A struct dtype missing its field names or types, or a struct layout whose children do not match its fields, is rejected when the file opens, as Rust does, instead of failing the scan with `ArrayIndexOutOfBoundsException`. ([#493](https://github.com/dfa1/vortex-java/issues/493)) +- A `vortex.zstd_buffers` array whose wrapped encoding id holds a control character fails with `VortexException` instead of `IllegalArgumentException`. ([#493](https://github.com/dfa1/vortex-java/issues/493)) - SQL `WHERE` on an `I16`, `U8` or `I8` column no longer fails with a `ClassCastException`. ([#556](https://github.com/dfa1/vortex-java/pull/556)) - SQL `COUNT(DISTINCT col)` and aggregates with a `FILTER (WHERE ...)` clause return the right answer; the statistics push-down returned the row count and ignored the filter. ([#554](https://github.com/dfa1/vortex-java/pull/554)) - SQL `MIN`/`MAX` of a string column whose values fall outside Latin-1 answers instead of failing the query. ([#554](https://github.com/dfa1/vortex-java/pull/554)) diff --git a/fuzz/src/test/resources/io/github/dfa1/vortex/fuzz/VortexReaderFuzzTestInputs/scanMalformedFile/crash-zstd-buffers-inner-id-control-char b/fuzz/src/test/resources/io/github/dfa1/vortex/fuzz/VortexReaderFuzzTestInputs/scanMalformedFile/crash-zstd-buffers-inner-id-control-char new file mode 100644 index 0000000000000000000000000000000000000000..6807a33205a54a861994bfb7155456d4b3cc53e8 GIT binary patch literal 8832 zcmdtocUTlxzX$N4t%zt4K>@*_h=3w2pcF-o6%eerA}UDR#YKc|uz*IfAqt`*AQ}s( zSYs>+Q40O>#KRu)L?6m%_^QL9HBwdD<4nE2|b;rbe z@nDJW$p?6@REU4^k@6sn^!mspra-*Hs|{m3DRNA?s?@=5pm3l@(uktw#N5 zM2)&!ru;#r?yFp0taJXWR=q*%tU5ueoUT%vDt*e;qYss-drFm~Rhx~Jug|ETg`H8?NtAq5NC)MvQuSt| zQgyOK*Ps%Y*dfL2nn=ga~RvY->VZR)>^VhvZhhr zl%`#j0%K)vXXOKBuFR;3T{%G>M8{~ z%3N*b6)mLz|84ooCT7ez%l`7EwUbqQ)rni`>CpP zAD`UJ9$pa*_G@iMX7RkVqjpqWu<}t@mP;GXUa;8inpNic?p}JvcE_ym&4x=am#sDS z@v-bIH7T3D+{Q91#xwiw!(}#I$MmsWCDARN)XvAkGK9bKPUWKQdRa!EH|0B`bxS4< zj_^6ZFQr(%`FXDgrsrzDyXNj3>m&~S?Rb8&?Q_@r18c9;_^-d#-+6lIq2n`;O-#;h zHg0@bm9Z|U&$ZspzM-p*n^e|D>Qzl_;_v7s%P5uSIYe}GFAqe zINOE#A77JXD*e!O?{4mynzU<%&Sp-gp?1f8la@6#-zh#VIPJgqnvOFswEc0zqr9ZM z$&;KP6&EC=oj30r?YpTkKJDnD@69)g^fwlY(sIlP@eD+j>kC8DB<6dCzFpIE3#X(7 zm|F<-a|*{7_@~*MuZ*f(TR1w+Xpye@yU6s_h2CjT%r@%8MEb5ObW5u=Gl;D0pr29L zH*JSmN<{jK!rlcv(^AbIhx;xs?4C9|b*9<$aQ$V4#%ZI>-f`7om5U48r&*iDg!wMg zPHQ&Q?tk4; z@MG_~i*Ek4Ihny0qW1Ykp?QknUy4j>vite(b1XbOr8?W*c6^>|@Zl(NwykfTg@e`M zQI(Ue{TDd?acHTJUsjPv-t}o?BdaotT*ihtiVrEujZOV0IeJYR5wR!TwrAdgY4^e% zmR~gV=Q|c3+);Y5y?^fN39zc6Xa2x>Q?Gm9xRh%|_I!J-4T(>4^jy4mUouT}-DyJ71# zR^6SI$PS-BJp&P8@rYCAwYmnc#L2u2tA7{Th*Lkqd zqs+5j+7EVrXEQhJ2lIuZw>mXjr+(bs{2*uI$E4=yb;I)C?`c@wJM+C?!|I+3d2e+p zw+?;!W@N=y&yPo+MXwq*JN?;%jA1k1SKV7}x^U<0d#jBXX1y6%w$=LMo76K~Ek0H@ zJy5x$*!d{~ubj&n)DW=D zt+DR1d*Jl|!Sh-Zx4^0ZbGL!DCj4Il)UKmzrZ@)v5U_u8b&h@D)&Qw%ZdH!$nt*|> zMU|I(1}+WIJ6vJXBQPPL%H`!*lkTE`RW5pGr*sRP9uU&GJf};be?U(cL0OKGcR(Xw zc;>QUAU~j3QfksZuxCIrzxcEX&p2QRzr19Mb|5dH^O{n((c}CA|B>jSv>0?m7I#l| z@|aX!vvZt$$^EOf<2KxK@rc;yd?Y09s4C`Im26&7k<6{py}D!03_Fi;7Pd!x5x2M~Rq4@W>8))7>AHKb%2utIVeDaJVRWQN+#prl z(F>A{!u^twcc+$)TY9UVN2iT?N7~03sVt5d=5>e}5sb@fjhjPTW8sej7yeBd4z zBkN~9T+Rib^SY^e9s2rq_didq?Z7h}5Y{x&d3SBZ@A0={{{ELTK5muMx_^jR!Hk>r z4c+<;3@g!U$2-V-I?OsTt0Y|3DSfleo&5ag?fs_iyD)uD@SOd^D~%H~UB4NMEW?(@qkv+f~dXe79KHvb3#0kj7?zUd z$d!-@S!4||8*(6z?V7W-@^}Zko<`J8IHlPg+j zOd^5EsSpe^NhmT5A|Q&0kh38MVo5ww1`0?dNyrqK4+}^taxpA}bg~ke30bg)WFvDR z5Aw-($SqI^+er~}7wmz3WIyr%9D<+75#%xW6;6;-$kT8J%1H&X8tULYxrn?BSK&Ij ziB!RDxI^lZ_u(NtCQp!0p$VGF3*;+!3-8HCB&Uu4K$q~4`d|Qt6d8l`g|QGo#v>=fWSByNkker%gpx31 z1Vn*|%tpq*T#yhMQUQsOL{gCRVF9F)CCD^b0U2Z!ay6`lbtD(L9yY)xvK3hf+o6c; zLhgZmu%8@29)@B#N`65ehm%l3N|9x77Ai>%@*JFpi{vu$DqM$~M1{N!ccFpYM?Qqd z@Ps@?HbFDIAg__{-~)UjS~?u31A5Sov`2P;j$lN(AWfkgn3L{EEBG3E5*uV6Z~#Z* zgye%O3?S}EPZ$K=WEgS;jDj)57daLJU_6ta!4RbWHQWyZ^=UBVps<0 zWF;~avS1C#M&>{sj64dz02{ymt+CV9Sx%t_qN{z3@FbfIJMvaD*H~{t74H z6giDN1LaUjYLIpC8(bilkXPUu+#t7*x8W`{ko(Ao@ED$ur^qH~h8N^D@*TW~kA&T5 zX@f5Ch(6K)I)V`~Mw&oZFe4U7OXvaC#0J?19H1ZRk8}nXa3cee9>DJWyvZ=+a2N@r zi4W2b{9zoKfSe2hm`0`}XF@21kqBfIh+sCEgNy?S$cO@&2uYAa<|7xtB3Me&km;}z zGD#M44P-+OS&!TZn_(*{L~e&7*hTgr_rZQRNPa{X!x1<}enp;uQ*fG`L7s&ws3qr+ z=iwq;CRdTy;U=ia@5p=50Qbp5;LrU(|_LNf6&U8 zn`oxamivFUs*fiEy7ss60y*z`2dD|(}hwZXPOiDO`ipGll`0gq$*ve%!N zPS}=*qcOx0m}s7tu)W#%ID$2pfRU!BSkWx)G=0aa7Hf~S-MaL}ib2Q3c$q?&+H75VV#UBH@heULB{K0`u|k|Ew9qd2x|f!&rsLYJml~Ik z6Uf>+s2KyU9Xsc3uhE%!&Fa~CX2;Z0-wD@D%QcK5Jyyn^eCeYUA2+sWu{PRlqtmh< zyWUtSTXsWs73N6`T!Tv-<%@FoY(I$9eb(azFu7fZdB3 z!W|4cyZ77;sle{9eZd~s^8)tlfjz@G2<%x)Eck*w=)*(29~+z9uoR+T6xajao*VJL z-4z}pK5VAPFcJa%0TV@2*J^iu{mtaC9n&k{WwDhCR@yI3NGPw|SeW;F2%hX4? zP?{i&ixhGrzT8ii5GRad;}lAD?3~kne}6hr#4VtDb`7-FN5-T7RNl2sdAJO3s9>{O z%Q*b<<@?6RNy5c4>~ods+KhQzLToJe^viwbhRf#&*|8eXoZaSF=7vk~O+w{u`W`Em zD>S1$lvoyPM*|bfY;}9m{1nvE@%_uR> zTq%{Ce;%tSVWfC&cNbJXhGd zJIB6<**0y)Au>Kru3#7C7^-X2cI51YIL*LDe7PSxu88>fSZ)E;wHZH6c^;Lsxs8q| zJXRuOr-?gEb#2B6b-1sia(4ZS!sUwC@HnB|Nfyhl-~0czP82W0tf-GqF`L6#-`HpP z^S%*cg(N(Zbpq3XW`06&ncZ8yd)ZjA#Sn92%S>jZ`L1A}rHR1iG@*_h=3w2pcF-o6%eerA}UDR#YKc|uz*IfAqt`*AQ}s( zSYs>+Q40O>#KRu)L?6m%_^QL9HBwdD<4nE2|b;rbe z@nDJW$p?6@REU4^k@6sn^!mspra-*Hs|{m3DRNA?s?@=5pm3l@(uktw#N5 zM2)&!ru;#r?yFp0taJXWR=q*%tU5ueoUT%vDt*e;qYss-drFm~Rhx~Jug|ETg`H8?NtAq5NC)MvQuSt| zQgyOK*Ps%Y*dfL2nn=ga~RvY->VZR)>^VhvZhhr zl%`#j0%K)vXXOKBuFR;3T{%G>M8{~ z%3N*b6)mLz|84ooCT7ez%l`7EwUbqQ)rni`>CpP zAD`UJ9$pa*_G@iMX7RkVqjpqWu<}t@mP;GXUa;8inpNic?p}JvcE_ym&4x=am#sDS z@v-bIH7T3D+{Q91#xwiw!(}#I$MmsWCDARN)XvAkGK9bKPUWKQdRa!EH|0B`bxS4< zj_^6ZFQr(%`FXDgrsrzDyXNj3>m&~S?Rb8&?Q_@r18c9;_^-d#-+6lIq2n`;O-#;h zHg0@bm9Z|U&$ZspzM-p*n^e|D>Qzl_;_v7s%P5uSIYe}GFAqe zINOE#A77JXD*e!O?{4mynzU<%&Sp-gp?1f8la@6#-zh#VIPJgqnvOFswEc0zqr9ZM z$&;KP6&EC=oj30r?YpTkKJDnD@69)g^fwlY(sIlP@eD+j>kC8DB<6dCzFpIE3#X(7 zm|F<-a|*{7_@~*MuZ*f(TR1w+Xpye@yU6s_h2CjT%r@%8MEb5ObW5u=Gl;D0pr29L zH*JSmN<{jK!rlcv(^AbIhx;xs?4C9|b*9<$aQ$V4#%ZI>-f`7om5U48r&*iDg!wMg zPHQ&Q?tk4; z@MG_~i*Ek4Ihny0qW1Ykp?QknUy4j>vite(b1XbOr8?W*c6^>|@Zl(NwykfTg@e`M zQI(Ue{TDd?acHTJUsjPv-t}o?BdaotT*ihtiVrEujZOV0IeJYR5wR!TwrAdgY4^e% zmR~gV=Q|c3+);Y5y?^fN39zc6Xa2x>Q?Gm9xRh%|_I!J-4T(>4^jy4mUouT}-DyJ71# zR^6SI$PS-BJp&P8@rYCAwYmnc#L2u2tA7{Th*Lkqd zqs+5j+7EVrXEQhJ2lIuZw>mXjr+(bs{2*uI$E4=yb;I)C?`c@wJM+C?!|I+3d2e+p zw+?;!W@N=y&yPo+MXwq*JN?;%jA1k1SKV7}x^U<0d#jBXX1y6%w$=LMo76K~Ek0H@ zJy5x$*!d{~ubj&n)DW=D zt+DR1d*Jl|!Sh-Zx4^0ZbGL!DCj4Il)UKmzrZ@)v5U_u8b&h@D)&Qw%ZdH!$nt*|> zMU|I(1}+WIJ6vJXBQPPL%H`!*lkTE`RW5pGr*sRP9uU&GJf};be?U(cL0OKGcR(Xw zc;>QUAU~j3QfksZuxCIrzxcEX&p2QRzr19Mb|5dH^O{n((c}CA|B>jSv>0?m7I#l| z@|aX!vvZt$$^EOf<2KxK@rc;yd?Y09s4C`Im26&7k<6{py}D!03_Fi;7Pd!x5x2M~Rq4@W>8))7>AHKb%2utIVeDaJVRWQN+#prl z(F>A{!u^twcc+$)TY9UVN2iT?N7~03sVt5d=5>e}5sb@fjhjPTW8sej7yeBd4z zBkN~9T+Rib^SY^e9s2rq_didq?Z7h}5Y{x&d3SBZ@A0={{{ELTK5muMx_^jR!Hk>r z4c+<;3@g!U$2-V-I?OsTt0Y|3DSfleo&5ag?fs_iyD)uD@SOd^D~%H~UB4NMEW?(@qkv+f~dXe79KHvb3#0kj7?zUd z$d!-@S!4||8*(6z?V7W-@^}Zko<`J8IHlPg+j zOd^5EsSpe^NhmT5A|Q&0kh38MVo5ww1`0?dNyrqK4+}^taxpA}bg~ke30bg)WFvDR z5Aw-($SqI^+er~}7wmz3WIyr%9D<+75#%xW6;6;-$kT8J%1H&X8tULYxrn?BSK&Ij ziB!RDxI^lZ_u(NtCQp!0p$VGF3*;+!3-8HCB&Uu4K$q~4`d|Qt6d8l`g|QGo#v>=fWSByNkker%gpx31 z1Vn*|%tpq*T#yhMQUQsOL{gCRVF9F)CCD^b0U2Z!ay6`lbtD(L9yY)xvK3hf+o6c; zLhgZmu%8@29)@B#N`65ehm%l3N|9x77Ai>%@*JFpi{vu$DqM$~M1{N!ccFpYM?Qqd z@Ps@?HbFDIAg__{-~)UjS~?u31A5Sov`2P;j$lN(AWfkgn3L{EEBG3E5*uV6Z~#Z* zgye%O3?S}EPZ$K=WEgS;jDj)57daLJU_6ta!4RbWHQWyZ^=UBVps<0 zWF;~avS1C#M&>{sj64dz02{ymt+CV9Sx%t_qN{z3@FbfIJMvaD*H~{t74H z6giDN1LaUjYLIpC8(bilkXPUu+#t7*x8W`{ko(Ao@ED$ur^qH~h8N^D@*TW~kA&T5 zX@f5Ch(6K)I)V`~Mw&oZFe4U7OXvaC#0J?19H1ZRk8}nXa3cee9>DJWyvZ=+a2N@r zi4W2b{9zoKfSe2hm`0`}XF@21kqBfIh+sCEgNy?S$cO@&2uYAa<|7xtB3Me&km;}z zGD#M44P-+OS&!TZn_(*{L~e&7*hTgr_rZQRNPa{X!x1<}enp;uQ*fG`L7s&ws3qr+ z=iwq;CRdTy;U=ia@5p=50Qbp5;LrU(|_LNf6&U8 zn`oxamivFUs*fiEy7ss60y*z`2dD|(}hwZXPOiDO`ipGll`0gq$*ve%!N zPS}=*qcOx0m}s7tu)W#%ID$2pfRU!BSkWx)G=0aa7Hf~S-MaL}ib2Q3c$q?&+H75VV#UBH@heULB{K0`u|k|Ew9qd2x|f!&rsLYJml~Ik z6Uf>+s2KyU9Xsc3uhE%!&Fa~CX2;Z0-wD@D%QcK5Jyyn^eCeYUA2+sWu{PRlqtmh< zyWUtSTXsWs73N6`T!Tv-<%@FoY(I$9eb(azFu7fZdB3 z!W|4cyZ77;sle{9eZd~s^8)tlfjz@G2<%x)Eck*w=)*(29~+z9uoR+T6xajao*VJL z-4z}pK5VAPFcJa%0TV@2*J^iu{mtaC9n&k{WwDhCR@yI3NGPw|SeW;F2%hX4? zP?{i&ixhGrzT8ii5GRad;}lAD?3~kne}6hr#4VtDb`7-FN5-T7RNl2sdAJO3s9>{O z%Q*b<<@?6RNy5c4>~ods+KhQzLToJe^viwbhRf#&*|8eXoZaSF=7vk~O+w{u`W`Em zD>S1$lvoyPM*|bfY;}9m{1nvE@%_uR> zTq%{Ce;%tSVWfC&cNbJXhGd zJIB6<**0y)Au>Kru3#7C7^-X2cI51YIL*LDe7PSxu88>fSZ)E;wHZH6c^;Lsxs8q| zJXRuOr-?gEb#2B6b-1sia(4ZS!sUwC@HnB|Nfyhl-~0czP82W0tf-GqF`L6#-`HpP z^S%*cg(N(Zbpq3XW`06&ncZ8yd)ZjA#Sn92%S>jZ`L1A}rHR1iG MAX_ALIGNMENT) { diff --git a/reader/src/test/java/io/github/dfa1/vortex/reader/decode/ZstdBuffersEncodingDecoderTest.java b/reader/src/test/java/io/github/dfa1/vortex/reader/decode/ZstdBuffersEncodingDecoderTest.java index 2926919f4..400056e8a 100644 --- a/reader/src/test/java/io/github/dfa1/vortex/reader/decode/ZstdBuffersEncodingDecoderTest.java +++ b/reader/src/test/java/io/github/dfa1/vortex/reader/decode/ZstdBuffersEncodingDecoderTest.java @@ -90,6 +90,18 @@ void missingMetadata_throws() { .hasMessageContaining("missing metadata"); } + @Test + void innerEncodingIdWithControlCharacter_throws() { + // Given — an inner id EncodingId.parse refuses (a fuzzer finding: it escaped as a raw + // IllegalArgumentException) + ArrayNode node = zstdBuffers(meta("vortex.\u000Fprimitive", VALUES.length, 4), 0); + + // When / Then + assertThatThrownBy(() -> SUT.decode(ctx(node, compress(VALUES)))) + .isInstanceOf(VortexException.class) + .hasMessageContaining("invalid inner encoding id"); + } + @Test void sizeCountDisagreesWithBuffers_throws() { // Given — one compressed buffer, no declared sizes From 230619a853fb44ef3bf091ae9e7b6c0a8db3421b Mon Sep 17 00:00:00 2001 From: Davide Angelocola Date: Sun, 18 Oct 2026 17:23:08 +0200 Subject: [PATCH 07/13] fix(reader): refuse a file that does not embed its dtype, as Rust does Found by the scanMalformedFile fuzz target: a postscript without a dtype segment opened with a null dtype, and the scan then threw a raw NullPointerException. Rust opens such a file only when the caller supplies the dtype (VortexOpenOptions::with_dtype) and otherwise fails at open ("Vortex file doesn't embed a DType ..."). vortex-java has no such option, so the file is unreadable: parseBlobs, shared by the file and HTTP readers, now refuses it. Breaking: such a file used to open. The verifier's CHANGELOG entry is marked Breaking too: it narrows dtype nesting to Rust's limit. Refs #493 Co-Authored-By: Claude Sonnet 5 Claude-Session: https://claude.ai/code/session_01WtuVGQ5h5Lk6Zn4JnkxF1K --- CHANGELOG.md | 3 +- .../scanMalformedFile/crash-no-dtype | Bin 0 -> 10714 bytes .../MalformedFileInteropIntegrationTest.java | 5 ++- .../fixtures/malformed/no-dtype.vortex | Bin 0 -> 10714 bytes .../dfa1/vortex/reader/PostscriptParser.java | 10 +++-- .../PostscriptParserBlobBoundsTest.java | 14 +++---- .../reader/PostscriptParserLayoutIdTest.java | 7 +++- .../PostscriptParserParseBlobsBoundsTest.java | 37 ++++++++++-------- 8 files changed, 43 insertions(+), 33 deletions(-) create mode 100644 fuzz/src/test/resources/io/github/dfa1/vortex/fuzz/VortexReaderFuzzTestInputs/scanMalformedFile/crash-no-dtype create mode 100644 integration/src/test/resources/fixtures/malformed/no-dtype.vortex diff --git a/CHANGELOG.md b/CHANGELOG.md index 1941d9a1f..0ca759b63 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -18,7 +18,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - Custom extension types: register an `ExtensionDecoder` on `ReadRegistry` and an `ExtensionEncoder` on `WriteRegistry`, with a `validate` hook on both sides, and `Chunk.as` decodes any registered type. ([#530](https://github.com/dfa1/vortex-java/issues/530)) ### Changed -- Every FlatBuffer in a file is verified before it is read, as Rust verifies it, so corrupt file metadata fails with `VortexException`, and like Rust the reader now refuses tables nested more than 64 deep. ([#493](https://github.com/dfa1/vortex-java/issues/493)) +- **Breaking:** a file that does not embed its dtype is refused when it opens, as Rust refuses it unless the caller supplies the dtype, instead of failing the scan with `NullPointerException`. ([#493](https://github.com/dfa1/vortex-java/issues/493)) +- **Breaking:** every FlatBuffer in a file is verified before it is read, as Rust verifies it, so corrupt file metadata fails with `VortexException`, and like Rust the reader now refuses tables nested more than 64 deep. ([#493](https://github.com/dfa1/vortex-java/issues/493)) - SQL `SUM` and `AVG` of a decimal column, grouped or not, are 4 to 6 times faster: they are totalled exactly as unscaled longs in the scan instead of by Calcite with a `BigDecimal` per row. ([#563](https://github.com/dfa1/vortex-java/pull/563)) - SQL `COUNT(DISTINCT col)` is up to 4 times faster: distinct values are counted in the scan, integers by their bits in a primitive map, instead of Calcite hashing a boxed value per row. ([#562](https://github.com/dfa1/vortex-java/pull/562)) - A SQL `WHERE` comparing a `SMALLINT` or `TINYINT` column with a literal is pushed into the scan and its aggregates, instead of being left to Calcite because of the cast Calcite adds. ([#560](https://github.com/dfa1/vortex-java/pull/560)) diff --git a/fuzz/src/test/resources/io/github/dfa1/vortex/fuzz/VortexReaderFuzzTestInputs/scanMalformedFile/crash-no-dtype b/fuzz/src/test/resources/io/github/dfa1/vortex/fuzz/VortexReaderFuzzTestInputs/scanMalformedFile/crash-no-dtype new file mode 100644 index 0000000000000000000000000000000000000000..372d316a8af1058a9d4911d32d52816ed5f13b62 GIT binary patch literal 10714 zcmeI%d0baj-oWwu`>_hBKM@gkWKqBk%zXJ-=bU@*xy$cp zS$1-u6Ftxqz0ezd&=<>~ANpee8p@F=k1ni$fmjiPuo4DiWemY8XsAY}I@Z9NSPMh3 zHilt1Mqnh?K|>UoXpF&nSRWf;Lu`bx7>AAVJ~T8X(+rzq3v7w4ur;>9w%88u$9Ob! zAkz^OuoHI1F4z^jVR!6-iP#ej50FX1-q;8GVlwu_6nqdsFPPv9T15GSEw zGMOp(6i&ryI2~u;Oq_+YaSqN!!#pzc@floz3vm%Xi;M9&T!PQzQZ&3k=0$u7i*Px< zj4N;@uEJOFReTK%tI4dvwYUz~KK!MAZ6ZpU}D}9Uc%4uGJb(q@G2UyqVBPO6}r!(a)*cH2BckF?Q*b{r<1DJ%p(X`W-ax(VA6nqd!-0$YoL_Jv@J3L` z0&7v~CM$T!(a^=GzKIQ~8q|KtfF`}IrOD|_&lHTQZY`Sh!l2FbYt|@U{?e!4ZH%w5 zBfe`TM`2QxT^&+u?CF?Z+mVnJ>9K1}v{&cH8u;ufj4RW1MpOTNjuvIRFKJu;gXIoa z;-5NIJh)LV)YhJrKitu~O47c5)s7tLU!%{d!L^QFOs}1MB{Tf^&8)~2XLj8a&b;VU zKWEITzzGcoRDHrx81`h`pqQ!m4Q@QM>5x`)o1bgwwL2mw^5goice>o9IP?29u~WS} z#TEtk+_<|=zpuSx2U|W}GUMy|jF{Br;jud&e$>l3ecDqUURyk8^4?b$y?!Ebgq zD}HT%-&qH;T>j@D3$ezB)|@h7`H<;W*h{mYSaEJ%VMOS{Cs(CCH>J+-EozpNp-t^ZAX#_bzMv>VX$GeQ~(R ziaPf4<`E}XHY@z(l`U7#zt$q=((0{`SZiCYxxW7GFTb|h)c@{{?b$!R*>2s>e}4C? z;NlL>W*>F;PH`srjk`IZ++wGPYfIqB;1gl_&c8RFSTm&Kg1Ub-Y?)`kF=nX8Dp#GPVurvuUr&Ruyd8?_)D9sKi(oOplquPiDlaiozt`3X9q{c z%Yf`412Vz+^r6l&AT7&PzPk*_MDM}}ddYxH@(nEPBLgzoZ-FaC24t!X$N_Vb;|9rq z9P;$>F=;X&t-_SX!(>1XuUMQnLI&g`i$aRBWkBXEK2>qF49GkgkOftjCXbT=IWyF; zbzYR$+pA;CZ1-zX_TBw5AWvIC?_ZMvc`Mbi+g}D`wX8@-JsFU#WkB|r6jx@T49K+k zEz3@n0U1!#zQQsYkZT=XD{YejS-is$a##lB`6K-uH)KFsG9X>oGQ&e(IiPjQrB-%=}lek|ZUZTB(rj8N`5}hPEOLURwD$z}%yF?F(M2Vgfy(Bax zN%WTJBhgnPS)!jrio}Bw|DXPqXW$>c+XAE~zE*(cDSv$>uQ!XQgolJ>IW5b_oza{- z`Mdv?vs`X31bKzgU`+coK0hr7R~ zZC#@E`1%A(PqkNeFLk-L6)b;s_-e9ivCmJ)Zijn6eW00-DfdRd{YQ3lIYKO#Beb8( zargO8lzp{d_VJqQVE0d}HP_+ZmUEwJ`#iba*Tu5D?pzyl;q}@UNtm|A%lXrOm~*Ff z>ow4Fs{7n}yYqUTyd<*kTBh4=`;9u5-L<{8zu~TB+NNG-P4=j=uR`fQMY-2m*7dvg z)qeeYjFj}*k}GJ}thJkDUsqF_E|mnCg5@XY%xJ-S>@OZr@h5FXM9a3NptxoRFK7IZTSyA=c^r#kyfx83op5 zmX|GAo?nnRR@O*gwBelkwtSeRD@a*&g_wB65QzTI+ktUHK5h39z zk-~oEC2#7?&B-rF&nd9-Sa+|!%>8wHLvwSptyg|sUw-D8v6(p;nbv#vsBd(7UVf&u zeU^0G}2+x_qzHBlze!Fa4Y= z{p``Pu472ISI0@{wm$NgOY33p?B9+-%xonDk5z$EOAeXuVkV?Rv62Qd}@e9X=g0uarh{X$H#C2PQ=IY z5BLQB5ew0@^CacTI0c`=sW=U%;|!dMvv4-fLDSCDl;`1mdq zC-5Ym!qa#LO*@}bK8xq@JYK+y_!(Zp&+#&Tfu??=`Lr{u&D&c)=QNp5a()rHUsJlg zjQ)|t+sfi?=6;pxa{V+Zo%GX6lfL#fRj{!FQ9pS!na_(#cG3MaM6;DC>(35NI+vo! z%%|8T`_-#?omamN0X&F@@I%y>yQU-f z5gtW*tz{jj{4t)ulXwbG;~D$}KgF|n4$q@$=OX3L@DhHGm+=d{f>-exeu>xdD>Uu= zjq(lr8gJq^_$_{i-{ar$2mBFlp=swQ%0J^Tc-w5pPMZwv{Qz%ANpee zmc?>d9$i=g1F<3op=l?Wa%Bv`Dp(b(VRfv5HL(_kVr?|-gj0^dNUVc(F$$wG##SE~ z_8I%${%V$k|77jA{eo3d!qcyr0jf)elS=5pwCr#rbEq5s3g!XR0%9kf5< zOP*RB!kwp>DgU-VplkDLr9vSn7|J(Bm}MQL@%md9=IVVi8)Bb+XdD)$}|SN~B4|OXw!L|FcX$ z=x;WDXx^E$FcTW3CK=q57W>(!)Q|EzlNM$|<6e^%WeeOvM9kV))c1jdU6iTE? zBuGR^cuKUBaiZR9;$i^n?$w`htSi-~d*Z@OXq2k&&cwy)d(>wpE_Sm{?*-1+=)9~^ zSp`|+GG&oKCpz?7z{#hT Hndta$-Ele( literal 0 HcmV?d00001 diff --git a/integration/src/test/java/io/github/dfa1/vortex/integration/MalformedFileInteropIntegrationTest.java b/integration/src/test/java/io/github/dfa1/vortex/integration/MalformedFileInteropIntegrationTest.java index 6c050a2b5..de54771d2 100644 --- a/integration/src/test/java/io/github/dfa1/vortex/integration/MalformedFileInteropIntegrationTest.java +++ b/integration/src/test/java/io/github/dfa1/vortex/integration/MalformedFileInteropIntegrationTest.java @@ -44,7 +44,10 @@ class MalformedFileInteropIntegrationTest { "struct-dtype-missing-names", // A vortex.zstd_buffers whose wrapped encoding id holds a control character: escaped as // IllegalArgumentException from EncodingId.parse. - "zstd-buffers-inner-id-control-char"}) + "zstd-buffers-inner-id-control-char", + // A file with no dtype blob, which Rust opens only when the caller supplies the dtype: + // vortex-java opened it with a null dtype and the scan threw NullPointerException. + "no-dtype"}) void bothReadersReject(String name) throws URISyntaxException { // Given Path file = Path.of(Objects.requireNonNull( diff --git a/integration/src/test/resources/fixtures/malformed/no-dtype.vortex b/integration/src/test/resources/fixtures/malformed/no-dtype.vortex new file mode 100644 index 0000000000000000000000000000000000000000..372d316a8af1058a9d4911d32d52816ed5f13b62 GIT binary patch literal 10714 zcmeI%d0baj-oWwu`>_hBKM@gkWKqBk%zXJ-=bU@*xy$cp zS$1-u6Ftxqz0ezd&=<>~ANpee8p@F=k1ni$fmjiPuo4DiWemY8XsAY}I@Z9NSPMh3 zHilt1Mqnh?K|>UoXpF&nSRWf;Lu`bx7>AAVJ~T8X(+rzq3v7w4ur;>9w%88u$9Ob! zAkz^OuoHI1F4z^jVR!6-iP#ej50FX1-q;8GVlwu_6nqdsFPPv9T15GSEw zGMOp(6i&ryI2~u;Oq_+YaSqN!!#pzc@floz3vm%Xi;M9&T!PQzQZ&3k=0$u7i*Px< zj4N;@uEJOFReTK%tI4dvwYUz~KK!MAZ6ZpU}D}9Uc%4uGJb(q@G2UyqVBPO6}r!(a)*cH2BckF?Q*b{r<1DJ%p(X`W-ax(VA6nqd!-0$YoL_Jv@J3L` z0&7v~CM$T!(a^=GzKIQ~8q|KtfF`}IrOD|_&lHTQZY`Sh!l2FbYt|@U{?e!4ZH%w5 zBfe`TM`2QxT^&+u?CF?Z+mVnJ>9K1}v{&cH8u;ufj4RW1MpOTNjuvIRFKJu;gXIoa z;-5NIJh)LV)YhJrKitu~O47c5)s7tLU!%{d!L^QFOs}1MB{Tf^&8)~2XLj8a&b;VU zKWEITzzGcoRDHrx81`h`pqQ!m4Q@QM>5x`)o1bgwwL2mw^5goice>o9IP?29u~WS} z#TEtk+_<|=zpuSx2U|W}GUMy|jF{Br;jud&e$>l3ecDqUURyk8^4?b$y?!Ebgq zD}HT%-&qH;T>j@D3$ezB)|@h7`H<;W*h{mYSaEJ%VMOS{Cs(CCH>J+-EozpNp-t^ZAX#_bzMv>VX$GeQ~(R ziaPf4<`E}XHY@z(l`U7#zt$q=((0{`SZiCYxxW7GFTb|h)c@{{?b$!R*>2s>e}4C? z;NlL>W*>F;PH`srjk`IZ++wGPYfIqB;1gl_&c8RFSTm&Kg1Ub-Y?)`kF=nX8Dp#GPVurvuUr&Ruyd8?_)D9sKi(oOplquPiDlaiozt`3X9q{c z%Yf`412Vz+^r6l&AT7&PzPk*_MDM}}ddYxH@(nEPBLgzoZ-FaC24t!X$N_Vb;|9rq z9P;$>F=;X&t-_SX!(>1XuUMQnLI&g`i$aRBWkBXEK2>qF49GkgkOftjCXbT=IWyF; zbzYR$+pA;CZ1-zX_TBw5AWvIC?_ZMvc`Mbi+g}D`wX8@-JsFU#WkB|r6jx@T49K+k zEz3@n0U1!#zQQsYkZT=XD{YejS-is$a##lB`6K-uH)KFsG9X>oGQ&e(IiPjQrB-%=}lek|ZUZTB(rj8N`5}hPEOLURwD$z}%yF?F(M2Vgfy(Bax zN%WTJBhgnPS)!jrio}Bw|DXPqXW$>c+XAE~zE*(cDSv$>uQ!XQgolJ>IW5b_oza{- z`Mdv?vs`X31bKzgU`+coK0hr7R~ zZC#@E`1%A(PqkNeFLk-L6)b;s_-e9ivCmJ)Zijn6eW00-DfdRd{YQ3lIYKO#Beb8( zargO8lzp{d_VJqQVE0d}HP_+ZmUEwJ`#iba*Tu5D?pzyl;q}@UNtm|A%lXrOm~*Ff z>ow4Fs{7n}yYqUTyd<*kTBh4=`;9u5-L<{8zu~TB+NNG-P4=j=uR`fQMY-2m*7dvg z)qeeYjFj}*k}GJ}thJkDUsqF_E|mnCg5@XY%xJ-S>@OZr@h5FXM9a3NptxoRFK7IZTSyA=c^r#kyfx83op5 zmX|GAo?nnRR@O*gwBelkwtSeRD@a*&g_wB65QzTI+ktUHK5h39z zk-~oEC2#7?&B-rF&nd9-Sa+|!%>8wHLvwSptyg|sUw-D8v6(p;nbv#vsBd(7UVf&u zeU^0G}2+x_qzHBlze!Fa4Y= z{p``Pu472ISI0@{wm$NgOY33p?B9+-%xonDk5z$EOAeXuVkV?Rv62Qd}@e9X=g0uarh{X$H#C2PQ=IY z5BLQB5ew0@^CacTI0c`=sW=U%;|!dMvv4-fLDSCDl;`1mdq zC-5Ym!qa#LO*@}bK8xq@JYK+y_!(Zp&+#&Tfu??=`Lr{u&D&c)=QNp5a()rHUsJlg zjQ)|t+sfi?=6;pxa{V+Zo%GX6lfL#fRj{!FQ9pS!na_(#cG3MaM6;DC>(35NI+vo! z%%|8T`_-#?omamN0X&F@@I%y>yQU-f z5gtW*tz{jj{4t)ulXwbG;~D$}KgF|n4$q@$=OX3L@DhHGm+=d{f>-exeu>xdD>Uu= zjq(lr8gJq^_$_{i-{ar$2mBFlp=swQ%0J^Tc-w5pPMZwv{Qz%ANpee zmc?>d9$i=g1F<3op=l?Wa%Bv`Dp(b(VRfv5HL(_kVr?|-gj0^dNUVc(F$$wG##SE~ z_8I%${%V$k|77jA{eo3d!qcyr0jf)elS=5pwCr#rbEq5s3g!XR0%9kf5< zOP*RB!kwp>DgU-VplkDLr9vSn7|J(Bm}MQL@%md9=IVVi8)Bb+XdD)$}|SN~B4|OXw!L|FcX$ z=x;WDXx^E$FcTW3CK=q57W>(!)Q|EzlNM$|<6e^%WeeOvM9kV))c1jdU6iTE? zBuGR^cuKUBaiZR9;$i^n?$w`htSi-~d*Z@OXq2k&&cwy)d(>wpE_Sm{?*-1+=)9~^ zSp`|+GG&oKCpz?7z{#hT Hndta$-Ele( literal 0 HcmV?d00001 diff --git a/reader/src/main/java/io/github/dfa1/vortex/reader/PostscriptParser.java b/reader/src/main/java/io/github/dfa1/vortex/reader/PostscriptParser.java index d959a05f7..bfb91ce3a 100644 --- a/reader/src/main/java/io/github/dfa1/vortex/reader/PostscriptParser.java +++ b/reader/src/main/java/io/github/dfa1/vortex/reader/PostscriptParser.java @@ -109,11 +109,13 @@ static ParsedFile parseBlobs(MemorySegment footerBuf, MemorySegment layoutBuf, M Footer footer = convertFooter(fbsFooter); Layout layout = convertLayout(fbsLayout, footer.layoutSpecs()); - DType dtype = null; - if (dtypeBuf != null && dtypeBuf.byteSize() > 0) { - dtype = convertDType(io.github.dfa1.vortex.core.fbs.FbsDType.getRootAsFbsDType(dtypeBuf)); - checkStructLayouts(layout, dtype); + // Rust opens a file without an embedded dtype only when the caller supplies one + // (VortexOpenOptions::with_dtype); Java has no such option, so such a file is unreadable. + if (dtypeBuf == null || dtypeBuf.byteSize() == 0) { + throw new VortexException("Vortex file doesn't embed a DType"); } + DType dtype = convertDType(io.github.dfa1.vortex.core.fbs.FbsDType.getRootAsFbsDType(dtypeBuf)); + checkStructLayouts(layout, dtype); return new ParsedFile(footer, dtype, layout); } catch (VortexException e) { diff --git a/reader/src/test/java/io/github/dfa1/vortex/reader/PostscriptParserBlobBoundsTest.java b/reader/src/test/java/io/github/dfa1/vortex/reader/PostscriptParserBlobBoundsTest.java index 9ac67f77e..1e40526e8 100644 --- a/reader/src/test/java/io/github/dfa1/vortex/reader/PostscriptParserBlobBoundsTest.java +++ b/reader/src/test/java/io/github/dfa1/vortex/reader/PostscriptParserBlobBoundsTest.java @@ -14,7 +14,6 @@ import java.lang.foreign.MemorySegment; -import static org.assertj.core.api.Assertions.assertThat; import static org.assertj.core.api.Assertions.assertThatCode; import static org.assertj.core.api.Assertions.assertThatThrownBy; @@ -135,17 +134,16 @@ void parse_dtypeBlobPastEof_throwsNamingDtype() { void parse_dtypeBlobLengthZero_skipsDtypeCheckEvenWithBadOffset() { // Given — dtype length 0 with a wildly out-of-range offset. The `length > 0` gate must // skip both the bounds check and the slice, so a zero-length dtype never trips on its - // offset and the file still parses (dtype absent). Kills the `length > 0` boundary/negate - // mutants that would otherwise run the check on an empty dtype. + // offset: the file fails as one with no dtype, not on the offset. Kills the `length > 0` + // boundary/negate mutants that would otherwise run the check on an empty dtype. Fixture f = validFile(); MemorySegment ps = buildPostscript(f.footerOff, f.footerLen, f.fileSize + 999, 0, f.layoutOff, f.layoutLen); - // When - PostscriptParser.ParsedFile result = PostscriptParser.parse(ps, f.segment, f.fileSize); - - // Then — parsed, with no dtype - assertThat(result.dtype()).isNull(); + // When / Then + assertThatThrownBy(() -> PostscriptParser.parse(ps, f.segment, f.fileSize)) + .isInstanceOf(VortexException.class) + .hasMessageContaining("doesn't embed a DType"); } // ── FlatBuffer blob builders (minimal, just enough to parse) ──────────────── diff --git a/reader/src/test/java/io/github/dfa1/vortex/reader/PostscriptParserLayoutIdTest.java b/reader/src/test/java/io/github/dfa1/vortex/reader/PostscriptParserLayoutIdTest.java index 3cb4f862e..7be398155 100644 --- a/reader/src/test/java/io/github/dfa1/vortex/reader/PostscriptParserLayoutIdTest.java +++ b/reader/src/test/java/io/github/dfa1/vortex/reader/PostscriptParserLayoutIdTest.java @@ -29,6 +29,9 @@ /// [IllegalArgumentException] that [LayoutId#parse] raises for a blank id. class PostscriptParserLayoutIdTest { + /// Every file must embed a dtype (Rust refuses one without), so layout-only cases pass a minimal one. + private static final MemorySegment I64_DTYPE = MemorySegment.ofBuffer(MalformedFiles.buildI64Dtype()); + @ParameterizedTest @ValueSource(strings = {"vortex.stats", "vortex.zoned"}) void convertLayout_zonedAlias_parsesToZonedLayoutWithPassthroughChild(String zonedSpec) { @@ -38,7 +41,7 @@ void convertLayout_zonedAlias_parsesToZonedLayoutWithPassthroughChild(String zon MemorySegment layout = zonedOverFlatLayout(); // When - Layout result = PostscriptParser.parseBlobs(footer, layout, null).layout(); + Layout result = PostscriptParser.parseBlobs(footer, layout, I64_DTYPE).layout(); // Then — both aliases resolve to a zoned layout with the same shape: isZoned() is true and // the single flat child is preserved for pass-through decoding. @@ -57,7 +60,7 @@ void convertLayout_blankLayoutId_throwsVortexException() { MemorySegment layout = flatLayout(); // When / Then - assertThatThrownBy(() -> PostscriptParser.parseBlobs(footer, layout, null)) + assertThatThrownBy(() -> PostscriptParser.parseBlobs(footer, layout, I64_DTYPE)) .isInstanceOf(VortexException.class) .hasMessageContaining("blank layout id"); } diff --git a/reader/src/test/java/io/github/dfa1/vortex/reader/PostscriptParserParseBlobsBoundsTest.java b/reader/src/test/java/io/github/dfa1/vortex/reader/PostscriptParserParseBlobsBoundsTest.java index f03a95725..0f3880677 100644 --- a/reader/src/test/java/io/github/dfa1/vortex/reader/PostscriptParserParseBlobsBoundsTest.java +++ b/reader/src/test/java/io/github/dfa1/vortex/reader/PostscriptParserParseBlobsBoundsTest.java @@ -29,6 +29,9 @@ /// legal value must parse, the first illegal value must throw a [VortexException]. class PostscriptParserParseBlobsBoundsTest { + /// Every file must embed a dtype (Rust refuses one without), so layout-only cases pass a minimal one. + private static final MemorySegment I64_DTYPE = MemorySegment.ofBuffer(MalformedFiles.buildI64Dtype()); + // ── FbsLayout encoding-index bound: encIdx < 0 || encIdx >= layoutSpecs.size() ── @Test @@ -38,7 +41,7 @@ void parseBlobs_layoutEncodingIndex_atLastSpec_parses() { MemorySegment layout = flatLayout(0); // When / Then — the top valid index must be accepted (kills `>=` relaxed to `>`) - assertThatCode(() -> PostscriptParser.parseBlobs(footer, layout, null)) + assertThatCode(() -> PostscriptParser.parseBlobs(footer, layout, I64_DTYPE)) .doesNotThrowAnyException(); } @@ -49,7 +52,7 @@ void parseBlobs_layoutEncodingIndex_equalToSize_throws() { MemorySegment layout = flatLayout(1); // When / Then - assertThatThrownBy(() -> PostscriptParser.parseBlobs(footer, layout, null)) + assertThatThrownBy(() -> PostscriptParser.parseBlobs(footer, layout, I64_DTYPE)) .isInstanceOf(VortexException.class) .hasMessageContaining("encoding index"); } @@ -64,7 +67,7 @@ void parseBlobs_layoutDepth_atLimit_parses() { MemorySegment layout = nestedLayout(63); // When / Then - assertThatCode(() -> PostscriptParser.parseBlobs(footer, layout, null)) + assertThatCode(() -> PostscriptParser.parseBlobs(footer, layout, I64_DTYPE)) .doesNotThrowAnyException(); } @@ -75,7 +78,7 @@ void parseBlobs_layoutDepth_oneOverLimit_throws() { MemorySegment layout = nestedLayout(64); // When / Then - assertThatThrownBy(() -> PostscriptParser.parseBlobs(footer, layout, null)) + assertThatThrownBy(() -> PostscriptParser.parseBlobs(footer, layout, I64_DTYPE)) .isInstanceOf(VortexException.class) .hasMessageContaining("nest deeper than 64"); } @@ -145,7 +148,7 @@ void parseBlobs_layoutMetadata_atLimit_parses() { MemorySegment layout = flatLayoutWithMetadata((int) PostscriptParser.MAX_LAYOUT_METADATA_BYTES.bytes()); // When / Then - assertThatCode(() -> PostscriptParser.parseBlobs(footer, layout, null)) + assertThatCode(() -> PostscriptParser.parseBlobs(footer, layout, I64_DTYPE)) .doesNotThrowAnyException(); } @@ -156,7 +159,7 @@ void parseBlobs_layoutMetadata_oneOverLimit_throws() { MemorySegment layout = flatLayoutWithMetadata((int) PostscriptParser.MAX_LAYOUT_METADATA_BYTES.bytes() + 1); // When / Then - assertThatThrownBy(() -> PostscriptParser.parseBlobs(footer, layout, null)) + assertThatThrownBy(() -> PostscriptParser.parseBlobs(footer, layout, I64_DTYPE)) .isInstanceOf(VortexException.class) .hasMessageContaining("metadata size"); } @@ -188,20 +191,20 @@ void parseBlobs_dtypeDepth_oneOverLimit_throws() { // ── parseBlobs dtype-presence guard: dtypeBuf != null && byteSize() > 0 ────────── - @Test - void parseBlobs_emptyDtypeBlob_yieldsNullDtype() { - // Given — a present but zero-length dtype blob. The guard treats an empty blob as "no dtype" - // (`byteSize() > 0`); relaxing `> 0` to `>= 0` would instead feed the empty buffer to - // getRootAsFbsDType and throw. Pins that an empty blob parses to a null dtype. + @org.junit.jupiter.params.ParameterizedTest + @org.junit.jupiter.params.provider.ValueSource(booleans = {true, false}) + void parseBlobs_noDtypeBlob_throws(boolean emptyBlob) { + // Given — no dtype blob, or a present but zero-length one. Rust opens such a file only when + // the caller supplies the dtype (VortexOpenOptions::with_dtype); Java has no such option, + // and a null dtype failed the scan later with a NullPointerException (a fuzzer finding). MemorySegment footer = footerWithLayoutSpecs("vortex.flat"); MemorySegment layout = flatLayout(0); - MemorySegment emptyDtype = MemorySegment.ofArray(new byte[0]); - - // When - DType result = PostscriptParser.parseBlobs(footer, layout, emptyDtype).dtype(); + MemorySegment dtype = emptyBlob ? MemorySegment.ofArray(new byte[0]) : null; - // Then - assertThat(result).isNull(); + // When / Then + assertThatThrownBy(() -> PostscriptParser.parseBlobs(footer, layout, dtype)) + .isInstanceOf(VortexException.class) + .hasMessageContaining("doesn't embed a DType"); } // ── helpers ────────────────────────────────────────────────────────────────── From 637700eb11344a1d71d4ae8b71e19a28418254be Mon Sep 17 00:00:00 2001 From: Davide Angelocola Date: Sun, 18 Oct 2026 17:28:02 +0200 Subject: [PATCH 08/13] fix(reader): one rule for child segment shape, checked where the data enters Found by the scan and filter fuzz targets: a dict column whose codes child was empty for a non-empty chunk threw a raw ArithmeticException, because the constant-child broadcast reads `i % capacity`; so did a primitive child with an empty buffer, through the Materialized* arrays' getInt, which broadcast the same way. Eighteen decoder call sites took SegmentBroadcast.capacity(seg, elemBytes) and broadcast with `% capacity`; some guarded an empty child their own way (ALP, FSST, RunEnd, RLE), the others (dict, delta, zigzag, varbin, patched, ALP-RD) did not. And a child neither full nor constant (2 values for 5 rows) was silently wrapped, repeating values, where Rust's children always have their full length. capacity(seg, elemBytes, count) now admits only the two legal shapes, at least `count` elements or exactly one (a constant child), and throws VortexException otherwise; the two-argument form is gone and the per-site guards it subsumes are deleted. The rule is enforced where the data enters, not only where it is read: DecodeContext.decodeChildSegment checks every primitive child segment against the count its decoder declares, broadcastCopy checks its source, and the eight Materialized* leaves take their element count from it in AbstractMaterializedArray, so a malformed buffer fails when the child is decoded or the array built, not when a row is read. Tests that pinned the old wrap-around now expect the error; decoder tests expecting their own later message for an empty child see the shared one; two fixtures were themselves malformed (Patched lane offsets for one chunk on a two-chunk array; FSST zero rows with no offsets, where a varbin always has n + 1). The integration suite, Rust- and Java-written files alike, is unaffected. vortex-jni rejects both files. Refs #493 Co-Authored-By: Claude Sonnet 5 Claude-Session: https://claude.ai/code/session_01WtuVGQ5h5Lk6Zn4JnkxF1K --- CHANGELOG.md | 1 + .../crash-materialized-empty-buffer | Bin 0 -> 4744 bytes .../scanMalformedFile/crash-dict-empty-codes | Bin 0 -> 3096 bytes .../MalformedFileInteropIntegrationTest.java | 8 ++- .../malformed/dict-empty-codes.vortex | Bin 0 -> 3096 bytes .../materialized-empty-buffer.vortex | Bin 0 -> 4728 bytes .../array/AbstractMaterializedArray.java | 5 +- .../reader/array/LazyFsstVarBinArray.java | 8 +-- .../reader/decode/AlpEncodingDecoder.java | 22 ++------ .../reader/decode/AlpRdEncodingDecoder.java | 2 +- .../vortex/reader/decode/DecodeContext.java | 14 +++-- .../reader/decode/DeltaEncodingDecoder.java | 4 +- .../reader/decode/DictEncodingDecoder.java | 2 +- .../reader/decode/OnPairEncodingDecoder.java | 2 +- .../reader/decode/PatchedEncodingDecoder.java | 12 ++--- .../reader/decode/RleEncodingDecoder.java | 6 +-- .../reader/decode/RunEndEncodingDecoder.java | 2 +- .../reader/decode/SegmentBroadcast.java | 29 +++++----- .../reader/decode/VarBinEncodingDecoder.java | 2 +- .../reader/decode/ZigZagEncodingDecoder.java | 2 +- .../reader/decode/AlpEncodingDecoderTest.java | 2 +- .../decode/ChunkedEncodingDecoderTest.java | 2 +- .../decode/DictEncodingDecoderTest.java | 22 ++++---- .../decode/FsstEncodingDecoderTest.java | 51 ++++++++---------- .../decode/PatchedEncodingDecoderTest.java | 8 +-- .../reader/decode/RleEncodingDecoderTest.java | 6 ++- .../reader/decode/SegmentBroadcastTest.java | 36 ++++++++++++- .../decode/SparseEncodingDecoderTest.java | 23 ++++---- 28 files changed, 148 insertions(+), 123 deletions(-) create mode 100644 fuzz/src/test/resources/io/github/dfa1/vortex/fuzz/VortexReaderFuzzTestInputs/filterMalformedFile/crash-materialized-empty-buffer create mode 100644 fuzz/src/test/resources/io/github/dfa1/vortex/fuzz/VortexReaderFuzzTestInputs/scanMalformedFile/crash-dict-empty-codes create mode 100644 integration/src/test/resources/fixtures/malformed/dict-empty-codes.vortex create mode 100644 integration/src/test/resources/fixtures/malformed/materialized-empty-buffer.vortex diff --git a/CHANGELOG.md b/CHANGELOG.md index 0ca759b63..b8b7fbee5 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -47,6 +47,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - A column whose dtype does not fit its bit-packed, delta or RLE encoding fails with `VortexException` instead of `ClassCastException`, and an empty RLE boolean chunk reads. ([#493](https://github.com/dfa1/vortex-java/issues/493)) - A struct dtype missing its field names or types, or a struct layout whose children do not match its fields, is rejected when the file opens, as Rust does, instead of failing the scan with `ArrayIndexOutOfBoundsException`. ([#493](https://github.com/dfa1/vortex-java/issues/493)) - A `vortex.zstd_buffers` array whose wrapped encoding id holds a control character fails with `VortexException` instead of `IllegalArgumentException`. ([#493](https://github.com/dfa1/vortex-java/issues/493)) +- A file whose encoded child holds fewer values than its array needs, other than the single value of a constant child, fails with `VortexException` instead of dividing by zero or silently repeating values. ([#493](https://github.com/dfa1/vortex-java/issues/493)) - SQL `WHERE` on an `I16`, `U8` or `I8` column no longer fails with a `ClassCastException`. ([#556](https://github.com/dfa1/vortex-java/pull/556)) - SQL `COUNT(DISTINCT col)` and aggregates with a `FILTER (WHERE ...)` clause return the right answer; the statistics push-down returned the row count and ignored the filter. ([#554](https://github.com/dfa1/vortex-java/pull/554)) - SQL `MIN`/`MAX` of a string column whose values fall outside Latin-1 answers instead of failing the query. ([#554](https://github.com/dfa1/vortex-java/pull/554)) diff --git a/fuzz/src/test/resources/io/github/dfa1/vortex/fuzz/VortexReaderFuzzTestInputs/filterMalformedFile/crash-materialized-empty-buffer b/fuzz/src/test/resources/io/github/dfa1/vortex/fuzz/VortexReaderFuzzTestInputs/filterMalformedFile/crash-materialized-empty-buffer new file mode 100644 index 0000000000000000000000000000000000000000..1bd68d66411b86af1ba7c5931eb44f5bca3c449a GIT binary patch literal 4744 zcma)APiRy}7@wD{$+~XSEnT)-3HDJ$mSV*}OOPTZh?JbfLl6&Q^0M#6y!K_2?!Haa zP|O~3km5l^N)H~Sh?LT!NGTqAkRnB-M-M%fQi=$cQi}a&`ulw|vpdPINhkc?%zWSX zzJK%0%-blX)G)v{2%Mfg^Nvi~>+IU=(izISGZPZ|x>1%S%vRZ(*0uv9whFdrL@#oh z{-k5Y%vU}0;v499$oLz`w~3CdQcp3L*4ABC74$%5RbFNA%wnDZ86aj`4;~+Fpua*DfPv&U1x(ekdKtBeI0%dt(AZ-V155u>Dv1#c_-Rur! zvNWekh4uUua+(iHrz~^xOF`Os@mm$X(m;R#tlKKea6qqygcmI{O^XN1EItb7& z+jQ(YCen5ShcuHZ+-k1c;8QPswe6YlgFZ0-(*XVZ&GBO#^UZ$PHKIN?$B(C6c-F$3 zN(!MfCY|y*z_ts2>pa{8VG@`CXkY!)&BK07eV__jGZp|X5c0Y{pLCON2L>T8-j?YcW8H|Ii@HLx2K6nAcOZ_f~|X9RYK;7cAJ71f9k zOH!lsnYHN)Z8z3`d%wpa`@A|4HF!96A|$C3bI91S^_BbHwwqFqC8^O6#4`Zbwerl= z3G!xhR^Y{D>=FGX*`L!OO#-xA*kpGvXx0{n za@MeCv$h$J*nUyTnI?8IrcLU!?TWFv2*Mn|njHnoYHw#1m`PSo5gcPzYNrCZe(`h9 z`%awn$p>hg=d5ibkYVkHAkgk9fWBQ8x)K}AGvjsb*zh2UbR`*=6`fSb6XV~6h`a#z|WA3&Q-AH)Wrudf=(yz2ZtC&yb%*w~ z9uM7eXb;Tzo}q;Z>+K`sAk6>a#yoEvV;et)(ySd$oU>cp+G4KSkbhm)&GU|7{6FacepK z+-%R@f3#)C+^l180Fin;Uu@i3gMLcfNCfM3tGi9_RnKD%w$ju8o+@Eq*m$k=0&WU6 z&-3xj1Ge?QZ%WGV=-}$H6+1c@pAT3LhSIjXQ}mhPd3zzvz3ckH`tH34N{HXaVy$)k z=%1k1N;QByTA=#4G4j8 zzy~tG?OwlDnrWyW7&^bTjlV1L`IX>2=8p{Tr7mwNXw1a*r8xQkv>d+IsM~p0^FiC! z%WI~MR*mQJ@jkr8pwW!LJ7;+KoUr~hqxV};y%wpJ-uD`=zSNuHyRJT=x! zpKXGE($KlbY`w(GQ8?9%SE4C-RW*!^<9oD>M1I_>GmMep&kX(5uFep*+kB8r&3u?f zQ^IIMNN0V{2F)}H>QVDR zBVj>&+m|=H&;XC$A~vq>dtsEM0l54Y>E6RHZ_qsxr;9<2L1IhF*?>;{fbTqP@y@q& z9;cx`MGA6`aZj8Ac%JB!Du**JkN1ZqCXfdw;Lt_;gBXNI@R9U36lXg>wsX+D@x6l( ZddN#iS`I#JApp6o2d6J8v#$b1@Nefh-GID1~h~#sxwMSqM>4P(b8F5%&7L!K<8ikG;JM zP@pSQr$7`Gl!!lof+A%^K}CTmC=g{7Mf?NF@%zok?&#K5DmP@LxAW$`dEaklR=la2 zUpF5qu$n(34x=V^3pYzCQS;>}1@-@tf-jt0xk)^R9E^PtnE?#bERXK4_VaAe$+2$! zGp_J@Vb+hpJ8O6+9J%PuAWAxsT&~JxgCt7!u=keXwM)5P79YfUJfvEAVE9dk-$_Lt z8u}@R9%e9@uMWd34ih;1qG~rydSRSF=8fTfS;{AaZdd-U${d9KpCkI({8z{C_;L{T zM9vtx6MHxA=fgPqMJ^d$X{R^=$6UtZ2z^F}sT1bW?KHc$@@;{w7udN1+bFO#lxg(~Osv;> zbf!Amo-5S$+OCqdwzqh*?NOCmrXR6pm^977FgUD}c5UaQ=$g+m+G6N3iqwlT?i;t6 z_Q!zh0NOsUQdun<%Rs)b_zi_q1+p*m6Es>K{fC}w6Ebzk>`c&T@1>&ERSvl3pwA+LdQj8N8;uaROU^RC&eA_`&da5t!JVSDY7-uX0VFofuWl zm|vdVX?)FjowhW4?NTpg**iQxob7rDJU~_*`8wrU-2hZmZqN2h3m0uebr`L3OsTVc zjS9`6e%froS_4>l4g0F^df>VHzsK4OMw@dB%{deQZaPTzqP^{d@a|l&2|mYR{5Wy= zC!1dxdH#)Tcc276GOCZ4G1JB0k!$)*A2~P^^p#JJjmuN&Snq?rXx4$s8!L#r2m7|PLHN2Q-`&`{`U%Da BZyW#s literal 0 HcmV?d00001 diff --git a/integration/src/test/java/io/github/dfa1/vortex/integration/MalformedFileInteropIntegrationTest.java b/integration/src/test/java/io/github/dfa1/vortex/integration/MalformedFileInteropIntegrationTest.java index de54771d2..f301702c3 100644 --- a/integration/src/test/java/io/github/dfa1/vortex/integration/MalformedFileInteropIntegrationTest.java +++ b/integration/src/test/java/io/github/dfa1/vortex/integration/MalformedFileInteropIntegrationTest.java @@ -47,7 +47,13 @@ class MalformedFileInteropIntegrationTest { "zstd-buffers-inner-id-control-char", // A file with no dtype blob, which Rust opens only when the caller supplies the dtype: // vortex-java opened it with a null dtype and the scan threw NullPointerException. - "no-dtype"}) + "no-dtype", + // A dict column whose codes child is empty for a non-empty chunk: the broadcast's + // `i % capacity` divided by zero (ArithmeticException). + "dict-empty-codes", + // A primitive child whose buffer is empty for a non-empty array: the Materialized* + // arrays broadcast with `i % elementCount` and divided by zero on the first read. + "materialized-empty-buffer"}) void bothReadersReject(String name) throws URISyntaxException { // Given Path file = Path.of(Objects.requireNonNull( diff --git a/integration/src/test/resources/fixtures/malformed/dict-empty-codes.vortex b/integration/src/test/resources/fixtures/malformed/dict-empty-codes.vortex new file mode 100644 index 0000000000000000000000000000000000000000..7b1597c1059b12ba2f63424fbfef8db5160dbccc GIT binary patch literal 3096 zcmd^By>App6o2d6J8v#$b1@Nefh-GID1~h~#sxwMSqM>4P(b8F5%&7L!K<8ikG;JM zP@pSQr$7`Gl!!lof+A%^K}CTmC=g{7Mf?NF@%zok?&#K5DmP@LxAW$`dEaklR=la2 zUpF5qu$n(34x=V^3pYzCQS;>}1@-@tf-jt0xk)^R9E^PtnE?#bERXK4_VaAe$+2$! zGp_J@Vb+hpJ8O6+9J%PuAWAxsT&~JxgCt7!u=keXwM)5P79YfUJfvEAVE9dk-$_Lt z8u}@R9%e9@uMWd34ih;1qG~rydSRSF=8fTfS;{AaZdd-U${d9KpCkI({8z{C_;L{T zM9vtx6MHxA=fgPqMJ^d$X{R^=$6UtZ2z^F}sT1bW?KHc$@@;{w7udN1+bFO#lxg(~Osv;> zbf!Amo-5S$+OCqdwzqh*?NOCmrXR6pm^977FgUD}c5UaQ=$g+m+G6N3iqwlT?i;t6 z_Q!zh0NOsUQdun<%Rs)b_zi_q1+p*m6Es>K{fC}w6Ebzk>`c&T@1>&ERSvl3pwA+LdQj8N8;uaROU^RC&eA_`&da5t!JVSDY7-uX0VFofuWl zm|vdVX?)FjowhW4?NTpg**iQxob7rDJU~_*`8wrU-2hZmZqN2h3m0uebr`L3OsTVc zjS9`6e%froS_4>l4g0F^df>VHzsK4OMw@dB%{deQZaPTzqP^{d@a|l&2|mYR{5Wy= zC!1dxdH#)Tcc276GOCZ4G1JB0k!$)*A2~P^^p#JJjmuN&Snq?rXx4$s8!L#r2m7|PLHN2Q-`&`{`U%Da BZyW#s literal 0 HcmV?d00001 diff --git a/integration/src/test/resources/fixtures/malformed/materialized-empty-buffer.vortex b/integration/src/test/resources/fixtures/malformed/materialized-empty-buffer.vortex new file mode 100644 index 0000000000000000000000000000000000000000..a09ff0bcd65f62f8143612a135a0e5dd8a9f99e7 GIT binary patch literal 4728 zcma)APiRy}7@wD{$+~XSEnT*03HDJ$mSQF95~N58A|)sB5P}CWdD-`3Ui-31ci*OI zC}s~iNbw*dr3VjEL`vyVq!bT5NRcAaqlcbKks^XpO0oY;f4^^Lb|=|2`i0+{neUtT z{hM!Q-ezj@`0&AOqx$tRJn1zX7y;hNo1&8kT;a z%*#Ay%&GLT9~!9F&j0f-=I9M|3CfRwe*_o-hUJNl&<@xhgl`37)AE(N*6B)b7oN9} zfKlKkf;a-)LI4xMS8)CzoG1bpFpn`V|4sWP^(p!(FmL!D{3mPY(P#QK3eYc0I(8is zc{_mvTF4Y`H&!jksh7T5dTRWj56u4zK>vPs{20f0s~>iasE_3M@q`P{T6jlEA+-DC zbAAr6boL*ehudIG0uuo3tA9Ir*oUDH6aj1RCG<(Gi{UBLLV=9&;5;UTE%n_OI|J$z z`mUY%XV}3sA#0C%eLUC}z?2q!${D*$Q`j6_+sj~H0?q+GzLlUArB-us9@-8o;>ZkJ1jTn!j?6#8xtINY;!;8Do{hPI{;tu@TiCpsU$T@pIMte zK)**{!0&O)d{&)M3(2Vy8B3j*L&o;4uiW>Ru1P(Xq((z9PXb)m@>5eM$eYbs(Y}P% z=8fw$?CIJc#ygrk*$cyVxWUuLo?}|cZqZ+o{W$~HBtW}`Eq3>SXKkS?XAN7Dwas|M z_RC7nG_i|5ZBnPDi^k>>81n#Yb_5t!d)lkOOtN~4kQlpCJ7vu46+id9@5D);e1Nuj z&RQD69M(Vx2JM~z=-UP1E3v^mGhWw@Jr5Y8*m+IID1F4}+GF3+Sz|8+!=67n)^a;m zcc-|qy61w`XKRK&!}bo-_Ph2ObBBhIzwSM-f(guj2xx!yJ=Y)BKJ((%fty?Q{f$1! zw~tzLeS&Bc`PiuR{U=>#gUqjcS5^=Nj=fps7OCV>jT7&xEnfu^7R zcXdBkad^jdf9HhZX}ei-58@>?;Dye7y?!dU;_xmS9p@e1HQgV);_$xH{ekNa@2=_J zc6h(&eqR3a44q_;4i+6Mj7S}vBV@~0?X~s=EJ}@S= z&^+wc&X9Z~=~y#zHq)N1CtJhh=^SJv2iETjCjN#3y9%5FGQhWZDFhzl4c5sF!1+H) zjd|WU`Zj)ag{&P8oU>cp+Gea;nE#fno97)t{{{78=Ci*Y-ZOrdF^BbB09^kNV$W}Z zuZ6)Q&-3xj1D1N< zHznn_b#V1a#kLN{=L1%PuF&)D6n$oR-X6^6-gW(8eRtmjCB$!IvDP|%^wPt9*X!)& zifhWjZ~l_W%l9pF%uP~+16Uf_XS?yONq?A|<;8j$t?Zvmf>gbU@f{er3akSmz~2pJ zAP+pm0^b3?0oH*KI1G#cJUH&bz)j!^&;n+FV?Y_m0Ka0|E#NA!4urrs-~$=pMz>$9 zjWkqu4WHlI#^2TW%xZ83;|E6eN=LRF)Mw+`avXgOUJl=D)a|^h<>2k@mNn9Pv&!@M zXb;&^P;W$#oi;LjPFQ~$(MQdwR*lqZ_j~nbEvhlr&y9>LZQ@v}#|v>9FH^0$VdVT> zV&&CErS2NOvtB&z)%|W;3qj+K*l?P zzVbJLv2Etzx0I2w_N^>PqcmQK?CUCrGiys8s*ANo8q`v?*T@{*)%j*k5_q)xJ^D4f zxR|KZM&{&C_{)aReQ*2;lBIfxq2jj){c!RFnNqwH$Y*`d1&uTbYEffDmb@%FcL9;cx`MGA6` zaZj88c%JB!Du**JkM{>9Cb0V_;LthygBXkl@R9r$6ldE%wsX+D`Mpt0y2ndMCWZ4e|D65@`KbdJ literal 0 HcmV?d00001 diff --git a/reader/src/main/java/io/github/dfa1/vortex/reader/array/AbstractMaterializedArray.java b/reader/src/main/java/io/github/dfa1/vortex/reader/array/AbstractMaterializedArray.java index 83f34659c..199cded69 100644 --- a/reader/src/main/java/io/github/dfa1/vortex/reader/array/AbstractMaterializedArray.java +++ b/reader/src/main/java/io/github/dfa1/vortex/reader/array/AbstractMaterializedArray.java @@ -1,6 +1,7 @@ package io.github.dfa1.vortex.reader.array; import io.github.dfa1.vortex.core.model.DType; +import io.github.dfa1.vortex.reader.decode.SegmentBroadcast; import java.lang.foreign.MemorySegment; import java.lang.foreign.SegmentAllocator; @@ -32,8 +33,10 @@ abstract class AbstractMaterializedArray { this.dtype = dtype; this.length = length; this.buffer = buffer; + // Only a full buffer or a single broadcast element: an empty one for a non-empty array made + // the leaves' `i % elementCount` divide by zero (a fuzzer finding), so it is refused here. this.elementCount = switch (dtype) { - case DType.Primitive p -> buffer.byteSize() / p.ptype().byteSize(); + case DType.Primitive p -> SegmentBroadcast.capacity(buffer, p.ptype().byteSize(), length); default -> length; }; } diff --git a/reader/src/main/java/io/github/dfa1/vortex/reader/array/LazyFsstVarBinArray.java b/reader/src/main/java/io/github/dfa1/vortex/reader/array/LazyFsstVarBinArray.java index 79bde5b4f..898e899b6 100644 --- a/reader/src/main/java/io/github/dfa1/vortex/reader/array/LazyFsstVarBinArray.java +++ b/reader/src/main/java/io/github/dfa1/vortex/reader/array/LazyFsstVarBinArray.java @@ -142,13 +142,7 @@ public int getByteLength(long i) { @Override public void forEachByteLength(IntConsumer c) { long n = length; - long cap = SegmentBroadcast.capacity(uncompressedLengths, uncompressedLengthsPType.byteSize()); - if (cap == 0) { - if (n == 0) { - return; - } - throw new VortexException(EncodingId.VORTEX_FSST, "empty uncompressed-lengths child"); - } + long cap = SegmentBroadcast.capacity(uncompressedLengths, uncompressedLengthsPType.byteSize(), n); if (cap >= n) { forEachClaimedLength(c, n); return; diff --git a/reader/src/main/java/io/github/dfa1/vortex/reader/decode/AlpEncodingDecoder.java b/reader/src/main/java/io/github/dfa1/vortex/reader/decode/AlpEncodingDecoder.java index 29a64495a..52e1318bc 100644 --- a/reader/src/main/java/io/github/dfa1/vortex/reader/decode/AlpEncodingDecoder.java +++ b/reader/src/main/java/io/github/dfa1/vortex/reader/decode/AlpEncodingDecoder.java @@ -83,8 +83,7 @@ private static Array decodeF64(DecodeContext ctx, ProtoALPMetadata meta, int exp // the encoder's verify step. double df = F10_F64[expF]; double de = IF10_F64[expE]; - long srcCap = SegmentBroadcast.capacity(src, 8); - checkSource(srcCap, n); + long srcCap = SegmentBroadcast.capacity(src, 8, n); if (meta.patches() == null) { if (srcCap >= n) { @@ -115,8 +114,7 @@ private static Array decodeF32(DecodeContext ctx, ProtoALPMetadata meta, int exp checkExponents(expE, expF, F10_F32.length); float df = F10_F32[expF]; float de = IF10_F32[expE]; - long srcCap = SegmentBroadcast.capacity(src, 4); - checkSource(srcCap, n); + long srcCap = SegmentBroadcast.capacity(src, 4, n); if (meta.patches() == null) { if (srcCap >= n) { @@ -167,13 +165,6 @@ private static void checkExponents(int expE, int expF, int maxExp) { /// /// @param srcCap number of elements physically present in the encoded child /// @param n logical row count - private static void checkSource(long srcCap, long n) { - if (srcCap == 0 && n > 0) { - throw new VortexException(EncodingId.VORTEX_ALP, - "empty encoded child for " + n + " rows"); - } - } - private static void applyPatches(DecodeContext ctx, ProtoPatchesMetadata pm, MemorySegment out, int elemBytes) { long numPatches = pm.len(); if (numPatches == 0) { @@ -188,13 +179,8 @@ private static void applyPatches(DecodeContext ctx, ProtoPatchesMetadata pm, Mem MemorySegment idxSeg = ctx.decodeChildSegment(1, new DType.Primitive(idxPtype, false), numPatches); MemorySegment valSeg = ctx.decodeChildSegment(2, ctx.dtype(), numPatches); - long idxCap = SegmentBroadcast.capacity(idxSeg, idxBytes); - long valCap = SegmentBroadcast.capacity(valSeg, elemBytes); - if (idxCap == 0 || valCap == 0) { - throw new VortexException(EncodingId.VORTEX_ALP, - "empty patch child for " + numPatches + " declared patches (indices=" - + idxSeg.byteSize() + " bytes, values=" + valSeg.byteSize() + " bytes)"); - } + long idxCap = SegmentBroadcast.capacity(idxSeg, idxBytes, numPatches); + long valCap = SegmentBroadcast.capacity(valSeg, elemBytes, numPatches); if (idxCap >= numPatches && valCap >= numPatches) { for (long i = 0; i < numPatches; i++) { long absIdx = PrimitiveArrays.readLong(idxSeg, i * idxBytes, idxPtype, EncodingId.VORTEX_ALP) - offset; diff --git a/reader/src/main/java/io/github/dfa1/vortex/reader/decode/AlpRdEncodingDecoder.java b/reader/src/main/java/io/github/dfa1/vortex/reader/decode/AlpRdEncodingDecoder.java index 06939c43b..26e508ad8 100644 --- a/reader/src/main/java/io/github/dfa1/vortex/reader/decode/AlpRdEncodingDecoder.java +++ b/reader/src/main/java/io/github/dfa1/vortex/reader/decode/AlpRdEncodingDecoder.java @@ -101,7 +101,7 @@ private static Patches decodePatches(DecodeContext ctx, ProtoPatchesMetadata pm) // Pull the small left-values table into a short[] so lookups don't pay an // Array-dispatch per patch hit. Patches are typically <1% of rows. MemorySegment valSeg = ctx.decodeChildSegment(3, DType.U16, numPatches); - long valCap = SegmentBroadcast.capacity(valSeg, 2); + long valCap = SegmentBroadcast.capacity(valSeg, 2, numPatches); short[] leftValues = new short[(int) numPatches]; for (int j = 0; j < numPatches; j++) { leftValues[j] = valSeg.getAtIndex(VortexFormat.LE_SHORT, j % valCap); diff --git a/reader/src/main/java/io/github/dfa1/vortex/reader/decode/DecodeContext.java b/reader/src/main/java/io/github/dfa1/vortex/reader/decode/DecodeContext.java index 3e437f655..c69176a86 100644 --- a/reader/src/main/java/io/github/dfa1/vortex/reader/decode/DecodeContext.java +++ b/reader/src/main/java/io/github/dfa1/vortex/reader/decode/DecodeContext.java @@ -59,19 +59,27 @@ public Array decodeChild(int i, DType dtype, long rowCount) { /// @param i zero-based child index within this node's children array /// @return the primary [MemorySegment] of the decoded child public MemorySegment decodeChildSegment(int i) { - var childCtx = new DecodeContext(child(i), dtype, rowCount, segmentBuffers, registry, arena); - return registry.decodeAsSegment(childCtx); + return decodeChildSegment(i, dtype, rowCount); } /// Recursively decode child `i` with an explicit dtype and row count, returning its primary segment. /// + /// A primitive child's segment has one of the two legal shapes, checked here once for every + /// decoder ([SegmentBroadcast#capacity(MemorySegment, int, long)]): at least `rowCount` + /// elements, or exactly one (a constant child broadcast across every row). + /// /// @param i zero-based child index within this node's children array /// @param dtype logical type to assign to the child context /// @param rowCount number of logical rows for the child /// @return the primary [MemorySegment] of the decoded child + /// @throws io.github.dfa1.vortex.core.error.VortexException if the segment has neither shape public MemorySegment decodeChildSegment(int i, DType dtype, long rowCount) { var childCtx = new DecodeContext(child(i), dtype, rowCount, segmentBuffers, registry, arena); - return registry.decodeAsSegment(childCtx); + MemorySegment segment = registry.decodeAsSegment(childCtx); + if (dtype instanceof DType.Primitive p) { + SegmentBroadcast.capacity(segment, p.ptype().byteSize(), rowCount); + } + return segment; } /// Returns buffer `bufferPosition` of child `i` without decoding that child. diff --git a/reader/src/main/java/io/github/dfa1/vortex/reader/decode/DeltaEncodingDecoder.java b/reader/src/main/java/io/github/dfa1/vortex/reader/decode/DeltaEncodingDecoder.java index fb0e0e490..e5433150f 100644 --- a/reader/src/main/java/io/github/dfa1/vortex/reader/decode/DeltaEncodingDecoder.java +++ b/reader/src/main/java/io/github/dfa1/vortex/reader/decode/DeltaEncodingDecoder.java @@ -90,8 +90,8 @@ public Array decode(DecodeContext ctx) { MemorySegment basesSeg = ctx.decodeChildSegment(0, dtype, basesLen); MemorySegment deltasSeg = ctx.decodeChildSegment(1, dtype, deltasLen); int elemBytes = ptype.byteSize(); - long basesCap = SegmentBroadcast.capacity(basesSeg, elemBytes); - long deltasCap = SegmentBroadcast.capacity(deltasSeg, elemBytes); + long basesCap = SegmentBroadcast.capacity(basesSeg, elemBytes, basesLen); + long deltasCap = SegmentBroadcast.capacity(deltasSeg, elemBytes, deltasLen); // The only row-scaled allocation: the output itself, off-heap and at the column's own // width. Everything below is fixed-size scratch — one chunk's worth, cache-resident, diff --git a/reader/src/main/java/io/github/dfa1/vortex/reader/decode/DictEncodingDecoder.java b/reader/src/main/java/io/github/dfa1/vortex/reader/decode/DictEncodingDecoder.java index d6f4b85ff..aa40b32ff 100644 --- a/reader/src/main/java/io/github/dfa1/vortex/reader/decode/DictEncodingDecoder.java +++ b/reader/src/main/java/io/github/dfa1/vortex/reader/decode/DictEncodingDecoder.java @@ -306,7 +306,7 @@ private static BoolArray rowValidity(DecodeContext ctx, MemorySegment codesBuf, return codesValidity; } MemorySegment bits = ctx.arena().allocate((rowCount + 7) >>> 3); - long codesCap = SegmentBroadcast.capacity(codesBuf, codePType.byteSize()); + long codesCap = SegmentBroadcast.capacity(codesBuf, codePType.byteSize(), rowCount); if (codesCap >= rowCount) { for (long i = 0; i < rowCount; i++) { boolean valid = (codesValidity == null || codesValidity.getBoolean(i)) diff --git a/reader/src/main/java/io/github/dfa1/vortex/reader/decode/OnPairEncodingDecoder.java b/reader/src/main/java/io/github/dfa1/vortex/reader/decode/OnPairEncodingDecoder.java index 3a34ac418..79e7b2fe2 100644 --- a/reader/src/main/java/io/github/dfa1/vortex/reader/decode/OnPairEncodingDecoder.java +++ b/reader/src/main/java/io/github/dfa1/vortex/reader/decode/OnPairEncodingDecoder.java @@ -190,7 +190,7 @@ private static long[] childLongs(DecodeContext ctx, int i, ProtoPType wirePtype, PType ptype = PType.fromOrdinal(wirePtype.value()); MemorySegment seg = ctx.decodeChildSegment(i, new DType.Primitive(ptype, false), count); int width = ptype.byteSize(); - if (n > 0 && SegmentBroadcast.capacity(seg, width) < n) { + if (SegmentBroadcast.capacity(seg, width, n) < n) { MemorySegment expanded = ctx.arena().allocate((long) n * width, width); SegmentBroadcast.broadcastCopy(seg, expanded, n, width); seg = expanded; diff --git a/reader/src/main/java/io/github/dfa1/vortex/reader/decode/PatchedEncodingDecoder.java b/reader/src/main/java/io/github/dfa1/vortex/reader/decode/PatchedEncodingDecoder.java index 4e2c57fbb..8c262cc7e 100644 --- a/reader/src/main/java/io/github/dfa1/vortex/reader/decode/PatchedEncodingDecoder.java +++ b/reader/src/main/java/io/github/dfa1/vortex/reader/decode/PatchedEncodingDecoder.java @@ -95,25 +95,25 @@ public Array decode(DecodeContext ctx) { private static MemorySegment patchedOutput(DecodeContext ctx, MemorySegment innerSeg, long n, int elemBytes, long nPatches, long nChunks, long nLanes, long offset, MemorySegment laneOffsetsSeg, MemorySegment patchIndicesSeg, MemorySegment patchValuesSeg) { - if (nPatches == 0 && SegmentBroadcast.capacity(innerSeg, elemBytes) >= n) { + if (nPatches == 0 && SegmentBroadcast.capacity(innerSeg, elemBytes, n) >= n) { return innerSeg.asSlice(0, n * elemBytes).asReadOnly(); } MemorySegment out = ctx.arena().allocate(n * elemBytes); SegmentBroadcast.broadcastCopy(innerSeg, out, n, elemBytes); if (nPatches > 0) { - applyPatches(out, n, nChunks, nLanes, offset, elemBytes, + applyPatches(out, n, nPatches, nChunks, nLanes, offset, elemBytes, laneOffsetsSeg, patchIndicesSeg, patchValuesSeg); } return out; } private static void applyPatches( - MemorySegment out, long n, long nChunks, long nLanes, long offset, int elemBytes, + MemorySegment out, long n, long nPatches, long nChunks, long nLanes, long offset, int elemBytes, MemorySegment laneOffsets, MemorySegment patchIndices, MemorySegment patchValues ) { - long laneCap = SegmentBroadcast.capacity(laneOffsets, 4); - long idxCap = SegmentBroadcast.capacity(patchIndices, 2); - long valCap = SegmentBroadcast.capacity(patchValues, elemBytes); + long laneCap = SegmentBroadcast.capacity(laneOffsets, 4, nChunks * nLanes + 1); + long idxCap = SegmentBroadcast.capacity(patchIndices, 2, nPatches); + long valCap = SegmentBroadcast.capacity(patchValues, elemBytes, nPatches); for (long chunk = 0; chunk < nChunks; chunk++) { long start = Integer.toUnsignedLong( laneOffsets.getAtIndex(VortexFormat.LE_INT, (chunk * nLanes) % laneCap)); diff --git a/reader/src/main/java/io/github/dfa1/vortex/reader/decode/RleEncodingDecoder.java b/reader/src/main/java/io/github/dfa1/vortex/reader/decode/RleEncodingDecoder.java index 3145b7ff8..d48a3df1d 100644 --- a/reader/src/main/java/io/github/dfa1/vortex/reader/decode/RleEncodingDecoder.java +++ b/reader/src/main/java/io/github/dfa1/vortex/reader/decode/RleEncodingDecoder.java @@ -194,11 +194,7 @@ static MemorySegment fitElements(SegmentAllocator arena, MemorySegment buf, long if (buf.byteSize() >= bytes) { return IoBounds.slice(buf, 0, bytes); } - long capacity = SegmentBroadcast.capacity(buf, elemBytes); - if (capacity != 1) { - throw new VortexException(EncodingId.FASTLANES_RLE, - "child holds " + capacity + " element(s) for a declared length of " + count); - } + // Shorter than `count`: broadcastCopy admits only the single element of a constant child. MemorySegment out = arena.allocate(bytes, elemBytes); SegmentBroadcast.broadcastCopy(buf, out, count, elemBytes); return out; diff --git a/reader/src/main/java/io/github/dfa1/vortex/reader/decode/RunEndEncodingDecoder.java b/reader/src/main/java/io/github/dfa1/vortex/reader/decode/RunEndEncodingDecoder.java index 7557dd664..d3e9c13b9 100644 --- a/reader/src/main/java/io/github/dfa1/vortex/reader/decode/RunEndEncodingDecoder.java +++ b/reader/src/main/java/io/github/dfa1/vortex/reader/decode/RunEndEncodingDecoder.java @@ -146,7 +146,7 @@ private static Array withRunValidity(Array result, BoolArray valuesValidity, Arr private static void validateEnds(MemorySegment endsSeg, PType endsPtype, long numRuns, long offset, long n) { PrimitiveArrays.requireUnsigned(endsPtype, EncodingId.VORTEX_RUNEND); int endsBytes = endsPtype.byteSize(); - long endsCap = SegmentBroadcast.capacity(endsSeg, endsBytes); + long endsCap = SegmentBroadcast.capacity(endsSeg, endsBytes, numRuns); if (endsCap <= 0) { throw new VortexException(EncodingId.VORTEX_RUNEND, "runend: empty ends buffer for " + numRuns + " run(s)"); diff --git a/reader/src/main/java/io/github/dfa1/vortex/reader/decode/SegmentBroadcast.java b/reader/src/main/java/io/github/dfa1/vortex/reader/decode/SegmentBroadcast.java index c7e040bb8..565341d97 100644 --- a/reader/src/main/java/io/github/dfa1/vortex/reader/decode/SegmentBroadcast.java +++ b/reader/src/main/java/io/github/dfa1/vortex/reader/decode/SegmentBroadcast.java @@ -38,16 +38,25 @@ public static long elementOffset(MemorySegment seg, long i, int elemBytes) { return (i % cap) * elemBytes; } - /// Returns the number of elements physically present in `seg` (≥ 1 for non-empty). + /// Returns the number of elements physically present in `seg`, a child declared `count` + /// elements long, after checking it has one of the two legal shapes: at least `count` + /// elements, or exactly one (a constant child, broadcast across every row). /// - /// Used by callers that want to detect the constant case explicitly — e.g. to hoist - /// a single read out of a hot loop. + /// Any other shape is malformed input: an empty child for a non-empty array made the + /// callers' `i % capacity` broadcast divide by zero, and a truncated one silently repeated + /// its values. Checked here, once, for every caller. /// /// @param seg segment returned by DecodeContext#decodeChildSegment /// @param elemBytes element width in bytes (>= 1) - /// @return physical element count - public static long capacity(MemorySegment seg, int elemBytes) { - return seg.byteSize() / elemBytes; + /// @param count the child's declared element count + /// @return physical element count: `>= count`, or 1 + /// @throws VortexException if `seg` holds neither `count` elements nor exactly one + public static long capacity(MemorySegment seg, int elemBytes, long count) { + long capacity = seg.byteSize() / elemBytes; + if (capacity >= count || capacity == 1) { + return capacity; + } + throw new VortexException("child holds " + capacity + " element(s) for a declared length of " + count); } /// Copies `count` elements from `src` to `dst`, broadcasting the lone @@ -59,13 +68,9 @@ public static long capacity(MemorySegment seg, int elemBytes) { /// @param dst destination segment (must hold ≥ `count * elemBytes` bytes) /// @param count number of elements to materialize /// @param elemBytes element width in bytes (>= 1) - /// @throws VortexException if `src` is empty + /// @throws VortexException if `src` holds neither `count` elements nor exactly one public static void broadcastCopy(MemorySegment src, MemorySegment dst, long count, int elemBytes) { - long cap = src.byteSize() / elemBytes; - if (cap == 0) { - throw new VortexException("source segment is empty (byteSize=" + src.byteSize() - + ", elemBytes=" + elemBytes + ")"); - } + long cap = capacity(src, elemBytes, count); if (cap >= count) { MemorySegment.copy(src, 0, dst, 0, count * elemBytes); return; diff --git a/reader/src/main/java/io/github/dfa1/vortex/reader/decode/VarBinEncodingDecoder.java b/reader/src/main/java/io/github/dfa1/vortex/reader/decode/VarBinEncodingDecoder.java index b9ce5e477..cb65b5692 100644 --- a/reader/src/main/java/io/github/dfa1/vortex/reader/decode/VarBinEncodingDecoder.java +++ b/reader/src/main/java/io/github/dfa1/vortex/reader/decode/VarBinEncodingDecoder.java @@ -40,7 +40,7 @@ public Array decode(DecodeContext ctx) { MemorySegment offsets = ctx.decodeChildSegment(0, offsetsDtype, n + 1); int offBytes = offsetsPtype.byteSize(); - long offCap = SegmentBroadcast.capacity(offsets, offBytes); + long offCap = SegmentBroadcast.capacity(offsets, offBytes, n + 1); if (offCap < n + 1) { MemorySegment materialized = ctx.arena().allocate((n + 1) * offBytes, offBytes); SegmentBroadcast.broadcastCopy(offsets, materialized, n + 1, offBytes); diff --git a/reader/src/main/java/io/github/dfa1/vortex/reader/decode/ZigZagEncodingDecoder.java b/reader/src/main/java/io/github/dfa1/vortex/reader/decode/ZigZagEncodingDecoder.java index 177441544..d415b1687 100644 --- a/reader/src/main/java/io/github/dfa1/vortex/reader/decode/ZigZagEncodingDecoder.java +++ b/reader/src/main/java/io/github/dfa1/vortex/reader/decode/ZigZagEncodingDecoder.java @@ -43,7 +43,7 @@ public Array decode(DecodeContext ctx) { MaskedArray.Unwrapped unwrapped = MaskedArray.unwrap(encoded); MemorySegment src = ctx.materialize(unwrapped.inner()); int elemBytes = signed.byteSize(); - long srcCap = SegmentBroadcast.capacity(src, elemBytes); + long srcCap = SegmentBroadcast.capacity(src, elemBytes, n); Array result = decodeUnwrapped(ctx, signed, n, src, srcCap); return MaskedArray.wrapIfPresent(result, unwrapped.validity()); diff --git a/reader/src/test/java/io/github/dfa1/vortex/reader/decode/AlpEncodingDecoderTest.java b/reader/src/test/java/io/github/dfa1/vortex/reader/decode/AlpEncodingDecoderTest.java index 31725e190..179633a4b 100644 --- a/reader/src/test/java/io/github/dfa1/vortex/reader/decode/AlpEncodingDecoderTest.java +++ b/reader/src/test/java/io/github/dfa1/vortex/reader/decode/AlpEncodingDecoderTest.java @@ -273,7 +273,7 @@ void emptyPatchIndexChild_throws() { // When / Then assertThatThrownBy(() -> SUT.decode(ctx)) .isInstanceOf(VortexException.class) - .hasMessageContaining("empty patch child"); + .hasMessageContaining("child holds 0 element(s)"); } private DecodeContext ctx(DType dtype, ProtoALPMetadata meta, MemorySegment encoded, long rowCount) { diff --git a/reader/src/test/java/io/github/dfa1/vortex/reader/decode/ChunkedEncodingDecoderTest.java b/reader/src/test/java/io/github/dfa1/vortex/reader/decode/ChunkedEncodingDecoderTest.java index 6d8d3615c..d47b97ae7 100644 --- a/reader/src/test/java/io/github/dfa1/vortex/reader/decode/ChunkedEncodingDecoderTest.java +++ b/reader/src/test/java/io/github/dfa1/vortex/reader/decode/ChunkedEncodingDecoderTest.java @@ -102,7 +102,7 @@ void emptyOffsetsChild_throws() { // When / Then assertThatThrownBy(() -> decode(DType.I64, 3, segs, offsetsChild, chunkChild)) .isInstanceOf(VortexException.class) - .hasMessageContaining("decoded child segment is empty"); + .hasMessageContaining("child holds 0 element(s)"); } /// `IndexOutOfBoundsException` — a decreasing offsets pair yields a negative chunk length. diff --git a/reader/src/test/java/io/github/dfa1/vortex/reader/decode/DictEncodingDecoderTest.java b/reader/src/test/java/io/github/dfa1/vortex/reader/decode/DictEncodingDecoderTest.java index dae08734f..09dfd2384 100644 --- a/reader/src/test/java/io/github/dfa1/vortex/reader/decode/DictEncodingDecoderTest.java +++ b/reader/src/test/java/io/github/dfa1/vortex/reader/decode/DictEncodingDecoderTest.java @@ -289,20 +289,16 @@ void valuesBufferShorterThanPool_repeatsTheSingleEntry(PType codePType) { @ParameterizedTest(name = "codes={0}") @EnumSource(value = PType.class, names = {"U8", "U16", "U32"}) - void codesBufferShorterThanRowCount_repeatsTheCodes(PType codePType) { - // Given — 4 rows declared, 2 codes on the wire + void codesBufferShorterThanRowCount_throws(PType codePType) { + // Given — 4 rows declared, 2 codes on the wire: neither every code nor one broadcast + // code. It used to wrap around and repeat the pair; Rust has no such shape. MemorySegment values = TestSegments.leInts(10, 20); MemorySegment codes = codeSegment(codePType, new long[]{1, 0}); - // When - Array result = decodeProtoSegments(DType.I32, codePType, codes, values, 2, 4); - - // Then — the code pair repeats across the row count - IntArray ints = (IntArray) result; - assertThat(ints.getInt(0)).isEqualTo(20); - assertThat(ints.getInt(1)).isEqualTo(10); - assertThat(ints.getInt(2)).isEqualTo(20); - assertThat(ints.getInt(3)).isEqualTo(10); + // When / Then + assertThatThrownBy(() -> decodeProtoSegments(DType.I32, codePType, codes, values, 2, 4)) + .isInstanceOf(VortexException.class) + .hasMessageContaining("child holds 2 element(s) for a declared length of 4"); } } @@ -731,7 +727,7 @@ void emptyValuesPool_throws() { // When / Then assertThatThrownBy(() -> decodeProtoSegments(DType.I32, PType.U8, codes, values, 1, 2)) .isInstanceOf(VortexException.class) - .hasMessageContaining("empty dict child"); + .hasMessageContaining("child holds 0 element(s)"); } @Test @@ -743,7 +739,7 @@ void emptyCodesChild_throws() { // When / Then assertThatThrownBy(() -> decodeProtoSegments(DType.I32, PType.U8, codes, values, 2, 2)) .isInstanceOf(VortexException.class) - .hasMessageContaining("empty dict child"); + .hasMessageContaining("child holds 0 element(s)"); } @Test diff --git a/reader/src/test/java/io/github/dfa1/vortex/reader/decode/FsstEncodingDecoderTest.java b/reader/src/test/java/io/github/dfa1/vortex/reader/decode/FsstEncodingDecoderTest.java index 3dc2018e2..c71f2c313 100644 --- a/reader/src/test/java/io/github/dfa1/vortex/reader/decode/FsstEncodingDecoderTest.java +++ b/reader/src/test/java/io/github/dfa1/vortex/reader/decode/FsstEncodingDecoderTest.java @@ -197,34 +197,27 @@ void forEachByteLength_emptyChildWithNonZeroRows_throwsFromBulkPath() { byte[] compressed = {}; long[] uncompLengths = {}; long[] codeOffsets = {0}; - VarBinArray sut = decodeFsst(1, symbols, symbolLengths, compressed, - PType.U8, uncompLengths, PType.U8, codeOffsets); - - // When / Then + // When / Then — refused when decoded, before any row is read assertThatExceptionOfType(VortexException.class) - .isThrownBy(() -> sut.forEachByteLength(len -> { })) - .withMessageContaining("empty"); + .isThrownBy(() -> decodeFsst(1, symbols, symbolLengths, compressed, + PType.U8, uncompLengths, PType.U8, codeOffsets)) + .withMessageContaining("child holds 0 element(s) for a declared length of 1"); } @Test - void forEachByteLength_broadcastCapacityBetweenOneAndRowCount_readsWithModulo() { - // Given — cap (2) is neither 0 nor >= n (5), so only the per-row modulo path can - // produce the result; existing broadcast fixtures elsewhere use capacity 1 or - // capacity == n, neither of which reaches this branch. + void forEachByteLength_childShorterThanRowCountButNotConstant_throws() { + // Given — 2 lengths for 5 rows: neither a full child nor a constant one. It used to wrap + // around and read 5, 7, 5, 7, 5; Rust has no such shape, so it is malformed input. long[] symbols = {}; byte[] symbolLengths = {}; byte[] compressed = {}; long[] uncompLengths = {5, 7}; long[] codeOffsets = {0}; - VarBinArray sut = decodeFsst(5, symbols, symbolLengths, compressed, - PType.U8, uncompLengths, PType.U8, codeOffsets); - - // When - List lengths = new ArrayList<>(); - sut.forEachByteLength(lengths::add); - - // Then - assertThat(lengths).containsExactly(5, 7, 5, 7, 5); + // When / Then — refused when decoded, before any row is read + assertThatExceptionOfType(VortexException.class) + .isThrownBy(() -> decodeFsst(5, symbols, symbolLengths, compressed, + PType.U8, uncompLengths, PType.U8, codeOffsets)) + .withMessageContaining("child holds 2 element(s) for a declared length of 5"); } @ParameterizedTest(name = "{0}") @@ -260,7 +253,7 @@ void zeroRows_decodesToNothing() { byte[] symbolLengths = {}; byte[] compressed = {}; long[] uncompLengths = {}; - long[] codeOffsets = {}; + long[] codeOffsets = {0}; // n + 1 offsets, as Rust writes them // When VarBinArray result = decodeFsst(0, symbols, symbolLengths, compressed, @@ -463,13 +456,12 @@ void emptyUncompressedLengthsChild_throws() { byte[] compressed = {0x00}; long[] uncompLengths = {}; // empty length child with n == 1 long[] codeOffsets = {0, 1}; - VarBinArray result = decodeFsst(1, symbols, symbolLengths, compressed, - PType.U8, uncompLengths, PType.U8, codeOffsets); - // When / Then — decode() succeeded; reading row 0's length is where this surfaces. + // When / Then — refused when decoded: the child is neither full nor one broadcast value assertThatExceptionOfType(VortexException.class) - .isThrownBy(() -> result.getByteLength(0)) - .withMessageContaining("empty"); + .isThrownBy(() -> decodeFsst(1, symbols, symbolLengths, compressed, + PType.U8, uncompLengths, PType.U8, codeOffsets)) + .withMessageContaining("child holds 0 element(s)"); } @Test @@ -480,13 +472,12 @@ void emptyCodesOffsetsChild_throws() { byte[] compressed = {0x00}; long[] uncompLengths = {2}; long[] codeOffsets = {}; // empty offsets child with n == 1 - VarBinArray result = decodeFsst(1, symbols, symbolLengths, compressed, - PType.U8, uncompLengths, PType.U8, codeOffsets); - // When / Then + // When / Then — refused when decoded: the child is neither full nor one broadcast value assertThatExceptionOfType(VortexException.class) - .isThrownBy(() -> result.getByteLength(0)) - .withMessageContaining("empty"); + .isThrownBy(() -> decodeFsst(1, symbols, symbolLengths, compressed, + PType.U8, uncompLengths, PType.U8, codeOffsets)) + .withMessageContaining("child holds 0 element(s)"); } @Test diff --git a/reader/src/test/java/io/github/dfa1/vortex/reader/decode/PatchedEncodingDecoderTest.java b/reader/src/test/java/io/github/dfa1/vortex/reader/decode/PatchedEncodingDecoderTest.java index bf55e6dc2..4e3ed6095 100644 --- a/reader/src/test/java/io/github/dfa1/vortex/reader/decode/PatchedEncodingDecoderTest.java +++ b/reader/src/test/java/io/github/dfa1/vortex/reader/decode/PatchedEncodingDecoderTest.java @@ -38,7 +38,7 @@ private static Array decode(int n, int[] innerI32, int[] laneOffsets, short[] pa return decode(DType.I32, n, TestSegments.leInts(innerI32), TestSegments.leInts(laneOffsets), TestSegments.leShorts(patchIndices), TestSegments.leInts(patchValues), - laneOffsets.length - 1); + (laneOffsets.length - 1) / Math.max(1, (n + 1023) / 1024)); } private static Array decode(DType dtype, int n, @@ -69,7 +69,8 @@ void decode_noPatches_returnsInnerUnchanged() { int[] inner = {10, 20, 30, 40}; // When - Array result = decode(n, inner, new int[]{0, 0}, new short[]{}, new int[]{}); + // One lane, so one offset per 1024-row chunk plus the end: n_chunks * n_lanes + 1 + Array result = decode(n, inner, new int[(n + 1023) / 1024 + 1], new short[]{}, new int[]{}); // Then assertThat(result).isInstanceOf(IntArray.class); @@ -172,7 +173,8 @@ void decode_variousLengths_noPatches(int n) { int[] inner = new int[n]; // When - Array result = decode(n, inner, new int[]{0, 0}, new short[]{}, new int[]{}); + // One lane, so one offset per 1024-row chunk plus the end: n_chunks * n_lanes + 1 + Array result = decode(n, inner, new int[(n + 1023) / 1024 + 1], new short[]{}, new int[]{}); // Then IntArray ints = (IntArray) result; diff --git a/reader/src/test/java/io/github/dfa1/vortex/reader/decode/RleEncodingDecoderTest.java b/reader/src/test/java/io/github/dfa1/vortex/reader/decode/RleEncodingDecoderTest.java index 7cb5c584a..93c66e432 100644 --- a/reader/src/test/java/io/github/dfa1/vortex/reader/decode/RleEncodingDecoderTest.java +++ b/reader/src/test/java/io/github/dfa1/vortex/reader/decode/RleEncodingDecoderTest.java @@ -383,7 +383,11 @@ private static Array decodeRle(DType dtype, long valuesLen, long rowCount, private static Array decodeRle(DType dtype, long valuesLen, long rowCount, MemorySegment indices, ProtoPType indicesPtype, MemorySegment offsets, MemorySegment values, int offset) { - long indicesLen = indices.byteSize() / (indicesPtype == ProtoPType.U8 ? 1 : 2); + long indicesLen = indices.byteSize() / switch (indicesPtype) { + case U8 -> 1; + case U32 -> 4; + default -> 2; + }; long offsetsLen = offsets.byteSize() / 8; // U64 offsets byte[] metaBytes = new ProtoRLEMetadata( valuesLen, indicesLen, indicesPtype, offsetsLen, ProtoPType.U64, offset).encode(); diff --git a/reader/src/test/java/io/github/dfa1/vortex/reader/decode/SegmentBroadcastTest.java b/reader/src/test/java/io/github/dfa1/vortex/reader/decode/SegmentBroadcastTest.java index 7080baeab..adf48f370 100644 --- a/reader/src/test/java/io/github/dfa1/vortex/reader/decode/SegmentBroadcastTest.java +++ b/reader/src/test/java/io/github/dfa1/vortex/reader/decode/SegmentBroadcastTest.java @@ -1,18 +1,50 @@ package io.github.dfa1.vortex.reader.decode; +import io.github.dfa1.vortex.core.error.VortexException; import io.github.dfa1.vortex.core.io.VortexFormat; import org.junit.jupiter.api.Test; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.CsvSource; import java.lang.foreign.Arena; import java.lang.foreign.MemorySegment; import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; /// Unit tests for [SegmentBroadcast]. These verify the wrap-on-broadcast /// semantics that protect downstream encoders/decoders from IOOB when a child /// segment was produced by [ConstantEncoding] (single-element buffer). class SegmentBroadcastTest { + @ParameterizedTest(name = "{0} elements for {1} rows") + @CsvSource({"4, 4", "5, 4", "1, 1000", "0, 0"}) + void capacity_fullOrConstantChild_isAccepted(long elements, long count) { + // Given — a full child, a longer one, a constant child broadcast, and an empty child of an + // empty array: the shapes a decoder may legally meet + MemorySegment seg = MemorySegment.ofArray(new long[(int) elements]); + + // When + long result = SegmentBroadcast.capacity(seg, 8, count); + + // Then + assertThat(result).isEqualTo(elements); + } + + @ParameterizedTest(name = "{0} elements for {1} rows") + @CsvSource({"0, 1", "0, 1000", "2, 3", "999, 1000"}) + void capacity_emptyOrTruncatedChild_throws(long elements, long count) { + // Given — an empty child of a non-empty array made every caller's `i % capacity` broadcast + // divide by zero (a fuzzer finding in the dict decoder); a truncated one silently repeated + // its values + MemorySegment seg = MemorySegment.ofArray(new long[(int) elements]); + + // When / Then + assertThatThrownBy(() -> SegmentBroadcast.capacity(seg, 8, count)) + .isInstanceOf(VortexException.class) + .hasMessageContaining("child holds " + elements + " element(s) for a declared length of " + count); + } + @Test void elementOffset_wrapsConstantSegment() { try (Arena arena = Arena.ofConfined()) { @@ -24,7 +56,7 @@ void elementOffset_wrapsConstantSegment() { assertThat(SegmentBroadcast.elementOffset(seg, 0, 8)).isZero(); assertThat(SegmentBroadcast.elementOffset(seg, 5, 8)).isZero(); assertThat(SegmentBroadcast.elementOffset(seg, 999_999, 8)).isZero(); - assertThat(SegmentBroadcast.capacity(seg, 8)).isEqualTo(1L); + assertThat(SegmentBroadcast.capacity(seg, 8, 1_000_000)).isEqualTo(1L); } } @@ -39,7 +71,7 @@ void elementOffset_passesThroughFullSegment() { assertThat(SegmentBroadcast.elementOffset(seg, i, 8)).isEqualTo(i * 8); } assertThat(SegmentBroadcast.elementOffset(seg, 4, 8)).isZero(); - assertThat(SegmentBroadcast.capacity(seg, 8)).isEqualTo(4L); + assertThat(SegmentBroadcast.capacity(seg, 8, 4)).isEqualTo(4L); } } diff --git a/reader/src/test/java/io/github/dfa1/vortex/reader/decode/SparseEncodingDecoderTest.java b/reader/src/test/java/io/github/dfa1/vortex/reader/decode/SparseEncodingDecoderTest.java index 07309aa45..742bfc0e7 100644 --- a/reader/src/test/java/io/github/dfa1/vortex/reader/decode/SparseEncodingDecoderTest.java +++ b/reader/src/test/java/io/github/dfa1/vortex/reader/decode/SparseEncodingDecoderTest.java @@ -368,7 +368,8 @@ void missingPatchesMetadata_throws() { /// error and used to blow up at the unchecked `(LongArray)` cast. @Test void patchValuesChildWrongType_throws() { - // Given — an i64 sparse array whose values child is `vortex.bool`, not primitive i64 + // Given — an i64 sparse array whose values child is `vortex.bool`, not primitive i64: its + // one-byte bitmap holds no i64, so it is refused as soon as the child is built ArrayNode boolValuesNode = new ArrayNode(EncodingId.VORTEX_BOOL, null, new ArrayNode[0], new int[]{2}); MemorySegment[] segs = {f64Fill(0.0), TestSegments.leInts(1), boolBitmap(true)}; ArrayNode indicesNode = primitiveNode(1); @@ -377,7 +378,7 @@ void patchValuesChildWrongType_throws() { assertThatThrownBy(() -> decode(DType.I64, 1, 0, PType.U32, 5, segs, indicesNode, boolValuesNode)) .isInstanceOf(VortexException.class) - .hasMessageContaining("patch values child decoded to unexpected type"); + .hasMessageContaining("child holds 0 element(s)"); } /// `ArithmeticException: / by zero` — the `Materialized*` accessors broadcast an @@ -394,7 +395,7 @@ void emptyPatchIndicesChild_throws() { assertThatThrownBy(() -> decode(DType.F64, 2, 0, PType.U32, 5, segs, indicesNode, valuesNode)) .isInstanceOf(VortexException.class) - .hasMessageContaining("empty patch indices child"); + .hasMessageContaining("child holds 0 element(s)"); } /// Same divide-by-zero, reached through the values child instead of the indices child. @@ -409,7 +410,7 @@ void emptyPatchValuesChild_throws() { assertThatThrownBy(() -> decode(DType.F64, 2, 0, PType.U32, 5, segs, indicesNode, valuesNode)) .isInstanceOf(VortexException.class) - .hasMessageContaining("empty patch values child"); + .hasMessageContaining("child holds 0 element(s)"); } /// `OutOfMemoryError` — a null fill makes the decoder build a per-patch validity bitmap @@ -460,7 +461,7 @@ void negativePatchCount_throws() { /// child broadcasts instead of failing; the contract this test can hold the sparse /// decoder to is ADR 0003's: no raw JDK exception, no read outside the value buffer. @Test - void varBinPatchCountBeyondValueOffsets_broadcastsWithoutRawException() { + void varBinPatchCountBeyondValueOffsets_throws() { // Given — 4 declared patches but only 3 offsets, covering 2 values ("b", "d") MemorySegment[] segs = { nullFill(), @@ -469,12 +470,12 @@ void varBinPatchCountBeyondValueOffsets_broadcastsWithoutRawException() { TestSegments.leInts(0, 1, 2) // only 3 offsets = 2 values }; - // When — row 3 resolves to patch 3, past what the offsets child covers - VarBinArray result = assertVarBin(decode(nullableUtf8(), 4, 0, PType.U32, 5, segs, - primitiveNode(1), varBinNode(2, 3))); - - // Then — the broadcast wraps to offsets [0, 1), still inside the 2-byte payload - assertThat(result.getString(3)).isEqualTo("b"); + // When / Then — 3 offsets for 4 values is neither a full nor a constant child: it used + // to wrap around and read patch 3 as "b" + assertThatThrownBy(() -> decode(nullableUtf8(), 4, 0, PType.U32, 5, segs, + primitiveNode(1), varBinNode(2, 3))) + .isInstanceOf(VortexException.class) + .hasMessageContaining("child holds 3 element(s) for a declared length of 5"); } /// `IndexOutOfBoundsException` from `MemorySegment.copy` under the eager merge — From a63bc10c88d78756cba2b4022cae99a431297e2a Mon Sep 17 00:00:00 2001 From: Davide Angelocola Date: Sun, 18 Oct 2026 17:55:44 +0200 Subject: [PATCH 09/13] fix(writer): zero-extend unsigned constants, as Rust reads them Found by the jniReadsJavaWrite differential fuzz target: a sparse column's patch indices, a single u8 index of 230, went through the constant encoding, which read `(byte[]) data)[0]` sign-extended and wrote the u64 scalar 2^64 - 26. vortex-jni refuses the file: "Uint64 value 18446744073709551590 out of range for dtype u8". u16 and u32 constants past their signed range had the same flaw. The constant encoder now takes the element through SparseEncodingEncoder.readBits, which already widens unsigned types correctly, instead of its own copy. Refs #493 Co-Authored-By: Claude Sonnet 5 Claude-Session: https://claude.ai/code/session_01WtuVGQ5h5Lk6Zn4JnkxF1K --- CHANGELOG.md | 1 + .../encode/ConstantEncodingEncoder.java | 13 +++++++--- .../encode/ConstantEncodingEncoderTest.java | 25 +++++++++++++++++++ 3 files changed, 35 insertions(+), 4 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index b8b7fbee5..5339b4fa2 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -47,6 +47,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - A column whose dtype does not fit its bit-packed, delta or RLE encoding fails with `VortexException` instead of `ClassCastException`, and an empty RLE boolean chunk reads. ([#493](https://github.com/dfa1/vortex-java/issues/493)) - A struct dtype missing its field names or types, or a struct layout whose children do not match its fields, is rejected when the file opens, as Rust does, instead of failing the scan with `ArrayIndexOutOfBoundsException`. ([#493](https://github.com/dfa1/vortex-java/issues/493)) - A `vortex.zstd_buffers` array whose wrapped encoding id holds a control character fails with `VortexException` instead of `IllegalArgumentException`. ([#493](https://github.com/dfa1/vortex-java/issues/493)) +- A constant `u8`, `u16` or `u32` column past the signed range of its width is written so vortex-jni reads it; its value was written sign-extended, which Rust refuses. ([#493](https://github.com/dfa1/vortex-java/issues/493)) - A file whose encoded child holds fewer values than its array needs, other than the single value of a constant child, fails with `VortexException` instead of dividing by zero or silently repeating values. ([#493](https://github.com/dfa1/vortex-java/issues/493)) - SQL `WHERE` on an `I16`, `U8` or `I8` column no longer fails with a `ClassCastException`. ([#556](https://github.com/dfa1/vortex-java/pull/556)) - SQL `COUNT(DISTINCT col)` and aggregates with a `FILTER (WHERE ...)` clause return the right answer; the statistics push-down returned the row count and ignored the filter. ([#554](https://github.com/dfa1/vortex-java/pull/554)) diff --git a/writer/src/main/java/io/github/dfa1/vortex/writer/encode/ConstantEncodingEncoder.java b/writer/src/main/java/io/github/dfa1/vortex/writer/encode/ConstantEncodingEncoder.java index d6e841380..d642226ee 100644 --- a/writer/src/main/java/io/github/dfa1/vortex/writer/encode/ConstantEncodingEncoder.java +++ b/writer/src/main/java/io/github/dfa1/vortex/writer/encode/ConstantEncodingEncoder.java @@ -137,11 +137,16 @@ private static OptionalLong constantBits(Object data, PType ptype) { if (!SimdOperationsSupport.preferred().allEqual(data, ptype)) { return OptionalLong.empty(); } - // No default: every PType is handled, so a new one fails to compile here instead of at write time + // No default: every PType is handled, so a new one fails to compile here instead of at write + // time. Unsigned values widen unsigned: a sign-extended u8 230 became the u64 scalar + // 2^64 - 26, which Rust refuses to read ("out of range for dtype u8"). return OptionalLong.of(switch (ptype) { - case I8, U8 -> ((byte[]) data)[0]; - case I16, U16, F16 -> ((short[]) data)[0]; - case I32, U32 -> ((int[]) data)[0]; + case I8 -> ((byte[]) data)[0]; + case U8 -> Byte.toUnsignedLong(((byte[]) data)[0]); + case I16, F16 -> ((short[]) data)[0]; + case U16 -> Short.toUnsignedLong(((short[]) data)[0]); + case I32 -> ((int[]) data)[0]; + case U32 -> Integer.toUnsignedLong(((int[]) data)[0]); case I64, U64 -> ((long[]) data)[0]; case F32 -> Float.floatToRawIntBits(((float[]) data)[0]); case F64 -> Double.doubleToRawLongBits(((double[]) data)[0]); diff --git a/writer/src/test/java/io/github/dfa1/vortex/writer/encode/ConstantEncodingEncoderTest.java b/writer/src/test/java/io/github/dfa1/vortex/writer/encode/ConstantEncodingEncoderTest.java index 92fbe891c..1772e5c05 100644 --- a/writer/src/test/java/io/github/dfa1/vortex/writer/encode/ConstantEncodingEncoderTest.java +++ b/writer/src/test/java/io/github/dfa1/vortex/writer/encode/ConstantEncodingEncoderTest.java @@ -1,6 +1,7 @@ package io.github.dfa1.vortex.writer.encode; import io.github.dfa1.vortex.core.model.DType; +import io.github.dfa1.vortex.core.model.PType; import io.github.dfa1.vortex.core.model.EncodingId; import io.github.dfa1.vortex.core.error.VortexException; import io.github.dfa1.vortex.core.proto.ProtoNullValue; @@ -179,6 +180,23 @@ void encode_f16_writesRustsF16ScalarField() throws java.io.IOException { assertThat(scalar.f16_value()).isEqualTo(0x3E00L); } + @org.junit.jupiter.params.ParameterizedTest(name = "{0}") + @org.junit.jupiter.params.provider.MethodSource("io.github.dfa1.vortex.writer.encode.ConstantEncodingEncoderTest#highUnsignedConstants") + void encode_unsignedAboveSignedRange_writesZeroExtendedScalar(PType ptype, Object data, long expected) + throws java.io.IOException { + // Given — a constant past the signed range of its width: it was sign-extended, so a u8 230 + // became the u64 scalar 2^64 - 26, which vortex-jni refuses ("out of range for dtype u8"; + // found by the differential fuzz target) + + // When + EncodeResult result = ENCODER.encode(new DType.Primitive(ptype, false), data, EncodeTestHelper.testCtx()); + + // Then + MemorySegment seg = result.encodedBuffers().getFirst().data(); + ProtoScalarValue scalar = ProtoScalarValue.decode(seg, 0, seg.byteSize()); + assertThat(scalar.uint64_value()).isEqualTo(expected); + } + @Test void decode_f16_acceptsTheLegacyUint64Scalar() { // Given — older Rust writers stored an f16 as uint64_value (its bits) @@ -475,4 +493,11 @@ void decode_nullScalar_returnsNullArray(DType dtype) { assertThat(result.length()).isEqualTo(rowCount); } } + + static Stream highUnsignedConstants() { + return Stream.of( + Arguments.of(PType.U8, new byte[]{(byte) 230, (byte) 230}, 230L), + Arguments.of(PType.U16, new short[]{(short) 0xFFF0, (short) 0xFFF0}, 0xFFF0L), + Arguments.of(PType.U32, new int[]{0xFFFF_FFF0, 0xFFFF_FFF0}, 0xFFFF_FFF0L)); + } } From 77d90ed0c55a0e09c28c3f69d8325cd16abe324b Mon Sep 17 00:00:00 2001 From: Davide Angelocola Date: Sun, 18 Oct 2026 17:55:44 +0200 Subject: [PATCH 10/13] fix(writer): encode a sequence only where Rust's sequence_encode would Found by the jniReadsJavaWrite differential fuzz target: a sorted u16 column past 0x7FFF was read sign-extended, and vortex-jni refuses the u64 scalar 2^64 - 2067 ("out of range for dtype u16"). The encoder now follows encode_primitive_array: values read zero-extended for unsigned types; the step is slice[1].checked_sub(base) in the column's own type, so a decreasing unsigned column or an i8 step of 255 is no sequence (both wrote a scalar Rust cannot read); an empty array or a zero step (constant data, left to the constant encoding) is no sequence; every later step is a checked_add in that type. Refs #493 Co-Authored-By: Claude Sonnet 5 Claude-Session: https://claude.ai/code/session_01WtuVGQ5h5Lk6Zn4JnkxF1K --- CHANGELOG.md | 1 + .../encode/SequenceEncodingEncoder.java | 122 ++++++++++++++---- .../encode/SequenceEncodingEncoderTest.java | 58 ++++++--- 3 files changed, 134 insertions(+), 47 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 5339b4fa2..ceece47dc 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -48,6 +48,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - A struct dtype missing its field names or types, or a struct layout whose children do not match its fields, is rejected when the file opens, as Rust does, instead of failing the scan with `ArrayIndexOutOfBoundsException`. ([#493](https://github.com/dfa1/vortex-java/issues/493)) - A `vortex.zstd_buffers` array whose wrapped encoding id holds a control character fails with `VortexException` instead of `IllegalArgumentException`. ([#493](https://github.com/dfa1/vortex-java/issues/493)) - A constant `u8`, `u16` or `u32` column past the signed range of its width is written so vortex-jni reads it; its value was written sign-extended, which Rust refuses. ([#493](https://github.com/dfa1/vortex-java/issues/493)) +- `vortex.sequence` is written only where Rust writes it: not for empty or constant data, nor for a step that overflows the column type (a decreasing unsigned column, an `i8` step of 255), and unsigned values past the signed range are written so vortex-jni reads them. ([#493](https://github.com/dfa1/vortex-java/issues/493)) - A file whose encoded child holds fewer values than its array needs, other than the single value of a constant child, fails with `VortexException` instead of dividing by zero or silently repeating values. ([#493](https://github.com/dfa1/vortex-java/issues/493)) - SQL `WHERE` on an `I16`, `U8` or `I8` column no longer fails with a `ClassCastException`. ([#556](https://github.com/dfa1/vortex-java/pull/556)) - SQL `COUNT(DISTINCT col)` and aggregates with a `FILTER (WHERE ...)` clause return the right answer; the statistics push-down returned the row count and ignored the filter. ([#554](https://github.com/dfa1/vortex-java/pull/554)) diff --git a/writer/src/main/java/io/github/dfa1/vortex/writer/encode/SequenceEncodingEncoder.java b/writer/src/main/java/io/github/dfa1/vortex/writer/encode/SequenceEncodingEncoder.java index e9b7131ad..988293568 100644 --- a/writer/src/main/java/io/github/dfa1/vortex/writer/encode/SequenceEncodingEncoder.java +++ b/writer/src/main/java/io/github/dfa1/vortex/writer/encode/SequenceEncodingEncoder.java @@ -1,6 +1,8 @@ package io.github.dfa1.vortex.writer.encode; import io.github.dfa1.vortex.core.model.DType; +import io.github.dfa1.vortex.core.simd.SimdOperations; +import io.github.dfa1.vortex.core.simd.SimdOperationsSupport; import io.github.dfa1.vortex.core.model.PType; import io.github.dfa1.vortex.core.error.VortexException; import io.github.dfa1.vortex.core.model.EncodingId; @@ -14,6 +16,10 @@ /// Write-only encoder for `vortex.sequence` — arithmetic sequences as (base, multiplier). public final class SequenceEncodingEncoder implements EncodingEncoder { + /// Elements widened per step of [#encodeInteger(PType, Object)]'s scan. + private static final int WIDEN_BLOCK = 1024; + + @Override public EncodingId encodingId() { return EncodingId.VORTEX_SEQUENCE; @@ -67,34 +73,103 @@ public EncodeResult encode(DType dtype, Object data, EncodeContext ctx) { return encodeInteger(((DType.Primitive) dtype).ptype(), data); } + /// Rust's `encode_primitive_array`: `base` is the first value and the multiplier + /// `slice[1].checked_sub(base)` in the column's own type, so a step that does not fit it (a + /// decreasing unsigned column, an `i8` step of 255) is no sequence; nor is an empty array or a + /// zero step (constant data, left to the constant encoding); every later step is a + /// `checked_add` in that type too. Values widen zero-extended for unsigned types: a + /// sign-extended `u16` became a `u64` scalar Rust refuses ("out of range for dtype u16"). private static EncodeResult encodeInteger(PType pt, Object data) { int n = Array.getLength(data); + if (n == 0) { + throw new VortexException(EncodingId.VORTEX_SEQUENCE, "an empty array is not a sequence"); + } + // Widened a block at a time: the cascade tries this on every integer column, and most fail + // by their third value, so neither a column-sized copy nor a full scan is spent on them. + long[] block = new long[Math.min(n, WIDEN_BLOCK)]; + SimdOperations ops = SimdOperationsSupport.preferred(); long base = 0; long multiplier = 0; - if (n > 0) { - base = readLong(pt, data, 0); - multiplier = n > 1 ? readLong(pt, data, 1) - base : 0; - for (int i = 2; i < n; i++) { - long expected = base + i * multiplier; - if (readLong(pt, data, i) != expected) { + long previous = 0; + for (int from = 0; from < n; from += block.length) { + int count = Math.min(block.length, n - from); + ops.widenArrayInto(data, from, count, pt, block); + for (int j = 0; j < count; j++) { + int i = from + j; + long value = block[j]; + if (i == 0) { + base = value; + } else if (i == 1) { + multiplier = checkedSub(pt, value, base); + if (multiplier == 0) { + throw new VortexException(EncodingId.VORTEX_SEQUENCE, "a zero step is not a sequence"); + } + } else if (value != checkedAdd(pt, previous, multiplier)) { throw new VortexException(EncodingId.VORTEX_SEQUENCE, "not an arithmetic sequence at index " + i); } + previous = value; } } - ProtoScalarValue baseScalar = buildIntScalar(pt, base); - ProtoScalarValue mulScalar = buildIntScalar(pt, multiplier); - // A perfect arithmetic sequence is monotonic (or constant) end to end, so its extremes are - // always the first and last element -- no separate scan needed. - boolean unsign = pt.isUnsigned(); - ProtoScalarValue statsMin = null; - ProtoScalarValue statsMax = null; - if (n > 0) { - long last = base + (n - 1) * multiplier; - boolean baseIsMin = unsign ? Long.compareUnsigned(base, last) <= 0 : base <= last; - statsMin = buildIntScalar(pt, baseIsMin ? base : last); - statsMax = buildIntScalar(pt, baseIsMin ? last : base); + long last = previous; + boolean baseIsMin = pt.isUnsigned() ? Long.compareUnsigned(base, last) <= 0 : base <= last; + return buildResult(buildIntScalar(pt, base), buildIntScalar(pt, multiplier), + buildIntScalar(pt, baseIsMin ? base : last), buildIntScalar(pt, baseIsMin ? last : base)); + } + + /// `a - b` in `pt`, or not applicable when the result does not fit `pt`. + private static long checkedSub(PType pt, long a, long b) { + return switch (pt) { + case U64 -> { + if (Long.compareUnsigned(a, b) < 0) { + throw notInType(pt); + } + yield a - b; + } + case I64 -> { + try { + yield Math.subtractExact(a, b); + } catch (ArithmeticException _) { + throw notInType(pt); + } + } + default -> fitting(pt, a - b); + }; + } + + /// `a + b` in `pt`, or not applicable when the result does not fit `pt`. + private static long checkedAdd(PType pt, long a, long b) { + return switch (pt) { + case U64 -> { + long sum = a + b; + if (Long.compareUnsigned(sum, a) < 0) { + throw notInType(pt); + } + yield sum; + } + case I64 -> { + try { + yield Math.addExact(a, b); + } catch (ArithmeticException _) { + throw notInType(pt); + } + } + default -> fitting(pt, a + b); + }; + } + + /// `value` (exact in a `long` for widths below 64 bits) if it lies in `pt`'s range. + private static long fitting(PType pt, long value) { + long bits = pt.bits(); + long min = pt.isUnsigned() ? 0 : -(1L << (bits - 1)); + long max = pt.isUnsigned() ? (1L << bits) - 1 : (1L << (bits - 1)) - 1; + if (value < min || value > max) { + throw notInType(pt); } - return buildResult(baseScalar, mulScalar, statsMin, statsMax); + return value; + } + + private static VortexException notInType(PType pt) { + return new VortexException(EncodingId.VORTEX_SEQUENCE, "sequence step overflows " + pt); } private static EncodeResult buildResult(ProtoScalarValue base, ProtoScalarValue mul, ProtoScalarValue statsMin, ProtoScalarValue statsMax) { @@ -111,13 +186,4 @@ private static ProtoScalarValue buildIntScalar(PType pt, long value) { }; } - private static long readLong(PType pt, Object data, int i) { - return switch (pt) { - case I8, U8 -> ((byte[]) data)[i]; - case I16, U16 -> ((short[]) data)[i]; - case I32, U32 -> ((int[]) data)[i]; - case I64, U64 -> ((long[]) data)[i]; - default -> throw new VortexException(EncodingId.VORTEX_SEQUENCE, "unsupported ptype: " + pt); - }; - } } diff --git a/writer/src/test/java/io/github/dfa1/vortex/writer/encode/SequenceEncodingEncoderTest.java b/writer/src/test/java/io/github/dfa1/vortex/writer/encode/SequenceEncodingEncoderTest.java index 9acce2a66..3800078e0 100644 --- a/writer/src/test/java/io/github/dfa1/vortex/writer/encode/SequenceEncodingEncoderTest.java +++ b/writer/src/test/java/io/github/dfa1/vortex/writer/encode/SequenceEncodingEncoderTest.java @@ -174,17 +174,50 @@ void encode_u16_roundTrips() { } @Test - void encode_emptyArray_roundTripsToZeroLength() { - // Given the n==0 branch: base and multiplier default to 0 + void encode_emptyArray_throws() { + // Given — Rust's sequence_encode: "we cannot encode an empty array" long[] data = {}; + // When / Then + assertThatThrownBy(() -> ENCODER.encode(DTypes.I64, data, EncodeTestHelper.testCtx())) + .isInstanceOf(VortexException.class) + .hasMessageContaining("empty array is not a sequence"); + } + + @org.junit.jupiter.params.ParameterizedTest(name = "{0} {1}") + @org.junit.jupiter.params.provider.MethodSource("notSequencesInRust") + void encode_stepRustRejects_throws(PType ptype, Object data, String reason) { + // Given — data whose step Rust's checked arithmetic in the column type refuses: these + // used to be written, and a sign-extended or out-of-range scalar made the file + // unreadable for Rust (found by the differential fuzz target) + + // When / Then + assertThatThrownBy(() -> ENCODER.encode(new DType.Primitive(ptype, false), data, EncodeTestHelper.testCtx())) + .isInstanceOf(VortexException.class) + .hasMessageContaining(reason); + } + + static Stream notSequencesInRust() { + return Stream.of( + Arguments.of(PType.I64, new long[]{5, 5, 5}, "zero step"), + Arguments.of(PType.U8, new byte[]{3, 2, 1}, "overflows U8"), + Arguments.of(PType.I8, new byte[]{-128, 127}, "overflows I8"), + Arguments.of(PType.I64, new long[]{Long.MIN_VALUE, Long.MAX_VALUE}, "overflows I64"), + Arguments.of(PType.U64, new long[]{-1L, 0L}, "overflows U64")); + } + + @Test + void encode_unsignedAboveSignedRange_writesZeroExtendedScalars() { + // Given — a sorted u16 sequence past 0x7FFF: its base was sign-extended into a u64 scalar + // of 2^64 - 2067, which vortex-jni refuses ("out of range for dtype u16") + short[] data = {(short) 63_469, (short) 63_470, (short) 63_471}; + // When - EncodeResult resultEncoded = ENCODER.encode(DTypes.I64, data, EncodeTestHelper.testCtx()); - DecodeContext ctx = encodeResultToCtx(resultEncoded, DTypes.I64, 0); - Array result = DECODER.decode(ctx); + EncodeResult result = ENCODER.encode(DTypes.U16, data, EncodeTestHelper.testCtx()); // Then - assertThat(result.length()).isZero(); + assertThat(result.statsMin().uint64_value()).isEqualTo(63_469L); + assertThat(result.statsMax().uint64_value()).isEqualTo(63_471L); } @Test @@ -427,19 +460,6 @@ void encode_i64_decreasing_reportsLastAsMinFirstAsMax() throws java.io.IOExcepti assertThat(result.statsMax().int64_value()).isEqualTo(40L); } - @Test - void encode_empty_statsAreNull() { - // Given - long[] data = {}; - - // When - EncodeResult result = ENCODER.encode(DTypes.I64, data, EncodeTestHelper.testCtx()); - - // Then - assertThat(result.statsMin()).isNull(); - assertThat(result.statsMax()).isNull(); - } - private static ProtoScalarValue scalar(byte[] bytes) throws java.io.IOException { MemorySegment seg = MemorySegment.ofArray(bytes); return ProtoScalarValue.decode(seg, 0, seg.byteSize()); From 9ec8aa3acf1d85f715958c2c9681aa82af69d80b Mon Sep 17 00:00:00 2001 From: Davide Angelocola Date: Sun, 18 Oct 2026 17:55:44 +0200 Subject: [PATCH 11/13] test(fuzz): write-read and vortex-jni differential targets; replay in every build; weekly fuzzing Write side: the fuzzer builds a small table, vortex-java writes it, and a reader must return exactly the values written, vortex-java (javaRoundTrip) and vortex-jni (jniReadsJavaWrite). Per column it picks a dtype, nullability, a shape (random, constant, small range, runs, sorted, sparse) and a seed, steering the cascading compressor into its edges while an input stays a few bytes. Both live in integration, where the JNI dependency is; failsafe now loads the byte-buddy agent as surefire does, since Jazzer installs itself through it. The fuzz module excluded its targets from ./mvnw test, guarding against unbounded fuzzing, which only JAZZER_FUZZ=1 starts. Without it a target replays its saved inputs, seeds plus every past finding, in about a second, so the exclusion is gone: a regression of any finding fails the ordinary build. The Fuzz workflow explores new inputs every Saturday (or on demand with a duration), one matrix job per target, uploading any new crash input. fuzz/oss-fuzz holds the project files for an OSS-Fuzz submission (JDK 25 shipped with the fuzzers, one Jazzer --target_method wrapper per target); checked locally without Docker, submission is the maintainer's step. Fuzzed 5 min each: javaRoundTrip 181k inputs, clean; jniReadsJavaWrite found the two writer bugs fixed before this, then 133k inputs clean. Refs #493 Co-Authored-By: Claude Sonnet 5 Claude-Session: https://claude.ai/code/session_01WtuVGQ5h5Lk6Zn4JnkxF1K --- .github/workflows/fuzz.yml | 77 ++++ CLAUDE.md | 17 +- fuzz/oss-fuzz/Dockerfile | 9 + fuzz/oss-fuzz/README.md | 25 ++ fuzz/oss-fuzz/build.sh | 45 +++ fuzz/oss-fuzz/project.yaml | 8 + fuzz/pom.xml | 26 +- integration/pom.xml | 9 + .../WriteReadFuzzIntegrationTest.java | 378 ++++++++++++++++++ .../crash-constant-u8-sign-extended | 1 + .../crash-sequence-u16-sign-extended | Bin 0 -> 19 bytes .../test/resources/junit-platform.properties | 4 + 12 files changed, 569 insertions(+), 30 deletions(-) create mode 100644 .github/workflows/fuzz.yml create mode 100644 fuzz/oss-fuzz/Dockerfile create mode 100644 fuzz/oss-fuzz/README.md create mode 100755 fuzz/oss-fuzz/build.sh create mode 100644 fuzz/oss-fuzz/project.yaml create mode 100644 integration/src/test/java/io/github/dfa1/vortex/integration/WriteReadFuzzIntegrationTest.java create mode 100644 integration/src/test/resources/io/github/dfa1/vortex/integration/WriteReadFuzzIntegrationTestInputs/jniReadsJavaWrite/crash-constant-u8-sign-extended create mode 100644 integration/src/test/resources/io/github/dfa1/vortex/integration/WriteReadFuzzIntegrationTestInputs/jniReadsJavaWrite/crash-sequence-u16-sign-extended diff --git a/.github/workflows/fuzz.yml b/.github/workflows/fuzz.yml new file mode 100644 index 000000000..e2f342ecd --- /dev/null +++ b/.github/workflows/fuzz.yml @@ -0,0 +1,77 @@ +name: Fuzz + +# Weekly coverage-guided fuzzing (ADR 0020, #493). Every @FuzzTest replays its saved inputs in the +# ordinary build; this job explores new ones, one target per matrix job because Jazzer fuzzes a +# single target per JVM. Scheduled, not on the PR path: a run is unbounded exploration, so a +# failure is a finding to triage (the crash input is uploaded; commit it next to the target's +# saved inputs with the fix), not a broken build. + +on: + schedule: + - cron: '0 3 * * 6' # Saturdays 03:00 UTC + workflow_dispatch: + inputs: + duration: + description: 'Fuzzing time per target (e.g. 10m, 1h)' + default: '30m' + +jobs: + fuzz: + runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + include: + - { module: fuzz, target: 'VortexReaderFuzzTest#openMalformedFile' } + - { module: fuzz, target: 'VortexReaderFuzzTest#scanMalformedFile' } + - { module: fuzz, target: 'VortexReaderFuzzTest#filterMalformedFile' } + - { module: fuzz, target: 'VortexReaderFuzzTest#parseMetadataBlobs' } + - { module: fuzz, target: 'FsstFuzzTest#compressDecompressRoundTrip' } + - { module: integration, target: 'WriteReadFuzzIntegrationTest#javaRoundTrip' } + - { module: integration, target: 'WriteReadFuzzIntegrationTest#jniReadsJavaWrite' } + name: ${{ matrix.target }} + + steps: + - uses: actions/checkout@v7 + + - name: Set up Azul Zulu JDK 25 + uses: actions/setup-java@v6.0.1 + with: + distribution: zulu + java-version: '25' + + - name: Cache Maven repository + uses: actions/cache@v6 + with: + path: ~/.m2/repository + key: ${{ runner.os }}-maven-${{ hashFiles('**/pom.xml') }} + restore-keys: | + ${{ runner.os }}-maven- + + - name: Fuzz + env: + JAZZER_FUZZ: '1' + JAZZER_MAX_DURATION: ${{ inputs.duration || '30m' }} + run: | + if [ "${{ matrix.module }}" = integration ]; then + ./mvnw -B verify -pl integration -am -Dit.test='${{ matrix.target }}' \ + -Dtest=none -Dsurefire.failIfNoSpecifiedTests=false + else + ./mvnw -B test -pl fuzz -am -Dtest='${{ matrix.target }}' \ + -Dsurefire.failIfNoSpecifiedTests=false + fi + + # Jazzer writes a finding next to the target's saved inputs; the committed crash-* inputs + # are regressions already fixed, so only the new, untracked ones are collected. + - name: Collect the new crash input + if: failure() + run: | + mkdir -p findings + git ls-files --others --exclude-standard | grep '/crash-' | xargs -r -I{} cp --parents {} findings/ + + - name: Upload the crash input + if: failure() + uses: actions/upload-artifact@v7 + with: + name: crash-${{ strategy.job-index }} + path: findings/ diff --git a/CLAUDE.md b/CLAUDE.md index c9b9318e2..103696d2a 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -84,8 +84,8 @@ and merge commits are disabled): squash a PR's commits locally into one, force-p scripts/simd-benchmark.sh # auto-vectorized vs Vector API kernels, table of speedups (also a CI job) scripts/hydrate-raincloud-corpus.sh --max-mb 200 # hydrate real-world conformance corpus (#205), then: ./mvnw verify -pl integration -am -Dvortex.it.excludedGroups= -Dit.test="RaincloudConformanceIntegrationTest" -./mvnw test -pl fuzz -am -Dvortex.fuzz.excludedGroups= # Jazzer regression mode (ADR 0020) -JAZZER_FUZZ=1 ./mvnw test -pl fuzz -am -Dvortex.fuzz.excludedGroups= # actually fuzz (unbounded) +JAZZER_FUZZ=1 JAZZER_MAX_DURATION=5m ./mvnw test -pl fuzz -am -Dtest='VortexReaderFuzzTest#scanMalformedFile' -Dsurefire.failIfNoSpecifiedTests=false # fuzz one target (ADR 0020) +JAZZER_FUZZ=1 JAZZER_MAX_DURATION=5m ./mvnw verify -pl integration -am -Dit.test='WriteReadFuzzIntegrationTest#jniReadsJavaWrite' -Dtest=none -Dsurefire.failIfNoSpecifiedTests=false ./mvnw test -pl fuzz -am -Dtest=SeedCorpusGenerator -Dsurefire.failIfNoSpecifiedTests=false -Dvortex.fuzz.regenerateSeeds=true # rewrite per-encoding seeds ``` @@ -96,9 +96,16 @@ Tag-excluded from a routine local build, even when named with `-Dit.test`/`-Dtes integration test taking over ~1 s. - **Raincloud corpus** (`@Tag("raincloud")`, failsafe): clear the exclusion with `-Dvortex.it.excludedGroups=` as shown above. Not run by the `ci` profile. -- **Jazzer fuzz** (`@Tag("fuzz")`, own `fuzz` module): clear it with `-Dvortex.fuzz.excludedGroups=`. - Without `JAZZER_FUZZ=1` it only replays the saved corpus; with it, it fuzzes until stopped — - never in CI or unattended. See [ADR 0020](adr/0020-jazzer-fuzz-infrastructure.md). + +Jazzer fuzz targets (`@FuzzTest`, ADR 0020) are **not** excluded: without `JAZZER_FUZZ=1` each +replays its saved inputs (`*Inputs//`: seeds plus every past finding) in about a second, +as part of the routine build. `JAZZER_FUZZ=1` explores new inputs, one target per run, bounded by +`JAZZER_MAX_DURATION`; the weekly `Fuzz` workflow does that for every target. Targets: reader +`open`/`scan`/`filter`/metadata-blob and FSST round-trip in `fuzz`; write→read round-trip and the +vortex-jni differential in `integration` (`WriteReadFuzzIntegrationTest`). A finding is written +next to the target's inputs as `crash-`: rename it after the bug, fix, and commit it with +the fix, plus a `MalformedFileInteropIntegrationTest` case when it is a file vortex-jni also +reads. Regenerate after editing `.fbs`/`.proto` (both generators are in-house, no external tools): diff --git a/fuzz/oss-fuzz/Dockerfile b/fuzz/oss-fuzz/Dockerfile new file mode 100644 index 000000000..31bd2d18f --- /dev/null +++ b/fuzz/oss-fuzz/Dockerfile @@ -0,0 +1,9 @@ +# OSS-Fuzz builder for vortex-java (#493). The base image's JDK is 17; the project needs 25, which +# build.sh uses to build and ships next to the fuzzers for them to run on. +FROM gcr.io/oss-fuzz-base/base-builder-jvm +RUN curl -sSL -o jdk.tar.gz \ + "https://api.adoptium.net/v3/binary/latest/25/ga/linux/x64/jdk/hotspot/normal/eclipse" && \ + mkdir $SRC/jdk25 && tar -xzf jdk.tar.gz -C $SRC/jdk25 --strip-components=1 && rm jdk.tar.gz +RUN git clone --depth 1 https://github.com/dfa1/vortex-java vortex-java +COPY build.sh $SRC/ +WORKDIR $SRC/vortex-java diff --git a/fuzz/oss-fuzz/README.md b/fuzz/oss-fuzz/README.md new file mode 100644 index 000000000..da18b930c --- /dev/null +++ b/fuzz/oss-fuzz/README.md @@ -0,0 +1,25 @@ +# OSS-Fuzz integration (prepared, not submitted) + +The files an [OSS-Fuzz](https://google.github.io/oss-fuzz/) project needs, to copy into +`projects/vortex-java/` of a google/oss-fuzz pull request (#493): + +- `project.yaml`: fill in `primary_contact` before submitting. +- `Dockerfile`: the JVM builder image, plus JDK 25 (the image ships 17) and a clone of this repo. +- `build.sh`: builds the `fuzz` module and writes one fuzzer per `@FuzzTest` (reader open, scan, + filtered scan, metadata blobs; FSST round trip), each seeded with its saved inputs. The + `integration` targets need the vortex-jni native library and are left out. + +Checked locally without Docker: `build.sh` (with a local JDK 25 as `$SRC/jdk25`) packages the +fuzzers, and a generated wrapper ran `scanMalformedFile` 20,000 times on Jazzer 0.30's standalone +driver. Not yet run through OSS-Fuzz's own tooling; before submitting, from a google/oss-fuzz +checkout with these files in `projects/vortex-java/`: + +```bash +python3 infra/helper.py build_image vortex-java +python3 infra/helper.py build_fuzzers vortex-java +python3 infra/helper.py check_build vortex-java +python3 infra/helper.py run_fuzzer vortex-java VortexReaderFuzzTest_scanMalformedFile +``` + +OSS-Fuzz accepts projects that are widely used or critical infrastructure; the weekly `Fuzz` +GitHub workflow covers the same targets meanwhile. diff --git a/fuzz/oss-fuzz/build.sh b/fuzz/oss-fuzz/build.sh new file mode 100755 index 000000000..9ff330a2d --- /dev/null +++ b/fuzz/oss-fuzz/build.sh @@ -0,0 +1,45 @@ +#!/bin/bash -eu +# OSS-Fuzz build for vortex-java (#493): every JUnit @FuzzTest of the `fuzz` module becomes one +# fuzzer, run by Jazzer's driver with --target_class/--target_method, seeded with its saved inputs. +# The `integration` targets are left out: they need the vortex-jni native library. + +export JAVA_HOME=$SRC/jdk25 +export PATH=$JAVA_HOME/bin:$PATH + +# One reactor invocation: the sibling modules are packaged, never installed, so copy-dependencies +# must resolve them from this same build. Jazzer comes from OSS-Fuzz, not Maven. +./mvnw -B -q package dependency:copy-dependencies -pl fuzz -am -DskipTests \ + -DincludeScope=test -DexcludeGroupIds=com.code-intelligence -DoutputDirectory="$OUT/libs" +cp -r fuzz/target/test-classes "$OUT/classes" +cp -r "$JAVA_HOME" "$OUT/jdk" + +INPUTS=fuzz/src/test/resources/io/github/dfa1/vortex/fuzz +for target in \ + VortexReaderFuzzTest:openMalformedFile \ + VortexReaderFuzzTest:scanMalformedFile \ + VortexReaderFuzzTest:filterMalformedFile \ + VortexReaderFuzzTest:parseMetadataBlobs \ + FsstFuzzTest:compressDecompressRoundTrip; do + class=${target%%:*} + method=${target##*:} + fuzzer=${class}_${method} + + if compgen -G "$INPUTS/${class}Inputs/$method/*" > /dev/null; then + zip -jq "$OUT/${fuzzer}_seed_corpus.zip" "$INPUTS/${class}Inputs/$method"/* + fi + + # The JDK 25 shipped above runs the fuzzer; OSS-Fuzz's own jazzer jars are on the classpath. + cat > "$OUT/$fuzzer" <vortex-fuzz Coverage-guided fuzz testing (ADR 0020) for the Vortex columnar file format Java implementation. - Opt-in only — see CLAUDE.md. + Saved inputs replay in every build; JAZZER_FUZZ=1 explores new ones — see CLAUDE.md. true - - fuzz @@ -47,23 +42,4 @@ - - - - org.apache.maven.plugins - maven-surefire-plugin - - - ${vortex.fuzz.excludedGroups} - - - - diff --git a/integration/pom.xml b/integration/pom.xml index 6c915e65e..5badc2461 100644 --- a/integration/pom.xml +++ b/integration/pom.xml @@ -107,6 +107,11 @@ junit-jupiter test + + com.code-intelligence + jazzer-junit + test + org.assertj assertj-core @@ -242,8 +247,12 @@ class selection, so running a raincloud test always requires clearing the property as well. --> ${vortex.it.excludedGroups} + @{argLine} + -javaagent:${net.bytebuddy:byte-buddy-agent:jar} --add-opens java.base/java.nio=ALL-UNNAMED --enable-native-access=ALL-UNNAMED --sun-misc-unsafe-memory-access=allow diff --git a/integration/src/test/java/io/github/dfa1/vortex/integration/WriteReadFuzzIntegrationTest.java b/integration/src/test/java/io/github/dfa1/vortex/integration/WriteReadFuzzIntegrationTest.java new file mode 100644 index 000000000..8cb2a59ce --- /dev/null +++ b/integration/src/test/java/io/github/dfa1/vortex/integration/WriteReadFuzzIntegrationTest.java @@ -0,0 +1,378 @@ +package io.github.dfa1.vortex.integration; + +import static org.junit.jupiter.api.Assertions.assertEquals; + +import java.io.IOException; +import java.nio.channels.FileChannel; +import java.nio.file.Path; +import java.nio.file.StandardOpenOption; +import java.util.ArrayList; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import java.util.Random; + +import com.code_intelligence.jazzer.api.FuzzedDataProvider; +import com.code_intelligence.jazzer.junit.FuzzTest; +import dev.vortex.api.DataSource; +import dev.vortex.api.Partition; +import dev.vortex.api.Scan; +import dev.vortex.api.ScanOptions; +import dev.vortex.api.Session; +import dev.vortex.arrow.ArrowAllocation; +import dev.vortex.jni.NativeLoader; +import io.github.dfa1.vortex.core.compute.PrimitiveArrays; +import io.github.dfa1.vortex.core.model.ColumnName; +import io.github.dfa1.vortex.core.model.DType; +import io.github.dfa1.vortex.core.model.EncodingId; +import io.github.dfa1.vortex.reader.Chunk; +import io.github.dfa1.vortex.reader.VortexReader; +import io.github.dfa1.vortex.reader.array.Array; +import io.github.dfa1.vortex.reader.array.BoolArray; +import io.github.dfa1.vortex.reader.array.ByteArray; +import io.github.dfa1.vortex.reader.array.DoubleArray; +import io.github.dfa1.vortex.reader.array.FloatArray; +import io.github.dfa1.vortex.reader.array.IntArray; +import io.github.dfa1.vortex.reader.array.LongArray; +import io.github.dfa1.vortex.reader.array.MaskedArray; +import io.github.dfa1.vortex.reader.array.NullArray; +import io.github.dfa1.vortex.reader.array.ShortArray; +import io.github.dfa1.vortex.reader.array.VarBinArray; +import io.github.dfa1.vortex.writer.VortexWriter; +import io.github.dfa1.vortex.writer.WriteOptions; +import io.github.dfa1.vortex.writer.encode.NullableData; +import org.apache.arrow.vector.FieldVector; +import org.apache.arrow.vector.VectorSchemaRoot; +import org.apache.arrow.vector.ipc.ArrowReader; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Tag; +import org.junit.jupiter.api.io.TempDir; + +/// Write-side correctness fuzzing (#493): the fuzzer builds a small table, vortex-java writes it, +/// and a reader must give back exactly the values written. +/// +/// The fuzzer does not choose every value: per column it picks a dtype, nullability and a *shape* +/// (random, constant, small range, runs, sorted, sparse) plus a seed, and the values follow. The +/// cascading compressor chooses its schemes from data statistics, so shapes are what steer it into +/// its edges (dict, RLE, FOR, delta, ALP, FSST, sparse patches, all-null, one-row chunks) while an +/// input stays a few bytes. +/// +/// - [#javaRoundTrip(FuzzedDataProvider)]: vortex-java reads it back. +/// - [#jniReadsJavaWrite(FuzzedDataProvider)]: vortex-jni reads it back, so a file vortex-java +/// writes that Rust reads differently, or refuses, is a finding. A Rust panic aborts the JVM, +/// which ends the fuzzing run but still names the input on stderr. +/// +/// Values compare in one canonical form: integers as `long` (unsigned ones zero-extended), floats +/// as their bit patterns (so `-0.0` and NaN are exact), booleans, strings and `null`. +@Tag("fuzz") +class WriteReadFuzzIntegrationTest { + + static { + NativeLoader.loadJni(); + } + + private static final List TYPES = List.of( + DType.I8, DType.I16, DType.I32, DType.I64, DType.U8, DType.U16, DType.U32, DType.U64, + DType.F32, DType.F64, DType.BOOL, DType.UTF8); + + @TempDir + Path tempDir; + + private Path file; + + @BeforeEach + void setUp() { + file = tempDir.resolve("fuzz.vortex"); + } + + @FuzzTest + void javaRoundTrip(FuzzedDataProvider data) throws IOException { + // Given + Table table = Table.generate(data); + table.write(file); + + // When + Map> result = read(() -> readJava(file, table), table); + + // Then + assertEquals(table.expected(), result, "vortex-java reading " + table); + } + + @FuzzTest + void jniReadsJavaWrite(FuzzedDataProvider data) throws IOException { + // Given + Table table = Table.generate(data); + table.write(file); + + // When + Map> result = read(() -> readJni(file, table), table); + + // Then + assertEquals(table.expected(), result, "vortex-jni reading " + table); + } + + @FunctionalInterface + private interface Read { + Map> values() throws IOException; + } + + /// Names the table in any read failure, so a finding says what was written. + private static Map> read(Read read, Table table) { + try { + return read.values(); + } catch (IOException | RuntimeException e) { + throw new AssertionError("reading " + table, e); + } + } + + /// One generated column: its schema entry, the array the writer takes, and the canonical values. + private record Column(ColumnName name, DType dtype, String shape, Object values, List expected) { + } + + private record Table(List columns, int rows, List chunkRows) { + + static Table generate(FuzzedDataProvider data) { + int rows = data.consumeInt(0, 3_000); + List chunkRows = new ArrayList<>(); + int left = rows; + for (int chunks = data.consumeInt(1, 3); chunks > 1 && left > 0; chunks--) { + int size = data.consumeInt(1, left); + chunkRows.add(size); + left -= size; + } + chunkRows.add(left); + List columns = new ArrayList<>(); + for (int c = data.consumeInt(1, 4); c > 0; c--) { + columns.add(column(data, ColumnName.of("c" + columns.size()), rows)); + } + return new Table(columns, rows, chunkRows); + } + + private static Column column(FuzzedDataProvider data, ColumnName name, int rows) { + DType base = data.pickValue(TYPES); + boolean nullable = data.consumeBoolean(); + Shape shape = data.pickValue(Shape.values()); + int nullEvery = nullable ? data.consumeInt(0, 4) : 0; // 0 = no nulls, 1 = all null + Random random = new Random(data.consumeLong()); + long parameter = data.consumeLong(); + List expected = new ArrayList<>(rows); + for (int i = 0; i < rows; i++) { + boolean isNull = nullEvery == 1 || nullEvery > 1 && random.nextInt(nullEvery) == 0; + expected.add(isNull ? null : canonical(base, shape.raw(random, i, parameter))); + } + DType dtype = nullable ? base.asNullable() : base; + return new Column(name, dtype, shape.name(), writerValues(dtype, expected), expected); + } + + /// The canonical value the raw shape draw stands for in `dtype`. + private static Object canonical(DType dtype, long raw) { + return switch (dtype) { + case DType.Bool _ -> (raw & 1) != 0; + case DType.Utf8 _ -> raw % 97 == 0 ? "" : "row-" + Long.toUnsignedString(raw, 36); + case DType.Primitive p -> switch (p.ptype()) { + case I8 -> (long) (byte) raw; + case I16 -> (long) (short) raw; + case I32 -> (long) (int) raw; + case U8 -> raw & 0xFFL; + case U16 -> raw & 0xFFFFL; + case U32 -> raw & 0xFFFF_FFFFL; + case F32 -> Float.floatToIntBits(floatOf(raw)); + case F64 -> Double.doubleToLongBits(doubleOf(raw)); + default -> raw; + }; + default -> throw new IllegalStateException(dtype.toString()); + }; + } + + /// Small raws become decimals (ALP's case), big ones arbitrary bit patterns (NaN, inf, -0.0). + private static double doubleOf(long raw) { + return Math.abs(raw) < 1_000_000 ? raw / 100.0 : Double.longBitsToDouble(raw); + } + + private static float floatOf(long raw) { + return Math.abs(raw) < 1_000_000 ? raw / 100.0f : Float.intBitsToFloat((int) raw); + } + + /// The array [VortexWriter#writeChunk(Map)] takes: the column type's primitive array (or + /// `String[]`), wrapped with its validity in [NullableData] when the column is nullable. + private static Object writerValues(DType dtype, List canonical) { + int n = canonical.size(); + if (dtype instanceof DType.Utf8) { + return canonical.toArray(new String[0]); + } + boolean[] valid = new boolean[n]; + long[] bits = new long[n]; + boolean[] bools = new boolean[n]; + for (int i = 0; i < n; i++) { + Object v = canonical.get(i); + valid[i] = v != null; + if (v instanceof Boolean b) { + bools[i] = b; + } else if (v != null) { + bits[i] = ((Number) v).longValue(); + } + } + Object values = dtype instanceof DType.Primitive p + ? PrimitiveArrays.fromBitsArray(bits, p.ptype(), EncodingId.VORTEX_PRIMITIVE) + : bools; + return dtype.nullable() ? new NullableData(values, valid) : values; + } + + void write(Path path) throws IOException { + DType.Struct schema = new DType.Struct(columns.stream().map(Column::name).toList(), + columns.stream().map(Column::dtype).toList(), false); + try (var channel = FileChannel.open(path, StandardOpenOption.CREATE, StandardOpenOption.WRITE, + StandardOpenOption.TRUNCATE_EXISTING); + var writer = VortexWriter.create(channel, schema, WriteOptions.defaults())) { + int from = 0; + for (int size : chunkRows) { + Map chunk = new LinkedHashMap<>(); + for (Column column : columns) { + chunk.put(column.name(), writerValues(column.dtype(), column.expected().subList(from, from + size))); + } + writer.writeChunk(chunk); + from += size; + } + } + } + + Map> expected() { + Map> expected = new LinkedHashMap<>(); + columns.forEach(c -> expected.put(c.name(), c.expected())); + return expected; + } + + @Override + public String toString() { + return rows + " rows in chunks " + chunkRows + ", columns " + columns.stream() + .map(c -> c.name() + ":" + c.dtype() + "/" + c.shape()).toList(); + } + } + + /// How a column's raw values follow from its seed and parameter. + private enum Shape { + RANDOM { + @Override + long raw(Random random, int row, long parameter) { + return random.nextLong(); + } + }, + CONSTANT { + @Override + long raw(Random random, int row, long parameter) { + return parameter; + } + }, + SMALL_RANGE { + @Override + long raw(Random random, int row, long parameter) { + return parameter + random.nextInt(1 + (int) Math.floorMod(parameter, 64L)); + } + }, + RUNS { + @Override + long raw(Random random, int row, long parameter) { + return row / (1 + Math.floorMod(parameter, 500L)); + } + }, + SORTED { + @Override + long raw(Random random, int row, long parameter) { + return parameter + (long) row * (1 + Math.floorMod(parameter, 7L)); + } + }, + SPARSE { + @Override + long raw(Random random, int row, long parameter) { + return random.nextInt(100) == 0 ? random.nextLong() : parameter; + } + }; + + abstract long raw(Random random, int row, long parameter); + } + + private static Map> readJava(Path path, Table table) throws IOException { + Map> read = new LinkedHashMap<>(); + table.columns().forEach(c -> read.put(c.name(), new ArrayList<>())); + try (VortexReader reader = VortexReader.open(path)) { + reader.scan(io.github.dfa1.vortex.reader.ScanOptions.all()).forEachRemaining(chunk -> { + for (Column column : table.columns()) { + appendJava(chunk, column, read.get(column.name())); + } + }); + } + return read; + } + + private static void appendJava(Chunk chunk, Column column, List out) { + MaskedArray.Unwrapped unwrapped = MaskedArray.unwrap(chunk.column(column.name())); + Array values = unwrapped.inner(); + BoolArray validity = unwrapped.validity(); + for (long i = 0; i < values.length(); i++) { + boolean isNull = values instanceof NullArray || validity != null && !validity.getBoolean(i); + out.add(isNull ? null : javaValue(values, i, column.dtype())); + } + } + + private static Object javaValue(Array values, long i, DType dtype) { + Object raw = switch (values) { + case LongArray a -> a.getLong(i); + case IntArray a -> (long) a.getInt(i); + case ShortArray a -> (long) a.getShort(i); + case ByteArray a -> (long) a.getByte(i); + case DoubleArray a -> Double.doubleToLongBits(a.getDouble(i)); + case FloatArray a -> Float.floatToIntBits(a.getFloat(i)); + case BoolArray a -> a.getBoolean(i); + case VarBinArray a -> a.getString(i); + default -> throw new IllegalStateException("unexpected " + values.getClass().getSimpleName() + " for " + dtype); + }; + return raw instanceof Long l ? zeroExtend(dtype, l) : raw; + } + + private static Map> readJni(Path path, Table table) throws IOException { + Map> read = new LinkedHashMap<>(); + table.columns().forEach(c -> read.put(c.name(), new ArrayList<>())); + Scan scan = DataSource.open(Session.create(), path.toAbsolutePath().toUri().toString()) + .scan(ScanOptions.builder().build()); + while (scan.hasNext()) { + Partition partition = scan.next(); + try (ArrowReader reader = partition.scanArrow(ArrowAllocation.rootAllocator())) { + while (reader.loadNextBatch()) { + VectorSchemaRoot root = reader.getVectorSchemaRoot(); + for (Column column : table.columns()) { + FieldVector vector = root.getVector(column.name().value()); + for (int i = 0; i < root.getRowCount(); i++) { + read.get(column.name()).add(vector.isNull(i) ? null : jniValue(vector.getObject(i), column.dtype())); + } + } + } + } + } + return read; + } + + private static Object jniValue(Object value, DType dtype) { + return switch (value) { + case Double d -> Double.doubleToLongBits(d); + case Float f -> Float.floatToIntBits(f); + case Boolean b -> b; + case Character c -> (long) c; + case Number n -> zeroExtend(dtype, n.longValue()); + case org.apache.arrow.vector.util.Text text -> text.toString(); + default -> value.toString(); + }; + } + + /// Java's signed readings of an unsigned column back to its canonical, zero-extended form. + private static long zeroExtend(DType dtype, long value) { + if (!(dtype instanceof DType.Primitive p)) { + return value; + } + return switch (p.ptype()) { + case U8 -> value & 0xFFL; + case U16 -> value & 0xFFFFL; + case U32 -> value & 0xFFFF_FFFFL; + default -> value; + }; + } +} diff --git a/integration/src/test/resources/io/github/dfa1/vortex/integration/WriteReadFuzzIntegrationTestInputs/jniReadsJavaWrite/crash-constant-u8-sign-extended b/integration/src/test/resources/io/github/dfa1/vortex/integration/WriteReadFuzzIntegrationTestInputs/jniReadsJavaWrite/crash-constant-u8-sign-extended new file mode 100644 index 000000000..ee0d16f0b --- /dev/null +++ b/integration/src/test/resources/io/github/dfa1/vortex/integration/WriteReadFuzzIntegrationTestInputs/jniReadsJavaWrite/crash-constant-u8-sign-extended @@ -0,0 +1 @@ +ûûÿÿ«)÷ \ No newline at end of file diff --git a/integration/src/test/resources/io/github/dfa1/vortex/integration/WriteReadFuzzIntegrationTestInputs/jniReadsJavaWrite/crash-sequence-u16-sign-extended b/integration/src/test/resources/io/github/dfa1/vortex/integration/WriteReadFuzzIntegrationTestInputs/jniReadsJavaWrite/crash-sequence-u16-sign-extended new file mode 100644 index 0000000000000000000000000000000000000000..da19a258f59ee3194e0dcaadfcae84be89ad8736 GIT binary patch literal 19 acmaF+oq?e&zv%yePOg&Firh@SJVpRp$_Nnv literal 0 HcmV?d00001 diff --git a/integration/src/test/resources/junit-platform.properties b/integration/src/test/resources/junit-platform.properties index 188453ce9..d24a88889 100644 --- a/integration/src/test/resources/junit-platform.properties +++ b/integration/src/test/resources/junit-platform.properties @@ -5,3 +5,7 @@ junit.jupiter.execution.parallel.mode.default=same_thread junit.jupiter.execution.parallel.mode.classes.default=same_thread junit.jupiter.execution.parallel.config.strategy=dynamic junit.jupiter.execution.parallel.config.dynamic.factor=1 + +# Jazzer (WriteReadFuzzIntegrationTest): instrument only this project's classes for coverage, +# not Arrow, vortex-jni or the test libraries on the classpath. +jazzer.instrument=io.github.dfa1.vortex.** From a9124f63bb0c9352d8c3238f451e162adfb3d935 Mon Sep 17 00:00:00 2001 From: Davide Angelocola Date: Sun, 18 Oct 2026 19:35:27 +0200 Subject: [PATCH 12/13] fix(writer): refuse a nullable root struct, which Rust cannot read Found reviewing the open-time struct check: a nullable struct layout carries a leading validity child (Rust's validate_children wants nfields + 1 children), but the root layout the writer builds has one child per column and no way to mark a row null. vortex-jni refuses every such file ("Assertion failed: nchildren == expected"), and since the open-time check the reader refuses it too. VortexWriter.create now rejects the schema up front. No importer writes a nullable root; writing a validity child instead would only ever say "all valid". Breaking: such a schema used to be accepted. Refs #493 Co-Authored-By: Claude Sonnet 5 Claude-Session: https://claude.ai/code/session_01WtuVGQ5h5Lk6Zn4JnkxF1K --- CHANGELOG.md | 1 + .../github/dfa1/vortex/writer/VortexWriter.java | 6 ++++++ .../dfa1/vortex/writer/VortexWriterTest.java | 17 +++++++++++++++++ 3 files changed, 24 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index ceece47dc..f395e837a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -18,6 +18,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - Custom extension types: register an `ExtensionDecoder` on `ReadRegistry` and an `ExtensionEncoder` on `WriteRegistry`, with a `validate` hook on both sides, and `Chunk.as` decodes any registered type. ([#530](https://github.com/dfa1/vortex-java/issues/530)) ### Changed +- **Breaking:** `VortexWriter.create` refuses a nullable root struct: Rust reads a nullable struct only with a validity child the writer never wrote, so such files were unreadable outside vortex-java. ([#493](https://github.com/dfa1/vortex-java/issues/493)) - **Breaking:** a file that does not embed its dtype is refused when it opens, as Rust refuses it unless the caller supplies the dtype, instead of failing the scan with `NullPointerException`. ([#493](https://github.com/dfa1/vortex-java/issues/493)) - **Breaking:** every FlatBuffer in a file is verified before it is read, as Rust verifies it, so corrupt file metadata fails with `VortexException`, and like Rust the reader now refuses tables nested more than 64 deep. ([#493](https://github.com/dfa1/vortex-java/issues/493)) - SQL `SUM` and `AVG` of a decimal column, grouped or not, are 4 to 6 times faster: they are totalled exactly as unscaled longs in the scan instead of by Calcite with a `BigDecimal` per row. ([#563](https://github.com/dfa1/vortex-java/pull/563)) diff --git a/writer/src/main/java/io/github/dfa1/vortex/writer/VortexWriter.java b/writer/src/main/java/io/github/dfa1/vortex/writer/VortexWriter.java index 6a6b2255a..a7e755efe 100644 --- a/writer/src/main/java/io/github/dfa1/vortex/writer/VortexWriter.java +++ b/writer/src/main/java/io/github/dfa1/vortex/writer/VortexWriter.java @@ -196,6 +196,12 @@ private VortexWriter( WritableByteChannel channel, DType.Struct schema, WriteOptions options, List encodings, WriteRegistry extensionRegistry ) { + // A nullable struct layout carries a leading validity child (Rust's validate_children: + // nfields + 1 children); the root layout written here has one child per column and no way + // to mark a row null, so Rust refused every such file ("nchildren == expected"). + if (schema.nullable()) { + throw new IllegalArgumentException("the root struct cannot be nullable: rows cannot be null"); + } // Wire contract, enforced by the reference writer ("StructLayout must have unique field // names"): duplicate-name schemas are constructible via the DType.Struct record (only // StructBuilder validates), so guard here — colChunks below is name-keyed and would diff --git a/writer/src/test/java/io/github/dfa1/vortex/writer/VortexWriterTest.java b/writer/src/test/java/io/github/dfa1/vortex/writer/VortexWriterTest.java index 2c24b0759..6da4455a5 100644 --- a/writer/src/test/java/io/github/dfa1/vortex/writer/VortexWriterTest.java +++ b/writer/src/test/java/io/github/dfa1/vortex/writer/VortexWriterTest.java @@ -111,6 +111,23 @@ void create_duplicateFieldNames_throwsIllegalArgumentException(@TempDir Path tmp } } + @Test + void create_nullableRootStruct_isRejected(@TempDir Path tmp) throws IOException { + // Given — a nullable root: Rust reads a nullable struct layout only with a leading validity + // child, which this writer never writes, so vortex-jni refused every such file + // ("nchildren == expected") and the reader now refuses it at open, as Rust does + var schema = new DType.Struct(List.of(ColumnName.of("c")), List.of(new DType.Primitive(PType.I64, false)), true); + Path file = tmp.resolve("nullable-root.vtx"); + + // When / Then + WriteOptions opts = WriteOptions.defaults(); + try (var ch = FileChannel.open(file, StandardOpenOption.CREATE, StandardOpenOption.WRITE)) { + assertThatThrownBy(() -> VortexWriter.create(ch, schema, opts)) + .isInstanceOf(IllegalArgumentException.class) + .hasMessageContaining("root struct cannot be nullable"); + } + } + @Test void schema_fieldNameWithNulByte_isUnbuildable() { // Given a NUL byte inside a field name: legal in our pure-Java stack, but it aborts the From 80f67ff346fa83fc932855216ad4a40cdee2299b Mon Sep 17 00:00:00 2001 From: Davide Angelocola Date: Sun, 18 Oct 2026 19:35:27 +0200 Subject: [PATCH 13/13] fix(reader): read an empty run-end array Found reviewing the child-shape rule: RunEnd's `endsCap <= 0` guard, left after capacity() took over the empty-child case, could only still fire for zero runs, and decode already refuses zero runs for rows, so it only ever rejected an empty array (0 runs, 0 rows) as "empty ends buffer for 0 run(s)". Rust builds and reads empty run-end arrays; an empty array now has nothing to validate. Refs #493 Co-Authored-By: Claude Sonnet 5 Claude-Session: https://claude.ai/code/session_01WtuVGQ5h5Lk6Zn4JnkxF1K --- CHANGELOG.md | 1 + .../reader/decode/RunEndEncodingDecoder.java | 6 +++--- .../reader/decode/RunEndEncodingDecoderTest.java | 14 ++++++++++++++ 3 files changed, 18 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index f395e837a..944c8adbf 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -47,6 +47,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - A file whose array FlatBuffer holds a corrupt offset fails the read with `VortexException` instead of `IndexOutOfBoundsException`. ([#493](https://github.com/dfa1/vortex-java/issues/493)) - A column whose dtype does not fit its bit-packed, delta or RLE encoding fails with `VortexException` instead of `ClassCastException`, and an empty RLE boolean chunk reads. ([#493](https://github.com/dfa1/vortex-java/issues/493)) - A struct dtype missing its field names or types, or a struct layout whose children do not match its fields, is rejected when the file opens, as Rust does, instead of failing the scan with `ArrayIndexOutOfBoundsException`. ([#493](https://github.com/dfa1/vortex-java/issues/493)) +- An empty `vortex.runend` array reads instead of being refused as an empty ends buffer. ([#493](https://github.com/dfa1/vortex-java/issues/493)) - A `vortex.zstd_buffers` array whose wrapped encoding id holds a control character fails with `VortexException` instead of `IllegalArgumentException`. ([#493](https://github.com/dfa1/vortex-java/issues/493)) - A constant `u8`, `u16` or `u32` column past the signed range of its width is written so vortex-jni reads it; its value was written sign-extended, which Rust refuses. ([#493](https://github.com/dfa1/vortex-java/issues/493)) - `vortex.sequence` is written only where Rust writes it: not for empty or constant data, nor for a step that overflows the column type (a decreasing unsigned column, an `i8` step of 255), and unsigned values past the signed range are written so vortex-jni reads them. ([#493](https://github.com/dfa1/vortex-java/issues/493)) diff --git a/reader/src/main/java/io/github/dfa1/vortex/reader/decode/RunEndEncodingDecoder.java b/reader/src/main/java/io/github/dfa1/vortex/reader/decode/RunEndEncodingDecoder.java index d3e9c13b9..c68da298d 100644 --- a/reader/src/main/java/io/github/dfa1/vortex/reader/decode/RunEndEncodingDecoder.java +++ b/reader/src/main/java/io/github/dfa1/vortex/reader/decode/RunEndEncodingDecoder.java @@ -147,9 +147,9 @@ private static void validateEnds(MemorySegment endsSeg, PType endsPtype, long nu PrimitiveArrays.requireUnsigned(endsPtype, EncodingId.VORTEX_RUNEND); int endsBytes = endsPtype.byteSize(); long endsCap = SegmentBroadcast.capacity(endsSeg, endsBytes, numRuns); - if (endsCap <= 0) { - throw new VortexException(EncodingId.VORTEX_RUNEND, - "runend: empty ends buffer for " + numRuns + " run(s)"); + if (numRuns == 0) { + // An empty array (decode already refused zero runs for rows): nothing to validate. + return; } long prev = PrimitiveArrays.readLong(endsSeg, 0, endsPtype, EncodingId.VORTEX_RUNEND); if (offset != 0 && prev < offset) { diff --git a/reader/src/test/java/io/github/dfa1/vortex/reader/decode/RunEndEncodingDecoderTest.java b/reader/src/test/java/io/github/dfa1/vortex/reader/decode/RunEndEncodingDecoderTest.java index d0974ca87..d4c56e4d3 100644 --- a/reader/src/test/java/io/github/dfa1/vortex/reader/decode/RunEndEncodingDecoderTest.java +++ b/reader/src/test/java/io/github/dfa1/vortex/reader/decode/RunEndEncodingDecoderTest.java @@ -243,6 +243,20 @@ void utf8WithNonVarBinValuesChild_throwsVortexException() { .hasMessageContaining("expected a VarBin values child"); } + @Test + void zeroRuns_withZeroRows_decodesEmpty() { + // Given — an empty run-end array, which Rust writes and reads; its empty ends buffer was + // refused as "empty ends buffer for 0 run(s)" + MemorySegment empty = MemorySegment.ofArray(new byte[0]); + MemorySegment[] segs = {empty, empty}; + + // When + Array result = decode(DType.I32, PType.U8, 0, 0, 0, segs, primitiveNode(0), primitiveNode(1)); + + // Then + assertThat(result.length()).isZero(); + } + private static Array decode(DType dtype, PType endsPtype, long numRuns, long offset, long n, MemorySegment[] segs, ArrayNode endsNode, ArrayNode valuesNode) { MemorySegment meta = MemorySegment.ofArray(