diff --git a/.github/workflows/fuzz.yml b/.github/workflows/fuzz.yml new file mode 100644 index 000000000..e2f342ecd --- /dev/null +++ b/.github/workflows/fuzz.yml @@ -0,0 +1,77 @@ +name: Fuzz + +# Weekly coverage-guided fuzzing (ADR 0020, #493). Every @FuzzTest replays its saved inputs in the +# ordinary build; this job explores new ones, one target per matrix job because Jazzer fuzzes a +# single target per JVM. Scheduled, not on the PR path: a run is unbounded exploration, so a +# failure is a finding to triage (the crash input is uploaded; commit it next to the target's +# saved inputs with the fix), not a broken build. + +on: + schedule: + - cron: '0 3 * * 6' # Saturdays 03:00 UTC + workflow_dispatch: + inputs: + duration: + description: 'Fuzzing time per target (e.g. 10m, 1h)' + default: '30m' + +jobs: + fuzz: + runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + include: + - { module: fuzz, target: 'VortexReaderFuzzTest#openMalformedFile' } + - { module: fuzz, target: 'VortexReaderFuzzTest#scanMalformedFile' } + - { module: fuzz, target: 'VortexReaderFuzzTest#filterMalformedFile' } + - { module: fuzz, target: 'VortexReaderFuzzTest#parseMetadataBlobs' } + - { module: fuzz, target: 'FsstFuzzTest#compressDecompressRoundTrip' } + - { module: integration, target: 'WriteReadFuzzIntegrationTest#javaRoundTrip' } + - { module: integration, target: 'WriteReadFuzzIntegrationTest#jniReadsJavaWrite' } + name: ${{ matrix.target }} + + steps: + - uses: actions/checkout@v7 + + - name: Set up Azul Zulu JDK 25 + uses: actions/setup-java@v6.0.1 + with: + distribution: zulu + java-version: '25' + + - name: Cache Maven repository + uses: actions/cache@v6 + with: + path: ~/.m2/repository + key: ${{ runner.os }}-maven-${{ hashFiles('**/pom.xml') }} + restore-keys: | + ${{ runner.os }}-maven- + + - name: Fuzz + env: + JAZZER_FUZZ: '1' + JAZZER_MAX_DURATION: ${{ inputs.duration || '30m' }} + run: | + if [ "${{ matrix.module }}" = integration ]; then + ./mvnw -B verify -pl integration -am -Dit.test='${{ matrix.target }}' \ + -Dtest=none -Dsurefire.failIfNoSpecifiedTests=false + else + ./mvnw -B test -pl fuzz -am -Dtest='${{ matrix.target }}' \ + -Dsurefire.failIfNoSpecifiedTests=false + fi + + # Jazzer writes a finding next to the target's saved inputs; the committed crash-* inputs + # are regressions already fixed, so only the new, untracked ones are collected. + - name: Collect the new crash input + if: failure() + run: | + mkdir -p findings + git ls-files --others --exclude-standard | grep '/crash-' | xargs -r -I{} cp --parents {} findings/ + + - name: Upload the crash input + if: failure() + uses: actions/upload-artifact@v7 + with: + name: crash-${{ strategy.job-index }} + path: findings/ diff --git a/CHANGELOG.md b/CHANGELOG.md index d2c62968f..944c8adbf 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -18,6 +18,9 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - Custom extension types: register an `ExtensionDecoder` on `ReadRegistry` and an `ExtensionEncoder` on `WriteRegistry`, with a `validate` hook on both sides, and `Chunk.as` decodes any registered type. ([#530](https://github.com/dfa1/vortex-java/issues/530)) ### Changed +- **Breaking:** `VortexWriter.create` refuses a nullable root struct: Rust reads a nullable struct only with a validity child the writer never wrote, so such files were unreadable outside vortex-java. ([#493](https://github.com/dfa1/vortex-java/issues/493)) +- **Breaking:** a file that does not embed its dtype is refused when it opens, as Rust refuses it unless the caller supplies the dtype, instead of failing the scan with `NullPointerException`. ([#493](https://github.com/dfa1/vortex-java/issues/493)) +- **Breaking:** every FlatBuffer in a file is verified before it is read, as Rust verifies it, so corrupt file metadata fails with `VortexException`, and like Rust the reader now refuses tables nested more than 64 deep. ([#493](https://github.com/dfa1/vortex-java/issues/493)) - SQL `SUM` and `AVG` of a decimal column, grouped or not, are 4 to 6 times faster: they are totalled exactly as unscaled longs in the scan instead of by Calcite with a `BigDecimal` per row. ([#563](https://github.com/dfa1/vortex-java/pull/563)) - SQL `COUNT(DISTINCT col)` is up to 4 times faster: distinct values are counted in the scan, integers by their bits in a primitive map, instead of Calcite hashing a boxed value per row. ([#562](https://github.com/dfa1/vortex-java/pull/562)) - A SQL `WHERE` comparing a `SMALLINT` or `TINYINT` column with a literal is pushed into the scan and its aggregates, instead of being left to Calcite because of the cast Calcite adds. ([#560](https://github.com/dfa1/vortex-java/pull/560)) @@ -41,6 +44,14 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - **Breaking:** `ExtensionId` is a sealed `WellKnown`/`Custom` type like `EncodingId`, so third-party extension ids can be named, and `ExtensionId.parse` returns the id instead of an `Optional`. ([#530](https://github.com/dfa1/vortex-java/issues/530)) ### Fixed +- A file whose array FlatBuffer holds a corrupt offset fails the read with `VortexException` instead of `IndexOutOfBoundsException`. ([#493](https://github.com/dfa1/vortex-java/issues/493)) +- A column whose dtype does not fit its bit-packed, delta or RLE encoding fails with `VortexException` instead of `ClassCastException`, and an empty RLE boolean chunk reads. ([#493](https://github.com/dfa1/vortex-java/issues/493)) +- A struct dtype missing its field names or types, or a struct layout whose children do not match its fields, is rejected when the file opens, as Rust does, instead of failing the scan with `ArrayIndexOutOfBoundsException`. ([#493](https://github.com/dfa1/vortex-java/issues/493)) +- An empty `vortex.runend` array reads instead of being refused as an empty ends buffer. ([#493](https://github.com/dfa1/vortex-java/issues/493)) +- A `vortex.zstd_buffers` array whose wrapped encoding id holds a control character fails with `VortexException` instead of `IllegalArgumentException`. ([#493](https://github.com/dfa1/vortex-java/issues/493)) +- A constant `u8`, `u16` or `u32` column past the signed range of its width is written so vortex-jni reads it; its value was written sign-extended, which Rust refuses. ([#493](https://github.com/dfa1/vortex-java/issues/493)) +- `vortex.sequence` is written only where Rust writes it: not for empty or constant data, nor for a step that overflows the column type (a decreasing unsigned column, an `i8` step of 255), and unsigned values past the signed range are written so vortex-jni reads them. ([#493](https://github.com/dfa1/vortex-java/issues/493)) +- A file whose encoded child holds fewer values than its array needs, other than the single value of a constant child, fails with `VortexException` instead of dividing by zero or silently repeating values. ([#493](https://github.com/dfa1/vortex-java/issues/493)) - SQL `WHERE` on an `I16`, `U8` or `I8` column no longer fails with a `ClassCastException`. ([#556](https://github.com/dfa1/vortex-java/pull/556)) - SQL `COUNT(DISTINCT col)` and aggregates with a `FILTER (WHERE ...)` clause return the right answer; the statistics push-down returned the row count and ignored the filter. ([#554](https://github.com/dfa1/vortex-java/pull/554)) - SQL `MIN`/`MAX` of a string column whose values fall outside Latin-1 answers instead of failing the query. ([#554](https://github.com/dfa1/vortex-java/pull/554)) diff --git a/CLAUDE.md b/CLAUDE.md index 09422ee36..103696d2a 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -84,8 +84,9 @@ and merge commits are disabled): squash a PR's commits locally into one, force-p scripts/simd-benchmark.sh # auto-vectorized vs Vector API kernels, table of speedups (also a CI job) scripts/hydrate-raincloud-corpus.sh --max-mb 200 # hydrate real-world conformance corpus (#205), then: ./mvnw verify -pl integration -am -Dvortex.it.excludedGroups= -Dit.test="RaincloudConformanceIntegrationTest" -./mvnw test -pl fuzz -am -Dvortex.fuzz.excludedGroups= # Jazzer regression mode (ADR 0020) -JAZZER_FUZZ=1 ./mvnw test -pl fuzz -am -Dvortex.fuzz.excludedGroups= # actually fuzz (unbounded) +JAZZER_FUZZ=1 JAZZER_MAX_DURATION=5m ./mvnw test -pl fuzz -am -Dtest='VortexReaderFuzzTest#scanMalformedFile' -Dsurefire.failIfNoSpecifiedTests=false # fuzz one target (ADR 0020) +JAZZER_FUZZ=1 JAZZER_MAX_DURATION=5m ./mvnw verify -pl integration -am -Dit.test='WriteReadFuzzIntegrationTest#jniReadsJavaWrite' -Dtest=none -Dsurefire.failIfNoSpecifiedTests=false +./mvnw test -pl fuzz -am -Dtest=SeedCorpusGenerator -Dsurefire.failIfNoSpecifiedTests=false -Dvortex.fuzz.regenerateSeeds=true # rewrite per-encoding seeds ``` Tag-excluded from a routine local build, even when named with `-Dit.test`/`-Dtest`: @@ -95,9 +96,16 @@ Tag-excluded from a routine local build, even when named with `-Dit.test`/`-Dtes integration test taking over ~1 s. - **Raincloud corpus** (`@Tag("raincloud")`, failsafe): clear the exclusion with `-Dvortex.it.excludedGroups=` as shown above. Not run by the `ci` profile. -- **Jazzer fuzz** (`@Tag("fuzz")`, own `fuzz` module): clear it with `-Dvortex.fuzz.excludedGroups=`. - Without `JAZZER_FUZZ=1` it only replays the saved corpus; with it, it fuzzes until stopped — - never in CI or unattended. See [ADR 0020](adr/0020-jazzer-fuzz-infrastructure.md). + +Jazzer fuzz targets (`@FuzzTest`, ADR 0020) are **not** excluded: without `JAZZER_FUZZ=1` each +replays its saved inputs (`*Inputs//`: seeds plus every past finding) in about a second, +as part of the routine build. `JAZZER_FUZZ=1` explores new inputs, one target per run, bounded by +`JAZZER_MAX_DURATION`; the weekly `Fuzz` workflow does that for every target. Targets: reader +`open`/`scan`/`filter`/metadata-blob and FSST round-trip in `fuzz`; write→read round-trip and the +vortex-jni differential in `integration` (`WriteReadFuzzIntegrationTest`). A finding is written +next to the target's inputs as `crash-`: rename it after the bug, fix, and commit it with +the fix, plus a `MalformedFileInteropIntegrationTest` case when it is a file vortex-jni also +reads. Regenerate after editing `.fbs`/`.proto` (both generators are in-house, no external tools): @@ -110,7 +118,10 @@ Both schema languages are compiled in-process to MemorySegment-native Java, with `flatc`/`protoc` and no `com.google.flatbuffers`/`protobuf-java` runtime (ADR 0017): - **`.fbs` → `fbs-gen`** (`io.github.dfa1.vortex.fbsgen`): generates readers extending `FbsTable` (vtable-based) or `FbsMemorySegment` (fixed-offset inline) and builders over - `FbsBuilder`, all in the same generated package `io.github.dfa1.vortex.core.fbs`. The + `FbsBuilder`, all in the same generated package `io.github.dfa1.vortex.core.fbs`. Every + generated `getRootAsX` first runs `FbsVerifier` over the whole buffer (Rust's + `flatbuffers::root`: bounds, vtables, required fields, ≤ 64 nested tables, ≤ 1M tables), so + reader code never guards FlatBuffer offsets itself. The runtime base classes `FbsTable`/`FbsMemorySegment` are package-private (only generated readers extend them); `FbsBuilder` is public because the writer module assembles FlatBuffers with it. Schema names with a trailing `_` (e.g. `Struct_`) have the underscore stripped in @@ -242,7 +253,8 @@ almost always this. The reader memory-maps and parses untrusted binary input. Every malformed input must throw `VortexException`, never `ArrayIndexOutOfBoundsException`, `NegativeArraySizeException`, `OutOfMemoryError`, `StackOverflowError`, a raw FlatBuffer runtime exception, or a Protobuf parser -exception. See the [`security` issues](https://github.com/dfa1/vortex-java/labels/security) for the current gap list (per-encoding adversarial +exception. Reject input where it enters, as Rust does: FlatBuffers are verified whole when parsed, +dtype and layout tree shape at open, and only segment contents at decode. See the [`security` issues](https://github.com/dfa1/vortex-java/labels/security) for the current gap list (per-encoding adversarial tests, resource caps, fuzz infra) and [ADR 0003](adr/0003-vortex-exception-sanitization.md) / [ADR 0004](adr/0004-resource-caps-read-options.md) for the design. diff --git a/core/src/main/java/io/github/dfa1/vortex/core/fbs/FbsArray.java b/core/src/main/java/io/github/dfa1/vortex/core/fbs/FbsArray.java index 22957a215..2dc458754 100644 --- a/core/src/main/java/io/github/dfa1/vortex/core/fbs/FbsArray.java +++ b/core/src/main/java/io/github/dfa1/vortex/core/fbs/FbsArray.java @@ -9,11 +9,22 @@ @Generated("io.github.dfa1.vortex.fbsgen.CodeGen") public final class FbsArray extends FbsTable { - /// Positions a reader at the root table of a finished buffer. + /// Verifies a finished buffer and positions a reader at its root table. /// @param seg the buffer /// @return a reader positioned at the root + /// @throws io.github.dfa1.vortex.core.error.VortexException if the buffer is malformed public static FbsArray getRootAsFbsArray(MemorySegment seg) { - return new FbsArray().assign(seg, FbsTable.rootPosition(seg, 0)); + return new FbsArray().assign(seg, FbsVerifier.verifyRoot(seg, FbsArray::verify)); + } + + /// Checks this table and everything reachable from it. + /// @param v the verifier walking the buffer + /// @param pos the table's absolute position + static void verify(FbsVerifier v, long pos) { + v.startTable(pos); + v.table(pos, 4, "root", false, FbsArrayNode::verify); + v.vector(pos, 6, 8, "buffers", false); + v.endTable(); } /// Positions this reader at a table. @@ -37,6 +48,12 @@ public int buffersLength() { return o != 0 ? vectorLength(o) : 0; } + /// Tells an absent `buffers` vector from an empty one, which [#buffersLength()] cannot. + /// @return whether the `buffers` vector is present + public boolean hasBuffers() { + return fieldOffset(6) != 0; + } + /// @param j element index /// @return the j-th `buffers` element public FbsBuffer buffers(int j) { diff --git a/core/src/main/java/io/github/dfa1/vortex/core/fbs/FbsArrayNode.java b/core/src/main/java/io/github/dfa1/vortex/core/fbs/FbsArrayNode.java index 8e2ffeba7..bbbc5665a 100644 --- a/core/src/main/java/io/github/dfa1/vortex/core/fbs/FbsArrayNode.java +++ b/core/src/main/java/io/github/dfa1/vortex/core/fbs/FbsArrayNode.java @@ -9,11 +9,25 @@ @Generated("io.github.dfa1.vortex.fbsgen.CodeGen") public final class FbsArrayNode extends FbsTable { - /// Positions a reader at the root table of a finished buffer. + /// Verifies a finished buffer and positions a reader at its root table. /// @param seg the buffer /// @return a reader positioned at the root + /// @throws io.github.dfa1.vortex.core.error.VortexException if the buffer is malformed public static FbsArrayNode getRootAsFbsArrayNode(MemorySegment seg) { - return new FbsArrayNode().assign(seg, FbsTable.rootPosition(seg, 0)); + return new FbsArrayNode().assign(seg, FbsVerifier.verifyRoot(seg, FbsArrayNode::verify)); + } + + /// Checks this table and everything reachable from it. + /// @param v the verifier walking the buffer + /// @param pos the table's absolute position + static void verify(FbsVerifier v, long pos) { + v.startTable(pos); + v.inline(pos, 4, 2, "encoding", false); + v.vector(pos, 6, 1, "metadata", false); + v.tableVector(pos, 8, "children", false, FbsArrayNode::verify); + v.vector(pos, 10, 2, "buffers", false); + v.table(pos, 12, "stats", false, FbsArrayStats::verify); + v.endTable(); } /// Positions this reader at a table. @@ -37,6 +51,12 @@ public int metadataLength() { return o != 0 ? vectorLength(o) : 0; } + /// Tells an absent `metadata` vector from an empty one, which [#metadataLength()] cannot. + /// @return whether the `metadata` vector is present + public boolean hasMetadata() { + return fieldOffset(6) != 0; + } + /// @param j element index /// @return the j-th `metadata` element public int metadata(int j) { @@ -56,6 +76,12 @@ public int childrenLength() { return o != 0 ? vectorLength(o) : 0; } + /// Tells an absent `children` vector from an empty one, which [#childrenLength()] cannot. + /// @return whether the `children` vector is present + public boolean hasChildren() { + return fieldOffset(8) != 0; + } + /// @param j element index /// @return the j-th `children` element public FbsArrayNode children(int j) { @@ -69,6 +95,12 @@ public int buffersLength() { return o != 0 ? vectorLength(o) : 0; } + /// Tells an absent `buffers` vector from an empty one, which [#buffersLength()] cannot. + /// @return whether the `buffers` vector is present + public boolean hasBuffers() { + return fieldOffset(10) != 0; + } + /// @param j element index /// @return the j-th `buffers` element public int buffers(int j) { diff --git a/core/src/main/java/io/github/dfa1/vortex/core/fbs/FbsArraySpec.java b/core/src/main/java/io/github/dfa1/vortex/core/fbs/FbsArraySpec.java index 3c231ed24..30e94389d 100644 --- a/core/src/main/java/io/github/dfa1/vortex/core/fbs/FbsArraySpec.java +++ b/core/src/main/java/io/github/dfa1/vortex/core/fbs/FbsArraySpec.java @@ -9,11 +9,21 @@ @Generated("io.github.dfa1.vortex.fbsgen.CodeGen") public final class FbsArraySpec extends FbsTable { - /// Positions a reader at the root table of a finished buffer. + /// Verifies a finished buffer and positions a reader at its root table. /// @param seg the buffer /// @return a reader positioned at the root + /// @throws io.github.dfa1.vortex.core.error.VortexException if the buffer is malformed public static FbsArraySpec getRootAsFbsArraySpec(MemorySegment seg) { - return new FbsArraySpec().assign(seg, FbsTable.rootPosition(seg, 0)); + return new FbsArraySpec().assign(seg, FbsVerifier.verifyRoot(seg, FbsArraySpec::verify)); + } + + /// Checks this table and everything reachable from it. + /// @param v the verifier walking the buffer + /// @param pos the table's absolute position + static void verify(FbsVerifier v, long pos) { + v.startTable(pos); + v.string(pos, 4, "id", true); + v.endTable(); } /// Positions this reader at a table. diff --git a/core/src/main/java/io/github/dfa1/vortex/core/fbs/FbsArrayStats.java b/core/src/main/java/io/github/dfa1/vortex/core/fbs/FbsArrayStats.java index eb8da4948..b97094a44 100644 --- a/core/src/main/java/io/github/dfa1/vortex/core/fbs/FbsArrayStats.java +++ b/core/src/main/java/io/github/dfa1/vortex/core/fbs/FbsArrayStats.java @@ -9,11 +9,31 @@ @Generated("io.github.dfa1.vortex.fbsgen.CodeGen") public final class FbsArrayStats extends FbsTable { - /// Positions a reader at the root table of a finished buffer. + /// Verifies a finished buffer and positions a reader at its root table. /// @param seg the buffer /// @return a reader positioned at the root + /// @throws io.github.dfa1.vortex.core.error.VortexException if the buffer is malformed public static FbsArrayStats getRootAsFbsArrayStats(MemorySegment seg) { - return new FbsArrayStats().assign(seg, FbsTable.rootPosition(seg, 0)); + return new FbsArrayStats().assign(seg, FbsVerifier.verifyRoot(seg, FbsArrayStats::verify)); + } + + /// Checks this table and everything reachable from it. + /// @param v the verifier walking the buffer + /// @param pos the table's absolute position + static void verify(FbsVerifier v, long pos) { + v.startTable(pos); + v.vector(pos, 4, 1, "min", false); + v.inline(pos, 6, 1, "min_precision", false); + v.vector(pos, 8, 1, "max", false); + v.inline(pos, 10, 1, "max_precision", false); + v.vector(pos, 12, 1, "sum", false); + v.inline(pos, 14, 1, "is_sorted", false); + v.inline(pos, 16, 1, "is_strict_sorted", false); + v.inline(pos, 18, 1, "is_constant", false); + v.inline(pos, 20, 8, "null_count", false); + v.inline(pos, 22, 8, "uncompressed_size_in_bytes", false); + v.inline(pos, 24, 8, "nan_count", false); + v.endTable(); } /// Positions this reader at a table. @@ -31,6 +51,12 @@ public int minLength() { return o != 0 ? vectorLength(o) : 0; } + /// Tells an absent `min` vector from an empty one, which [#minLength()] cannot. + /// @return whether the `min` vector is present + public boolean hasMin() { + return fieldOffset(4) != 0; + } + /// @param j element index /// @return the j-th `min` element public int min(int j) { @@ -56,6 +82,12 @@ public int maxLength() { return o != 0 ? vectorLength(o) : 0; } + /// Tells an absent `max` vector from an empty one, which [#maxLength()] cannot. + /// @return whether the `max` vector is present + public boolean hasMax() { + return fieldOffset(8) != 0; + } + /// @param j element index /// @return the j-th `max` element public int max(int j) { @@ -81,6 +113,12 @@ public int sumLength() { return o != 0 ? vectorLength(o) : 0; } + /// Tells an absent `sum` vector from an empty one, which [#sumLength()] cannot. + /// @return whether the `sum` vector is present + public boolean hasSum() { + return fieldOffset(12) != 0; + } + /// @param j element index /// @return the j-th `sum` element public int sum(int j) { diff --git a/core/src/main/java/io/github/dfa1/vortex/core/fbs/FbsBinary.java b/core/src/main/java/io/github/dfa1/vortex/core/fbs/FbsBinary.java index 90bfb9b86..5eecef674 100644 --- a/core/src/main/java/io/github/dfa1/vortex/core/fbs/FbsBinary.java +++ b/core/src/main/java/io/github/dfa1/vortex/core/fbs/FbsBinary.java @@ -9,11 +9,21 @@ @Generated("io.github.dfa1.vortex.fbsgen.CodeGen") public final class FbsBinary extends FbsTable { - /// Positions a reader at the root table of a finished buffer. + /// Verifies a finished buffer and positions a reader at its root table. /// @param seg the buffer /// @return a reader positioned at the root + /// @throws io.github.dfa1.vortex.core.error.VortexException if the buffer is malformed public static FbsBinary getRootAsFbsBinary(MemorySegment seg) { - return new FbsBinary().assign(seg, FbsTable.rootPosition(seg, 0)); + return new FbsBinary().assign(seg, FbsVerifier.verifyRoot(seg, FbsBinary::verify)); + } + + /// Checks this table and everything reachable from it. + /// @param v the verifier walking the buffer + /// @param pos the table's absolute position + static void verify(FbsVerifier v, long pos) { + v.startTable(pos); + v.inline(pos, 4, 1, "nullable", false); + v.endTable(); } /// Positions this reader at a table. diff --git a/core/src/main/java/io/github/dfa1/vortex/core/fbs/FbsBool.java b/core/src/main/java/io/github/dfa1/vortex/core/fbs/FbsBool.java index 85646d6d2..cff207a9b 100644 --- a/core/src/main/java/io/github/dfa1/vortex/core/fbs/FbsBool.java +++ b/core/src/main/java/io/github/dfa1/vortex/core/fbs/FbsBool.java @@ -9,11 +9,21 @@ @Generated("io.github.dfa1.vortex.fbsgen.CodeGen") public final class FbsBool extends FbsTable { - /// Positions a reader at the root table of a finished buffer. + /// Verifies a finished buffer and positions a reader at its root table. /// @param seg the buffer /// @return a reader positioned at the root + /// @throws io.github.dfa1.vortex.core.error.VortexException if the buffer is malformed public static FbsBool getRootAsFbsBool(MemorySegment seg) { - return new FbsBool().assign(seg, FbsTable.rootPosition(seg, 0)); + return new FbsBool().assign(seg, FbsVerifier.verifyRoot(seg, FbsBool::verify)); + } + + /// Checks this table and everything reachable from it. + /// @param v the verifier walking the buffer + /// @param pos the table's absolute position + static void verify(FbsVerifier v, long pos) { + v.startTable(pos); + v.inline(pos, 4, 1, "nullable", false); + v.endTable(); } /// Positions this reader at a table. diff --git a/core/src/main/java/io/github/dfa1/vortex/core/fbs/FbsCompressionSpec.java b/core/src/main/java/io/github/dfa1/vortex/core/fbs/FbsCompressionSpec.java index b287a3572..af2d6c072 100644 --- a/core/src/main/java/io/github/dfa1/vortex/core/fbs/FbsCompressionSpec.java +++ b/core/src/main/java/io/github/dfa1/vortex/core/fbs/FbsCompressionSpec.java @@ -9,11 +9,21 @@ @Generated("io.github.dfa1.vortex.fbsgen.CodeGen") public final class FbsCompressionSpec extends FbsTable { - /// Positions a reader at the root table of a finished buffer. + /// Verifies a finished buffer and positions a reader at its root table. /// @param seg the buffer /// @return a reader positioned at the root + /// @throws io.github.dfa1.vortex.core.error.VortexException if the buffer is malformed public static FbsCompressionSpec getRootAsFbsCompressionSpec(MemorySegment seg) { - return new FbsCompressionSpec().assign(seg, FbsTable.rootPosition(seg, 0)); + return new FbsCompressionSpec().assign(seg, FbsVerifier.verifyRoot(seg, FbsCompressionSpec::verify)); + } + + /// Checks this table and everything reachable from it. + /// @param v the verifier walking the buffer + /// @param pos the table's absolute position + static void verify(FbsVerifier v, long pos) { + v.startTable(pos); + v.inline(pos, 4, 1, "scheme", false); + v.endTable(); } /// Positions this reader at a table. diff --git a/core/src/main/java/io/github/dfa1/vortex/core/fbs/FbsDType.java b/core/src/main/java/io/github/dfa1/vortex/core/fbs/FbsDType.java index 54e8bae42..5bfaf20f0 100644 --- a/core/src/main/java/io/github/dfa1/vortex/core/fbs/FbsDType.java +++ b/core/src/main/java/io/github/dfa1/vortex/core/fbs/FbsDType.java @@ -9,11 +9,41 @@ @Generated("io.github.dfa1.vortex.fbsgen.CodeGen") public final class FbsDType extends FbsTable { - /// Positions a reader at the root table of a finished buffer. + /// Verifies a finished buffer and positions a reader at its root table. /// @param seg the buffer /// @return a reader positioned at the root + /// @throws io.github.dfa1.vortex.core.error.VortexException if the buffer is malformed public static FbsDType getRootAsFbsDType(MemorySegment seg) { - return new FbsDType().assign(seg, FbsTable.rootPosition(seg, 0)); + return new FbsDType().assign(seg, FbsVerifier.verifyRoot(seg, FbsDType::verify)); + } + + /// Checks this table and everything reachable from it. + /// @param v the verifier walking the buffer + /// @param pos the table's absolute position + static void verify(FbsVerifier v, long pos) { + v.startTable(pos); + int typeType = v.unionType(pos, 4, 6, "type"); + if (typeType != 0) { + long member = v.unionValue(pos, 6, "type"); + switch (typeType) { + case 1 -> FbsNull.verify(v, member); + case 2 -> FbsBool.verify(v, member); + case 3 -> FbsPrimitive.verify(v, member); + case 4 -> FbsDecimal.verify(v, member); + case 5 -> FbsUtf8.verify(v, member); + case 6 -> FbsBinary.verify(v, member); + case 7 -> FbsStruct.verify(v, member); + case 8 -> FbsList.verify(v, member); + case 9 -> FbsExtension.verify(v, member); + case 10 -> FbsFixedSizeList.verify(v, member); + case 11 -> FbsVariant.verify(v, member); + case 12 -> FbsUnion.verify(v, member); + case 13 -> FbsMap.verify(v, member); + default -> { + } + } + } + v.endTable(); } /// Positions this reader at a table. diff --git a/core/src/main/java/io/github/dfa1/vortex/core/fbs/FbsDecimal.java b/core/src/main/java/io/github/dfa1/vortex/core/fbs/FbsDecimal.java index 2fa2d2f24..a7c60f20a 100644 --- a/core/src/main/java/io/github/dfa1/vortex/core/fbs/FbsDecimal.java +++ b/core/src/main/java/io/github/dfa1/vortex/core/fbs/FbsDecimal.java @@ -9,11 +9,23 @@ @Generated("io.github.dfa1.vortex.fbsgen.CodeGen") public final class FbsDecimal extends FbsTable { - /// Positions a reader at the root table of a finished buffer. + /// Verifies a finished buffer and positions a reader at its root table. /// @param seg the buffer /// @return a reader positioned at the root + /// @throws io.github.dfa1.vortex.core.error.VortexException if the buffer is malformed public static FbsDecimal getRootAsFbsDecimal(MemorySegment seg) { - return new FbsDecimal().assign(seg, FbsTable.rootPosition(seg, 0)); + return new FbsDecimal().assign(seg, FbsVerifier.verifyRoot(seg, FbsDecimal::verify)); + } + + /// Checks this table and everything reachable from it. + /// @param v the verifier walking the buffer + /// @param pos the table's absolute position + static void verify(FbsVerifier v, long pos) { + v.startTable(pos); + v.inline(pos, 4, 1, "precision", false); + v.inline(pos, 6, 1, "scale", false); + v.inline(pos, 8, 1, "nullable", false); + v.endTable(); } /// Positions this reader at a table. diff --git a/core/src/main/java/io/github/dfa1/vortex/core/fbs/FbsEncryptionSpec.java b/core/src/main/java/io/github/dfa1/vortex/core/fbs/FbsEncryptionSpec.java index 01cbbdd0f..039cb68d1 100644 --- a/core/src/main/java/io/github/dfa1/vortex/core/fbs/FbsEncryptionSpec.java +++ b/core/src/main/java/io/github/dfa1/vortex/core/fbs/FbsEncryptionSpec.java @@ -9,11 +9,20 @@ @Generated("io.github.dfa1.vortex.fbsgen.CodeGen") public final class FbsEncryptionSpec extends FbsTable { - /// Positions a reader at the root table of a finished buffer. + /// Verifies a finished buffer and positions a reader at its root table. /// @param seg the buffer /// @return a reader positioned at the root + /// @throws io.github.dfa1.vortex.core.error.VortexException if the buffer is malformed public static FbsEncryptionSpec getRootAsFbsEncryptionSpec(MemorySegment seg) { - return new FbsEncryptionSpec().assign(seg, FbsTable.rootPosition(seg, 0)); + return new FbsEncryptionSpec().assign(seg, FbsVerifier.verifyRoot(seg, FbsEncryptionSpec::verify)); + } + + /// Checks this table and everything reachable from it. + /// @param v the verifier walking the buffer + /// @param pos the table's absolute position + static void verify(FbsVerifier v, long pos) { + v.startTable(pos); + v.endTable(); } /// Positions this reader at a table. diff --git a/core/src/main/java/io/github/dfa1/vortex/core/fbs/FbsExtension.java b/core/src/main/java/io/github/dfa1/vortex/core/fbs/FbsExtension.java index 7c84a107e..4d2e3eb49 100644 --- a/core/src/main/java/io/github/dfa1/vortex/core/fbs/FbsExtension.java +++ b/core/src/main/java/io/github/dfa1/vortex/core/fbs/FbsExtension.java @@ -9,11 +9,23 @@ @Generated("io.github.dfa1.vortex.fbsgen.CodeGen") public final class FbsExtension extends FbsTable { - /// Positions a reader at the root table of a finished buffer. + /// Verifies a finished buffer and positions a reader at its root table. /// @param seg the buffer /// @return a reader positioned at the root + /// @throws io.github.dfa1.vortex.core.error.VortexException if the buffer is malformed public static FbsExtension getRootAsFbsExtension(MemorySegment seg) { - return new FbsExtension().assign(seg, FbsTable.rootPosition(seg, 0)); + return new FbsExtension().assign(seg, FbsVerifier.verifyRoot(seg, FbsExtension::verify)); + } + + /// Checks this table and everything reachable from it. + /// @param v the verifier walking the buffer + /// @param pos the table's absolute position + static void verify(FbsVerifier v, long pos) { + v.startTable(pos); + v.string(pos, 4, "id", false); + v.table(pos, 6, "storage_dtype", false, FbsDType::verify); + v.vector(pos, 8, 1, "metadata", false); + v.endTable(); } /// Positions this reader at a table. @@ -43,6 +55,12 @@ public int metadataLength() { return o != 0 ? vectorLength(o) : 0; } + /// Tells an absent `metadata` vector from an empty one, which [#metadataLength()] cannot. + /// @return whether the `metadata` vector is present + public boolean hasMetadata() { + return fieldOffset(8) != 0; + } + /// @param j element index /// @return the j-th `metadata` element public int metadata(int j) { diff --git a/core/src/main/java/io/github/dfa1/vortex/core/fbs/FbsFileStatistics.java b/core/src/main/java/io/github/dfa1/vortex/core/fbs/FbsFileStatistics.java index 206602aa6..4a24e0ecb 100644 --- a/core/src/main/java/io/github/dfa1/vortex/core/fbs/FbsFileStatistics.java +++ b/core/src/main/java/io/github/dfa1/vortex/core/fbs/FbsFileStatistics.java @@ -9,11 +9,21 @@ @Generated("io.github.dfa1.vortex.fbsgen.CodeGen") public final class FbsFileStatistics extends FbsTable { - /// Positions a reader at the root table of a finished buffer. + /// Verifies a finished buffer and positions a reader at its root table. /// @param seg the buffer /// @return a reader positioned at the root + /// @throws io.github.dfa1.vortex.core.error.VortexException if the buffer is malformed public static FbsFileStatistics getRootAsFbsFileStatistics(MemorySegment seg) { - return new FbsFileStatistics().assign(seg, FbsTable.rootPosition(seg, 0)); + return new FbsFileStatistics().assign(seg, FbsVerifier.verifyRoot(seg, FbsFileStatistics::verify)); + } + + /// Checks this table and everything reachable from it. + /// @param v the verifier walking the buffer + /// @param pos the table's absolute position + static void verify(FbsVerifier v, long pos) { + v.startTable(pos); + v.tableVector(pos, 4, "field_stats", false, FbsArrayStats::verify); + v.endTable(); } /// Positions this reader at a table. @@ -31,6 +41,12 @@ public int fieldStatsLength() { return o != 0 ? vectorLength(o) : 0; } + /// Tells an absent `fieldStats` vector from an empty one, which [#fieldStatsLength()] cannot. + /// @return whether the `fieldStats` vector is present + public boolean hasFieldStats() { + return fieldOffset(4) != 0; + } + /// @param j element index /// @return the j-th `fieldStats` element public FbsArrayStats fieldStats(int j) { diff --git a/core/src/main/java/io/github/dfa1/vortex/core/fbs/FbsFixedSizeList.java b/core/src/main/java/io/github/dfa1/vortex/core/fbs/FbsFixedSizeList.java index 1c0c5c83b..608e0a4e6 100644 --- a/core/src/main/java/io/github/dfa1/vortex/core/fbs/FbsFixedSizeList.java +++ b/core/src/main/java/io/github/dfa1/vortex/core/fbs/FbsFixedSizeList.java @@ -9,11 +9,23 @@ @Generated("io.github.dfa1.vortex.fbsgen.CodeGen") public final class FbsFixedSizeList extends FbsTable { - /// Positions a reader at the root table of a finished buffer. + /// Verifies a finished buffer and positions a reader at its root table. /// @param seg the buffer /// @return a reader positioned at the root + /// @throws io.github.dfa1.vortex.core.error.VortexException if the buffer is malformed public static FbsFixedSizeList getRootAsFbsFixedSizeList(MemorySegment seg) { - return new FbsFixedSizeList().assign(seg, FbsTable.rootPosition(seg, 0)); + return new FbsFixedSizeList().assign(seg, FbsVerifier.verifyRoot(seg, FbsFixedSizeList::verify)); + } + + /// Checks this table and everything reachable from it. + /// @param v the verifier walking the buffer + /// @param pos the table's absolute position + static void verify(FbsVerifier v, long pos) { + v.startTable(pos); + v.table(pos, 4, "element_type", false, FbsDType::verify); + v.inline(pos, 6, 4, "size", false); + v.inline(pos, 8, 1, "nullable", false); + v.endTable(); } /// Positions this reader at a table. diff --git a/core/src/main/java/io/github/dfa1/vortex/core/fbs/FbsFooter.java b/core/src/main/java/io/github/dfa1/vortex/core/fbs/FbsFooter.java index f0d6fc371..a0329e14f 100644 --- a/core/src/main/java/io/github/dfa1/vortex/core/fbs/FbsFooter.java +++ b/core/src/main/java/io/github/dfa1/vortex/core/fbs/FbsFooter.java @@ -9,11 +9,25 @@ @Generated("io.github.dfa1.vortex.fbsgen.CodeGen") public final class FbsFooter extends FbsTable { - /// Positions a reader at the root table of a finished buffer. + /// Verifies a finished buffer and positions a reader at its root table. /// @param seg the buffer /// @return a reader positioned at the root + /// @throws io.github.dfa1.vortex.core.error.VortexException if the buffer is malformed public static FbsFooter getRootAsFbsFooter(MemorySegment seg) { - return new FbsFooter().assign(seg, FbsTable.rootPosition(seg, 0)); + return new FbsFooter().assign(seg, FbsVerifier.verifyRoot(seg, FbsFooter::verify)); + } + + /// Checks this table and everything reachable from it. + /// @param v the verifier walking the buffer + /// @param pos the table's absolute position + static void verify(FbsVerifier v, long pos) { + v.startTable(pos); + v.tableVector(pos, 4, "array_specs", false, FbsArraySpec::verify); + v.tableVector(pos, 6, "layout_specs", false, FbsLayoutSpec::verify); + v.vector(pos, 8, 16, "segment_specs", false); + v.tableVector(pos, 10, "compression_specs", false, FbsCompressionSpec::verify); + v.tableVector(pos, 12, "encryption_specs", false, FbsEncryptionSpec::verify); + v.endTable(); } /// Positions this reader at a table. @@ -31,6 +45,12 @@ public int arraySpecsLength() { return o != 0 ? vectorLength(o) : 0; } + /// Tells an absent `arraySpecs` vector from an empty one, which [#arraySpecsLength()] cannot. + /// @return whether the `arraySpecs` vector is present + public boolean hasArraySpecs() { + return fieldOffset(4) != 0; + } + /// @param j element index /// @return the j-th `arraySpecs` element public FbsArraySpec arraySpecs(int j) { @@ -44,6 +64,12 @@ public int layoutSpecsLength() { return o != 0 ? vectorLength(o) : 0; } + /// Tells an absent `layoutSpecs` vector from an empty one, which [#layoutSpecsLength()] cannot. + /// @return whether the `layoutSpecs` vector is present + public boolean hasLayoutSpecs() { + return fieldOffset(6) != 0; + } + /// @param j element index /// @return the j-th `layoutSpecs` element public FbsLayoutSpec layoutSpecs(int j) { @@ -57,6 +83,12 @@ public int segmentSpecsLength() { return o != 0 ? vectorLength(o) : 0; } + /// Tells an absent `segmentSpecs` vector from an empty one, which [#segmentSpecsLength()] cannot. + /// @return whether the `segmentSpecs` vector is present + public boolean hasSegmentSpecs() { + return fieldOffset(8) != 0; + } + /// @param j element index /// @return the j-th `segmentSpecs` element public FbsSegmentSpec segmentSpecs(int j) { @@ -70,6 +102,12 @@ public int compressionSpecsLength() { return o != 0 ? vectorLength(o) : 0; } + /// Tells an absent `compressionSpecs` vector from an empty one, which [#compressionSpecsLength()] cannot. + /// @return whether the `compressionSpecs` vector is present + public boolean hasCompressionSpecs() { + return fieldOffset(10) != 0; + } + /// @param j element index /// @return the j-th `compressionSpecs` element public FbsCompressionSpec compressionSpecs(int j) { @@ -83,6 +121,12 @@ public int encryptionSpecsLength() { return o != 0 ? vectorLength(o) : 0; } + /// Tells an absent `encryptionSpecs` vector from an empty one, which [#encryptionSpecsLength()] cannot. + /// @return whether the `encryptionSpecs` vector is present + public boolean hasEncryptionSpecs() { + return fieldOffset(12) != 0; + } + /// @param j element index /// @return the j-th `encryptionSpecs` element public FbsEncryptionSpec encryptionSpecs(int j) { diff --git a/core/src/main/java/io/github/dfa1/vortex/core/fbs/FbsLayout.java b/core/src/main/java/io/github/dfa1/vortex/core/fbs/FbsLayout.java index 94a7d86ef..772b57c9a 100644 --- a/core/src/main/java/io/github/dfa1/vortex/core/fbs/FbsLayout.java +++ b/core/src/main/java/io/github/dfa1/vortex/core/fbs/FbsLayout.java @@ -9,11 +9,25 @@ @Generated("io.github.dfa1.vortex.fbsgen.CodeGen") public final class FbsLayout extends FbsTable { - /// Positions a reader at the root table of a finished buffer. + /// Verifies a finished buffer and positions a reader at its root table. /// @param seg the buffer /// @return a reader positioned at the root + /// @throws io.github.dfa1.vortex.core.error.VortexException if the buffer is malformed public static FbsLayout getRootAsFbsLayout(MemorySegment seg) { - return new FbsLayout().assign(seg, FbsTable.rootPosition(seg, 0)); + return new FbsLayout().assign(seg, FbsVerifier.verifyRoot(seg, FbsLayout::verify)); + } + + /// Checks this table and everything reachable from it. + /// @param v the verifier walking the buffer + /// @param pos the table's absolute position + static void verify(FbsVerifier v, long pos) { + v.startTable(pos); + v.inline(pos, 4, 2, "encoding", false); + v.inline(pos, 6, 8, "row_count", false); + v.vector(pos, 8, 1, "metadata", false); + v.tableVector(pos, 10, "children", false, FbsLayout::verify); + v.vector(pos, 12, 4, "segments", false); + v.endTable(); } /// Positions this reader at a table. @@ -43,6 +57,12 @@ public int metadataLength() { return o != 0 ? vectorLength(o) : 0; } + /// Tells an absent `metadata` vector from an empty one, which [#metadataLength()] cannot. + /// @return whether the `metadata` vector is present + public boolean hasMetadata() { + return fieldOffset(8) != 0; + } + /// @param j element index /// @return the j-th `metadata` element public int metadata(int j) { @@ -62,6 +82,12 @@ public int childrenLength() { return o != 0 ? vectorLength(o) : 0; } + /// Tells an absent `children` vector from an empty one, which [#childrenLength()] cannot. + /// @return whether the `children` vector is present + public boolean hasChildren() { + return fieldOffset(10) != 0; + } + /// @param j element index /// @return the j-th `children` element public FbsLayout children(int j) { @@ -75,6 +101,12 @@ public int segmentsLength() { return o != 0 ? vectorLength(o) : 0; } + /// Tells an absent `segments` vector from an empty one, which [#segmentsLength()] cannot. + /// @return whether the `segments` vector is present + public boolean hasSegments() { + return fieldOffset(12) != 0; + } + /// @param j element index /// @return the j-th `segments` element public long segments(int j) { diff --git a/core/src/main/java/io/github/dfa1/vortex/core/fbs/FbsLayoutSpec.java b/core/src/main/java/io/github/dfa1/vortex/core/fbs/FbsLayoutSpec.java index 696d83c23..9a7d7cd09 100644 --- a/core/src/main/java/io/github/dfa1/vortex/core/fbs/FbsLayoutSpec.java +++ b/core/src/main/java/io/github/dfa1/vortex/core/fbs/FbsLayoutSpec.java @@ -9,11 +9,21 @@ @Generated("io.github.dfa1.vortex.fbsgen.CodeGen") public final class FbsLayoutSpec extends FbsTable { - /// Positions a reader at the root table of a finished buffer. + /// Verifies a finished buffer and positions a reader at its root table. /// @param seg the buffer /// @return a reader positioned at the root + /// @throws io.github.dfa1.vortex.core.error.VortexException if the buffer is malformed public static FbsLayoutSpec getRootAsFbsLayoutSpec(MemorySegment seg) { - return new FbsLayoutSpec().assign(seg, FbsTable.rootPosition(seg, 0)); + return new FbsLayoutSpec().assign(seg, FbsVerifier.verifyRoot(seg, FbsLayoutSpec::verify)); + } + + /// Checks this table and everything reachable from it. + /// @param v the verifier walking the buffer + /// @param pos the table's absolute position + static void verify(FbsVerifier v, long pos) { + v.startTable(pos); + v.string(pos, 4, "id", true); + v.endTable(); } /// Positions this reader at a table. diff --git a/core/src/main/java/io/github/dfa1/vortex/core/fbs/FbsList.java b/core/src/main/java/io/github/dfa1/vortex/core/fbs/FbsList.java index 1824bdf86..5eabecfc9 100644 --- a/core/src/main/java/io/github/dfa1/vortex/core/fbs/FbsList.java +++ b/core/src/main/java/io/github/dfa1/vortex/core/fbs/FbsList.java @@ -9,11 +9,22 @@ @Generated("io.github.dfa1.vortex.fbsgen.CodeGen") public final class FbsList extends FbsTable { - /// Positions a reader at the root table of a finished buffer. + /// Verifies a finished buffer and positions a reader at its root table. /// @param seg the buffer /// @return a reader positioned at the root + /// @throws io.github.dfa1.vortex.core.error.VortexException if the buffer is malformed public static FbsList getRootAsFbsList(MemorySegment seg) { - return new FbsList().assign(seg, FbsTable.rootPosition(seg, 0)); + return new FbsList().assign(seg, FbsVerifier.verifyRoot(seg, FbsList::verify)); + } + + /// Checks this table and everything reachable from it. + /// @param v the verifier walking the buffer + /// @param pos the table's absolute position + static void verify(FbsVerifier v, long pos) { + v.startTable(pos); + v.table(pos, 4, "element_type", false, FbsDType::verify); + v.inline(pos, 6, 1, "nullable", false); + v.endTable(); } /// Positions this reader at a table. diff --git a/core/src/main/java/io/github/dfa1/vortex/core/fbs/FbsMap.java b/core/src/main/java/io/github/dfa1/vortex/core/fbs/FbsMap.java index 32d9e6836..81e1d88fc 100644 --- a/core/src/main/java/io/github/dfa1/vortex/core/fbs/FbsMap.java +++ b/core/src/main/java/io/github/dfa1/vortex/core/fbs/FbsMap.java @@ -9,11 +9,24 @@ @Generated("io.github.dfa1.vortex.fbsgen.CodeGen") public final class FbsMap extends FbsTable { - /// Positions a reader at the root table of a finished buffer. + /// Verifies a finished buffer and positions a reader at its root table. /// @param seg the buffer /// @return a reader positioned at the root + /// @throws io.github.dfa1.vortex.core.error.VortexException if the buffer is malformed public static FbsMap getRootAsFbsMap(MemorySegment seg) { - return new FbsMap().assign(seg, FbsTable.rootPosition(seg, 0)); + return new FbsMap().assign(seg, FbsVerifier.verifyRoot(seg, FbsMap::verify)); + } + + /// Checks this table and everything reachable from it. + /// @param v the verifier walking the buffer + /// @param pos the table's absolute position + static void verify(FbsVerifier v, long pos) { + v.startTable(pos); + v.table(pos, 4, "key_type", false, FbsDType::verify); + v.table(pos, 6, "value_type", false, FbsDType::verify); + v.inline(pos, 8, 1, "keys_sorted", false); + v.inline(pos, 10, 1, "nullable", false); + v.endTable(); } /// Positions this reader at a table. diff --git a/core/src/main/java/io/github/dfa1/vortex/core/fbs/FbsNull.java b/core/src/main/java/io/github/dfa1/vortex/core/fbs/FbsNull.java index 52c23efe7..fbfed5787 100644 --- a/core/src/main/java/io/github/dfa1/vortex/core/fbs/FbsNull.java +++ b/core/src/main/java/io/github/dfa1/vortex/core/fbs/FbsNull.java @@ -9,11 +9,20 @@ @Generated("io.github.dfa1.vortex.fbsgen.CodeGen") public final class FbsNull extends FbsTable { - /// Positions a reader at the root table of a finished buffer. + /// Verifies a finished buffer and positions a reader at its root table. /// @param seg the buffer /// @return a reader positioned at the root + /// @throws io.github.dfa1.vortex.core.error.VortexException if the buffer is malformed public static FbsNull getRootAsFbsNull(MemorySegment seg) { - return new FbsNull().assign(seg, FbsTable.rootPosition(seg, 0)); + return new FbsNull().assign(seg, FbsVerifier.verifyRoot(seg, FbsNull::verify)); + } + + /// Checks this table and everything reachable from it. + /// @param v the verifier walking the buffer + /// @param pos the table's absolute position + static void verify(FbsVerifier v, long pos) { + v.startTable(pos); + v.endTable(); } /// Positions this reader at a table. diff --git a/core/src/main/java/io/github/dfa1/vortex/core/fbs/FbsPostscript.java b/core/src/main/java/io/github/dfa1/vortex/core/fbs/FbsPostscript.java index 81bff4404..7ecea622b 100644 --- a/core/src/main/java/io/github/dfa1/vortex/core/fbs/FbsPostscript.java +++ b/core/src/main/java/io/github/dfa1/vortex/core/fbs/FbsPostscript.java @@ -9,11 +9,24 @@ @Generated("io.github.dfa1.vortex.fbsgen.CodeGen") public final class FbsPostscript extends FbsTable { - /// Positions a reader at the root table of a finished buffer. + /// Verifies a finished buffer and positions a reader at its root table. /// @param seg the buffer /// @return a reader positioned at the root + /// @throws io.github.dfa1.vortex.core.error.VortexException if the buffer is malformed public static FbsPostscript getRootAsFbsPostscript(MemorySegment seg) { - return new FbsPostscript().assign(seg, FbsTable.rootPosition(seg, 0)); + return new FbsPostscript().assign(seg, FbsVerifier.verifyRoot(seg, FbsPostscript::verify)); + } + + /// Checks this table and everything reachable from it. + /// @param v the verifier walking the buffer + /// @param pos the table's absolute position + static void verify(FbsVerifier v, long pos) { + v.startTable(pos); + v.table(pos, 4, "dtype", false, FbsPostscriptSegment::verify); + v.table(pos, 6, "layout", false, FbsPostscriptSegment::verify); + v.table(pos, 8, "statistics", false, FbsPostscriptSegment::verify); + v.table(pos, 10, "footer", false, FbsPostscriptSegment::verify); + v.endTable(); } /// Positions this reader at a table. diff --git a/core/src/main/java/io/github/dfa1/vortex/core/fbs/FbsPostscriptSegment.java b/core/src/main/java/io/github/dfa1/vortex/core/fbs/FbsPostscriptSegment.java index 28679bb02..1ea600c92 100644 --- a/core/src/main/java/io/github/dfa1/vortex/core/fbs/FbsPostscriptSegment.java +++ b/core/src/main/java/io/github/dfa1/vortex/core/fbs/FbsPostscriptSegment.java @@ -9,11 +9,25 @@ @Generated("io.github.dfa1.vortex.fbsgen.CodeGen") public final class FbsPostscriptSegment extends FbsTable { - /// Positions a reader at the root table of a finished buffer. + /// Verifies a finished buffer and positions a reader at its root table. /// @param seg the buffer /// @return a reader positioned at the root + /// @throws io.github.dfa1.vortex.core.error.VortexException if the buffer is malformed public static FbsPostscriptSegment getRootAsFbsPostscriptSegment(MemorySegment seg) { - return new FbsPostscriptSegment().assign(seg, FbsTable.rootPosition(seg, 0)); + return new FbsPostscriptSegment().assign(seg, FbsVerifier.verifyRoot(seg, FbsPostscriptSegment::verify)); + } + + /// Checks this table and everything reachable from it. + /// @param v the verifier walking the buffer + /// @param pos the table's absolute position + static void verify(FbsVerifier v, long pos) { + v.startTable(pos); + v.inline(pos, 4, 8, "offset", false); + v.inline(pos, 6, 4, "length", false); + v.inline(pos, 8, 1, "alignment_exponent", false); + v.table(pos, 10, "_compression", false, FbsCompressionSpec::verify); + v.table(pos, 12, "_encryption", false, FbsEncryptionSpec::verify); + v.endTable(); } /// Positions this reader at a table. diff --git a/core/src/main/java/io/github/dfa1/vortex/core/fbs/FbsPrimitive.java b/core/src/main/java/io/github/dfa1/vortex/core/fbs/FbsPrimitive.java index 2ff86a1c6..460b4b490 100644 --- a/core/src/main/java/io/github/dfa1/vortex/core/fbs/FbsPrimitive.java +++ b/core/src/main/java/io/github/dfa1/vortex/core/fbs/FbsPrimitive.java @@ -9,11 +9,22 @@ @Generated("io.github.dfa1.vortex.fbsgen.CodeGen") public final class FbsPrimitive extends FbsTable { - /// Positions a reader at the root table of a finished buffer. + /// Verifies a finished buffer and positions a reader at its root table. /// @param seg the buffer /// @return a reader positioned at the root + /// @throws io.github.dfa1.vortex.core.error.VortexException if the buffer is malformed public static FbsPrimitive getRootAsFbsPrimitive(MemorySegment seg) { - return new FbsPrimitive().assign(seg, FbsTable.rootPosition(seg, 0)); + return new FbsPrimitive().assign(seg, FbsVerifier.verifyRoot(seg, FbsPrimitive::verify)); + } + + /// Checks this table and everything reachable from it. + /// @param v the verifier walking the buffer + /// @param pos the table's absolute position + static void verify(FbsVerifier v, long pos) { + v.startTable(pos); + v.inline(pos, 4, 1, "ptype", false); + v.inline(pos, 6, 1, "nullable", false); + v.endTable(); } /// Positions this reader at a table. diff --git a/core/src/main/java/io/github/dfa1/vortex/core/fbs/FbsStruct.java b/core/src/main/java/io/github/dfa1/vortex/core/fbs/FbsStruct.java index 3bdb5c4d3..cdaa470b8 100644 --- a/core/src/main/java/io/github/dfa1/vortex/core/fbs/FbsStruct.java +++ b/core/src/main/java/io/github/dfa1/vortex/core/fbs/FbsStruct.java @@ -9,11 +9,23 @@ @Generated("io.github.dfa1.vortex.fbsgen.CodeGen") public final class FbsStruct extends FbsTable { - /// Positions a reader at the root table of a finished buffer. + /// Verifies a finished buffer and positions a reader at its root table. /// @param seg the buffer /// @return a reader positioned at the root + /// @throws io.github.dfa1.vortex.core.error.VortexException if the buffer is malformed public static FbsStruct getRootAsFbsStruct(MemorySegment seg) { - return new FbsStruct().assign(seg, FbsTable.rootPosition(seg, 0)); + return new FbsStruct().assign(seg, FbsVerifier.verifyRoot(seg, FbsStruct::verify)); + } + + /// Checks this table and everything reachable from it. + /// @param v the verifier walking the buffer + /// @param pos the table's absolute position + static void verify(FbsVerifier v, long pos) { + v.startTable(pos); + v.stringVector(pos, 4, "names", false); + v.tableVector(pos, 6, "dtypes", false, FbsDType::verify); + v.inline(pos, 8, 1, "nullable", false); + v.endTable(); } /// Positions this reader at a table. @@ -31,6 +43,12 @@ public int namesLength() { return o != 0 ? vectorLength(o) : 0; } + /// Tells an absent `names` vector from an empty one, which [#namesLength()] cannot. + /// @return whether the `names` vector is present + public boolean hasNames() { + return fieldOffset(4) != 0; + } + /// @param j element index /// @return the j-th `names` string public String names(int j) { @@ -44,6 +62,12 @@ public int dtypesLength() { return o != 0 ? vectorLength(o) : 0; } + /// Tells an absent `dtypes` vector from an empty one, which [#dtypesLength()] cannot. + /// @return whether the `dtypes` vector is present + public boolean hasDtypes() { + return fieldOffset(6) != 0; + } + /// @param j element index /// @return the j-th `dtypes` element public FbsDType dtypes(int j) { diff --git a/core/src/main/java/io/github/dfa1/vortex/core/fbs/FbsTable.java b/core/src/main/java/io/github/dfa1/vortex/core/fbs/FbsTable.java index 544e5ce0d..efb02707d 100644 --- a/core/src/main/java/io/github/dfa1/vortex/core/fbs/FbsTable.java +++ b/core/src/main/java/io/github/dfa1/vortex/core/fbs/FbsTable.java @@ -24,10 +24,8 @@ /// /// Bounds: these accessors do not validate offsets or lengths read from the buffer /// (e.g. [#readStringAt(long)] allocates `new byte[len]` with `len` taken straight from -/// the blob). This matches the previous `com.google.flatbuffers` runtime — the caller is -/// expected to have framed the FlatBuffer from a trusted region. File-level framing -/// (segment offsets/lengths, trailer, postscript) is range-checked upstream by the -/// reader's `IoBounds`; intra-FlatBuffer fields are not. +/// the blob). They need not: a reader is only ever positioned by a generated `getRootAsX`, +/// which runs [FbsVerifier] over the whole buffer first, as Rust's `flatbuffers::root` does. /// /// Package-private: only the generated table accessors in this package extend it. class FbsTable { @@ -51,14 +49,6 @@ protected final void init(MemorySegment segment, long position) { this.pos = position; } - /// Resolves the root table position of a finished FlatBuffer. - /// - /// @param segment buffer whose first 4 bytes hold the uoffset to the root table - /// @param start byte position of the buffer start within `segment` - /// @return the absolute position of the root table - static long rootPosition(MemorySegment segment, long start) { - return start + segment.get(LE_INT, start); - } /// Looks up a field's offset relative to this table. /// diff --git a/core/src/main/java/io/github/dfa1/vortex/core/fbs/FbsUnion.java b/core/src/main/java/io/github/dfa1/vortex/core/fbs/FbsUnion.java index 03222243e..9f8c0cd74 100644 --- a/core/src/main/java/io/github/dfa1/vortex/core/fbs/FbsUnion.java +++ b/core/src/main/java/io/github/dfa1/vortex/core/fbs/FbsUnion.java @@ -9,11 +9,24 @@ @Generated("io.github.dfa1.vortex.fbsgen.CodeGen") public final class FbsUnion extends FbsTable { - /// Positions a reader at the root table of a finished buffer. + /// Verifies a finished buffer and positions a reader at its root table. /// @param seg the buffer /// @return a reader positioned at the root + /// @throws io.github.dfa1.vortex.core.error.VortexException if the buffer is malformed public static FbsUnion getRootAsFbsUnion(MemorySegment seg) { - return new FbsUnion().assign(seg, FbsTable.rootPosition(seg, 0)); + return new FbsUnion().assign(seg, FbsVerifier.verifyRoot(seg, FbsUnion::verify)); + } + + /// Checks this table and everything reachable from it. + /// @param v the verifier walking the buffer + /// @param pos the table's absolute position + static void verify(FbsVerifier v, long pos) { + v.startTable(pos); + v.stringVector(pos, 4, "names", false); + v.tableVector(pos, 6, "dtypes", false, FbsDType::verify); + v.vector(pos, 8, 1, "type_ids", false); + v.inline(pos, 10, 1, "nullable", false); + v.endTable(); } /// Positions this reader at a table. @@ -31,6 +44,12 @@ public int namesLength() { return o != 0 ? vectorLength(o) : 0; } + /// Tells an absent `names` vector from an empty one, which [#namesLength()] cannot. + /// @return whether the `names` vector is present + public boolean hasNames() { + return fieldOffset(4) != 0; + } + /// @param j element index /// @return the j-th `names` string public String names(int j) { @@ -44,6 +63,12 @@ public int dtypesLength() { return o != 0 ? vectorLength(o) : 0; } + /// Tells an absent `dtypes` vector from an empty one, which [#dtypesLength()] cannot. + /// @return whether the `dtypes` vector is present + public boolean hasDtypes() { + return fieldOffset(6) != 0; + } + /// @param j element index /// @return the j-th `dtypes` element public FbsDType dtypes(int j) { @@ -57,6 +82,12 @@ public int typeIdsLength() { return o != 0 ? vectorLength(o) : 0; } + /// Tells an absent `typeIds` vector from an empty one, which [#typeIdsLength()] cannot. + /// @return whether the `typeIds` vector is present + public boolean hasTypeIds() { + return fieldOffset(8) != 0; + } + /// @param j element index /// @return the j-th `typeIds` element public int typeIds(int j) { diff --git a/core/src/main/java/io/github/dfa1/vortex/core/fbs/FbsUtf8.java b/core/src/main/java/io/github/dfa1/vortex/core/fbs/FbsUtf8.java index 2515335b8..ec4de9712 100644 --- a/core/src/main/java/io/github/dfa1/vortex/core/fbs/FbsUtf8.java +++ b/core/src/main/java/io/github/dfa1/vortex/core/fbs/FbsUtf8.java @@ -9,11 +9,21 @@ @Generated("io.github.dfa1.vortex.fbsgen.CodeGen") public final class FbsUtf8 extends FbsTable { - /// Positions a reader at the root table of a finished buffer. + /// Verifies a finished buffer and positions a reader at its root table. /// @param seg the buffer /// @return a reader positioned at the root + /// @throws io.github.dfa1.vortex.core.error.VortexException if the buffer is malformed public static FbsUtf8 getRootAsFbsUtf8(MemorySegment seg) { - return new FbsUtf8().assign(seg, FbsTable.rootPosition(seg, 0)); + return new FbsUtf8().assign(seg, FbsVerifier.verifyRoot(seg, FbsUtf8::verify)); + } + + /// Checks this table and everything reachable from it. + /// @param v the verifier walking the buffer + /// @param pos the table's absolute position + static void verify(FbsVerifier v, long pos) { + v.startTable(pos); + v.inline(pos, 4, 1, "nullable", false); + v.endTable(); } /// Positions this reader at a table. diff --git a/core/src/main/java/io/github/dfa1/vortex/core/fbs/FbsVariant.java b/core/src/main/java/io/github/dfa1/vortex/core/fbs/FbsVariant.java index df17fe087..2ffbdb033 100644 --- a/core/src/main/java/io/github/dfa1/vortex/core/fbs/FbsVariant.java +++ b/core/src/main/java/io/github/dfa1/vortex/core/fbs/FbsVariant.java @@ -9,11 +9,21 @@ @Generated("io.github.dfa1.vortex.fbsgen.CodeGen") public final class FbsVariant extends FbsTable { - /// Positions a reader at the root table of a finished buffer. + /// Verifies a finished buffer and positions a reader at its root table. /// @param seg the buffer /// @return a reader positioned at the root + /// @throws io.github.dfa1.vortex.core.error.VortexException if the buffer is malformed public static FbsVariant getRootAsFbsVariant(MemorySegment seg) { - return new FbsVariant().assign(seg, FbsTable.rootPosition(seg, 0)); + return new FbsVariant().assign(seg, FbsVerifier.verifyRoot(seg, FbsVariant::verify)); + } + + /// Checks this table and everything reachable from it. + /// @param v the verifier walking the buffer + /// @param pos the table's absolute position + static void verify(FbsVerifier v, long pos) { + v.startTable(pos); + v.inline(pos, 4, 1, "nullable", false); + v.endTable(); } /// Positions this reader at a table. diff --git a/core/src/main/java/io/github/dfa1/vortex/core/fbs/FbsVerifier.java b/core/src/main/java/io/github/dfa1/vortex/core/fbs/FbsVerifier.java new file mode 100644 index 000000000..b6608a85f --- /dev/null +++ b/core/src/main/java/io/github/dfa1/vortex/core/fbs/FbsVerifier.java @@ -0,0 +1,281 @@ +package io.github.dfa1.vortex.core.fbs; + +import static io.github.dfa1.vortex.core.io.VortexFormat.LE_INT; +import static io.github.dfa1.vortex.core.io.VortexFormat.LE_SHORT; + +import java.lang.foreign.MemorySegment; +import java.lang.foreign.ValueLayout; + +import io.github.dfa1.vortex.core.error.VortexException; + +/// Checks a whole FlatBuffer before any accessor reads it, as the Rust reference does with +/// `flatbuffers::root::()` on every buffer it parses. +/// +/// The [FbsTable] accessors follow offsets without bounds checks, so an unverified crafted buffer +/// fails deep inside the reader as a raw JDK exception, at whichever site first touches the bad +/// offset. Every generated `getRootAsX` runs this first instead: it walks every table reachable from +/// the root through its generated `verify(FbsVerifier, long)` and rejects the buffer with a +/// [VortexException] if any offset, vtable, vector or string falls outside it. Once a buffer passes, +/// the accessors cannot read out of bounds. +/// +/// Rules (the flatbuffers verifier's, with its default limits, which Vortex keeps): +/// - every table's soffset, vtable and object lie in the buffer, and the vtable is at least 4 bytes; +/// - every present field lies in the buffer, and every uoffset points into it; +/// - every vector's elements and every string's bytes plus NUL terminator lie in it; +/// - a `(required)` field is present; +/// - tables nest at most [#MAX_DEPTH] deep, and a buffer holds at most [#MAX_TABLES] tables. +/// +/// Not yet checked, though Rust does: alignment, UTF-8 validity of strings, and the +/// total-apparent-size cap. +final class FbsVerifier { + + /// flatbuffers' default `max_depth`, which Vortex keeps (`VORTEX_MAX_LAYOUT_DEPTH`). + static final int MAX_DEPTH = 64; + + /// flatbuffers' default `max_tables`, which Vortex keeps (`VORTEX_MAX_LAYOUT_TABLES`). + static final int MAX_TABLES = 1_000_000; + + /// Verifies one table: the generated `verify` of the table's class. + @FunctionalInterface + interface TableVerifier { + + /// @param verifier the verifier walking the buffer + /// @param position the table's absolute position + void verify(FbsVerifier verifier, long position); + } + + private final MemorySegment seg; + private final long size; + private int depth; + private int tables; + + private FbsVerifier(MemorySegment seg) { + this.seg = seg; + this.size = seg.byteSize(); + } + + /// Verifies a finished buffer whose root table `root` describes. + /// + /// @param seg the buffer + /// @param root the root table's verifier + /// @return the absolute position of the root table + /// @throws VortexException if the buffer is malformed + static long verifyRoot(MemorySegment seg, TableVerifier root) { + var verifier = new FbsVerifier(seg); + long position = verifier.uoffset(0); + root.verify(verifier, position); + return position; + } + + /// Opens a table: checks its soffset, vtable and object, and the depth and table limits. + /// Every `startTable` is paired with an [#endTable()]. + /// + /// @param position the table's absolute position + void startTable(long position) { + if (++depth > MAX_DEPTH) { + throw malformed("tables nest deeper than " + MAX_DEPTH); + } + if (++tables > MAX_TABLES) { + throw malformed("more than " + MAX_TABLES + " tables"); + } + check(position, Integer.BYTES, "table"); + long vtable = position - seg.get(LE_INT, position); + check(vtable, 2 * Short.BYTES, "vtable"); + int vtableSize = Short.toUnsignedInt(seg.get(LE_SHORT, vtable)); + int objectSize = Short.toUnsignedInt(seg.get(LE_SHORT, vtable + Short.BYTES)); + if (vtableSize < 2 * Short.BYTES || vtableSize % Short.BYTES != 0) { + throw malformed("vtable size " + vtableSize + " at " + vtable); + } + check(vtable, vtableSize, "vtable"); + check(position, objectSize, "table object"); + } + + /// Closes the table the last [#startTable(long)] opened. + void endTable() { + depth--; + } + + /// Checks an inline field (a scalar, an enum, or a struct) of `size` bytes. + /// + /// @param table the table's absolute position + /// @param slot the field's vtable slot offset (`4 + fieldIndex * 2`) + /// @param size the field's inline size in bytes + /// @param name the field name, for the error message + /// @param required whether the schema marks the field `(required)` + void inline(long table, int slot, int size, String name, boolean required) { + field(table, slot, size, name, required); + } + + /// Checks a string field. + /// + /// @param table the table's absolute position + /// @param slot the field's vtable slot offset + /// @param name the field name, for the error message + /// @param required whether the schema marks the field `(required)` + void string(long table, int slot, String name, boolean required) { + long at = field(table, slot, Integer.BYTES, name, required); + if (at >= 0) { + stringAt(uoffset(at)); + } + } + + /// Checks a vector field of inline elements (scalars, enums or structs). + /// + /// @param table the table's absolute position + /// @param slot the field's vtable slot offset + /// @param elementSize the inline size of one element in bytes + /// @param name the field name, for the error message + /// @param required whether the schema marks the field `(required)` + void vector(long table, int slot, int elementSize, String name, boolean required) { + long at = field(table, slot, Integer.BYTES, name, required); + if (at >= 0) { + vectorAt(uoffset(at), elementSize); + } + } + + /// Checks a vector-of-strings field and every string in it. + /// + /// @param table the table's absolute position + /// @param slot the field's vtable slot offset + /// @param name the field name, for the error message + /// @param required whether the schema marks the field `(required)` + void stringVector(long table, int slot, String name, boolean required) { + long at = field(table, slot, Integer.BYTES, name, required); + if (at >= 0) { + long elements = uoffset(at); + long count = vectorAt(elements, Integer.BYTES); + for (long i = 0; i < count; i++) { + stringAt(uoffset(elements + Integer.BYTES + i * Integer.BYTES)); + } + } + } + + /// Checks a vector-of-tables field and every table in it. + /// + /// @param table the table's absolute position + /// @param slot the field's vtable slot offset + /// @param name the field name, for the error message + /// @param required whether the schema marks the field `(required)` + /// @param element the element table's verifier + void tableVector(long table, int slot, String name, boolean required, TableVerifier element) { + long at = field(table, slot, Integer.BYTES, name, required); + if (at >= 0) { + long elements = uoffset(at); + long count = vectorAt(elements, Integer.BYTES); + for (long i = 0; i < count; i++) { + element.verify(this, uoffset(elements + Integer.BYTES + i * Integer.BYTES)); + } + } + } + + /// Checks a table field and the table it points to. + /// + /// @param table the table's absolute position + /// @param slot the field's vtable slot offset + /// @param name the field name, for the error message + /// @param required whether the schema marks the field `(required)` + /// @param child the child table's verifier + void table(long table, int slot, String name, boolean required, TableVerifier child) { + long at = field(table, slot, Integer.BYTES, name, required); + if (at >= 0) { + child.verify(this, uoffset(at)); + } + } + + /// Checks a union's discriminator field and returns it, so the generated code can verify the + /// member table its value field points to with [#unionValue(long, int, String)]. A member type + /// without a value, or a value without a member type, is malformed, as in Rust's `visit_union` + /// (`InconsistentUnion`): the accessors would return null for the member the type names. + /// + /// @param table the table's absolute position + /// @param slot the discriminator's vtable slot offset + /// @param valueSlot the value's vtable slot offset + /// @param name the union field name, for the error message + /// @return the discriminator (0 = NONE) + int unionType(long table, int slot, int valueSlot, String name) { + long at = field(table, slot, Byte.BYTES, name + "_type", false); + int type = at >= 0 ? Byte.toUnsignedInt(seg.get(ValueLayout.JAVA_BYTE, at)) : 0; + if ((type != 0) != (field(table, valueSlot, Integer.BYTES, name, false) >= 0)) { + throw malformed("union '" + name + "' type " + type + " inconsistent with its value in table at " + table); + } + return type; + } + + /// Resolves a union's value field to the member table's position. + /// + /// @param table the table's absolute position + /// @param slot the value's vtable slot offset + /// @param name the union field name, for the error message + /// @return the member table's absolute position, or -1 when the value is absent + long unionValue(long table, int slot, String name) { + long at = field(table, slot, Integer.BYTES, name, false); + return at >= 0 ? uoffset(at) : -1; + } + + /// Returns a field's absolute position after checking it lies in the buffer, or -1 if absent. + private long field(long table, int slot, int size, String name, boolean required) { + long vtable = table - seg.get(LE_INT, table); + int vtableSize = Short.toUnsignedInt(seg.get(LE_SHORT, vtable)); + int offset = slot < vtableSize ? Short.toUnsignedInt(seg.get(LE_SHORT, vtable + slot)) : 0; + if (offset == 0) { + if (required) { + throw malformed("required field '" + name + "' missing in table at " + table); + } + return -1; + } + // The message is built only on failure: this runs for every field of every table verified. + if (!inBuffer(table + offset, size)) { + throw outside("field '" + name + "'", table + offset, size); + } + return table + offset; + } + + /// Follows the uoffset at `at` and returns the position it points to, inside the buffer. + private long uoffset(long at) { + check(at, Integer.BYTES, "offset"); + long target = at + Integer.toUnsignedLong(seg.get(LE_INT, at)); + check(target, 1, "offset target"); + return target; + } + + /// Checks a vector's length prefix and elements; returns its element count. + private long vectorAt(long at, int elementSize) { + check(at, Integer.BYTES, "vector length"); + long count = Integer.toUnsignedLong(seg.get(LE_INT, at)); + if (!inBuffer(at + Integer.BYTES, count * elementSize)) { + throw outside("vector of " + count + " elements", at + Integer.BYTES, count * elementSize); + } + return count; + } + + /// Checks a string's length prefix, bytes and NUL terminator. + private void stringAt(long at) { + long length = vectorAt(at, Byte.BYTES); + long terminator = at + Integer.BYTES + length; + check(terminator, 1, "string terminator"); + if (seg.get(ValueLayout.JAVA_BYTE, terminator) != 0) { + throw malformed("string at " + at + " is not NUL-terminated"); + } + } + + /// Checks that `[at, at + length)` lies in the buffer. + private void check(long at, long length, String what) { + if (!inBuffer(at, length)) { + throw outside(what, at, length); + } + } + + /// Whether `[at, at + length)` lies in the buffer (the arithmetic cannot overflow: both are below + /// 2^35 here, from u32 counts times element sizes of at most 8). + private boolean inBuffer(long at, long length) { + return at >= 0 && length >= 0 && at + length <= size; + } + + private VortexException outside(String what, long at, long length) { + return malformed(what + " [" + at + ", " + (at + length) + ") outside the " + size + "-byte buffer"); + } + + private static VortexException malformed(String reason) { + return new VortexException("malformed flatbuffer: " + reason); + } +} diff --git a/core/src/test/java/io/github/dfa1/vortex/core/fbs/FbsVerifierTest.java b/core/src/test/java/io/github/dfa1/vortex/core/fbs/FbsVerifierTest.java new file mode 100644 index 000000000..24724a7f4 --- /dev/null +++ b/core/src/test/java/io/github/dfa1/vortex/core/fbs/FbsVerifierTest.java @@ -0,0 +1,279 @@ +package io.github.dfa1.vortex.core.fbs; + +import static io.github.dfa1.vortex.core.io.VortexFormat.LE_INT; +import static io.github.dfa1.vortex.core.io.VortexFormat.LE_SHORT; +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatCode; +import static org.assertj.core.api.Assertions.assertThatThrownBy; + +import java.lang.foreign.MemorySegment; +import java.lang.foreign.ValueLayout; +import java.util.Arrays; + +import io.github.dfa1.vortex.core.error.VortexException; +import org.junit.jupiter.api.Nested; +import org.junit.jupiter.api.Test; + +/// Each rule of [FbsVerifier], driven through the generated `getRootAsX` entry points: a valid buffer +/// from [FbsBuilder] with one field corrupted in place, so every case fails on exactly one rule. +class FbsVerifierTest { + + @Test + void builderOutput_passes() { + // Given — a layout using every field kind it has: scalars, a byte vector, a u32 vector and a + // vector of child tables + MemorySegment seg = layout(new long[]{1, 2, 3}, 2); + + // When + FbsLayout result = FbsLayout.getRootAsFbsLayout(seg); + + // Then + assertThat(result.childrenLength()).isEqualTo(2); + assertThat(result.segments(2)).isEqualTo(3); + } + + @Nested + class Tables { + + @Test + void bufferTooShortForRootOffset_rejected() { + // Given + MemorySegment seg = MemorySegment.ofArray(new byte[3]); + + // When / Then + assertMalformed(seg, "offset"); + } + + @Test + void rootOffsetPastBuffer_rejected() { + // Given + MemorySegment seg = layout(new long[]{1}, 0); + seg.set(LE_INT, 0, (int) seg.byteSize()); + + // When / Then + assertMalformed(seg, "offset target"); + } + + @Test + void vtableOutsideBuffer_rejected() { + // Given — the root's soffset points a vtable far before the buffer start + MemorySegment seg = layout(new long[]{1}, 0); + seg.set(LE_INT, rootPosition(seg), 1_000_000); + + // When / Then + assertMalformed(seg, "vtable"); + } + + @Test + void vtableSmallerThanItsHeader_rejected() { + // Given — a vtable claiming 2 bytes, too short for its own size + object-size header + MemorySegment seg = layout(new long[]{1}, 0); + seg.set(LE_SHORT, vtable(seg), (short) 2); + + // When / Then + assertMalformed(seg, "vtable size 2"); + } + + @Test + void objectPastBuffer_rejected() { + // Given — the vtable's object size runs the table past the buffer end + MemorySegment seg = layout(new long[]{1}, 0); + seg.set(LE_SHORT, vtable(seg) + 2, (short) 0xFFFF); + + // When / Then + assertMalformed(seg, "table object"); + } + + @Test + void fieldPastBuffer_rejected() { + // Given — the row_count slot (vtable offset 6) points 8 bytes from the buffer end + // minus 4, so the u64 field straddles it + MemorySegment seg = layout(new long[]{1}, 0); + long table = rootPosition(seg); + seg.set(LE_SHORT, vtable(seg) + 6, (short) (seg.byteSize() - 4 - table)); + seg.set(LE_SHORT, vtable(seg) + 2, (short) (seg.byteSize() - table)); + + // When / Then + assertMalformed(seg, "field 'row_count'"); + } + } + + @Nested + class VectorsAndStrings { + + @Test + void vectorLengthPastBuffer_rejected() { + // Given — the segments vector's length prefix claims 100 000 u32s + MemorySegment seg = layout(new long[]{1, 2}, 0); + seg.set(LE_INT, vectorStart(seg, FbsLayout.getRootAsFbsLayout(seg), 12), 100_000); + + // When / Then + assertMalformed(seg, "vector of 100000 elements"); + } + + @Test + void stringWithoutNulTerminator_rejected() { + // Given — an ArraySpec id whose terminating NUL is overwritten + FbsBuilder b = new FbsBuilder(64); + b.finish(FbsArraySpec.createFbsArraySpec(b, b.createString("vortex.primitive"))); + MemorySegment seg = MemorySegment.ofArray(b.sizedByteArray()); + long string = vectorStart(seg, FbsArraySpec.getRootAsFbsArraySpec(seg), 4); + seg.set(ValueLayout.JAVA_BYTE, string + 4 + "vortex.primitive".length(), (byte) 'x'); + + // When / Then + assertThatThrownBy(() -> FbsArraySpec.getRootAsFbsArraySpec(seg)) + .isInstanceOf(VortexException.class) + .hasMessageContaining("not NUL-terminated"); + } + + @Test + void requiredFieldMissing_rejected() { + // Given — an ArraySpec without its `id: string (required)` + FbsBuilder b = new FbsBuilder(64); + b.finish(FbsArraySpec.createFbsArraySpec(b, 0)); + MemorySegment seg = MemorySegment.ofArray(b.sizedByteArray()); + + // When / Then + assertThatThrownBy(() -> FbsArraySpec.getRootAsFbsArraySpec(seg)) + .isInstanceOf(VortexException.class) + .hasMessageContaining("required field 'id' missing"); + } + } + + @Nested + class Unions { + + @Test + void selectedMemberIsVerified() { + // Given — a Struct_ dtype whose names vector claims far more strings than it holds: only + // the union member's own verify can see it + FbsBuilder b = new FbsBuilder(128); + int names = FbsStruct.createNamesVector(b, new int[]{b.createString("a")}); + int dtypes = FbsStruct.createDtypesVector(b, new int[0]); + int struct = FbsStruct.createFbsStruct(b, names, dtypes, false); + b.finish(FbsDType.createFbsDType(b, FbsType.FbsStruct, struct)); + MemorySegment seg = MemorySegment.ofArray(b.sizedByteArray()); + FbsStruct member = FbsDType.getRootAsFbsDType(seg).type(new FbsStruct()); + seg.set(LE_INT, vectorStart(seg, member, 4), 100_000); + + // When / Then + assertThatThrownBy(() -> FbsDType.getRootAsFbsDType(seg)) + .isInstanceOf(VortexException.class) + .hasMessageContaining("vector of 100000 elements"); + } + + @org.junit.jupiter.params.ParameterizedTest(name = "type {0}, value present {1}") + @org.junit.jupiter.params.provider.CsvSource({"7, false", "0, true"}) + void typeAndValueDisagree_rejected(byte type, boolean withValue) { + // Given — a member type with no value, or a value with no member type: Rust's + // visit_union rejects both (InconsistentUnion); the accessors would return a null member + FbsBuilder b = new FbsBuilder(64); + int value = withValue ? FbsNull.createFbsNull(b) : 0; + b.finish(FbsDType.createFbsDType(b, type, value)); + MemorySegment seg = MemorySegment.ofArray(b.sizedByteArray()); + + // When / Then + assertThatThrownBy(() -> FbsDType.getRootAsFbsDType(seg)) + .isInstanceOf(VortexException.class) + .hasMessageContaining("union 'type' type " + type + " inconsistent with its value"); + } + + @Test + void unknownMemberIsAccepted() { + // Given — a discriminator this schema does not know: like flatc's verifiers, accepted, + // since no accessor reads its value (the reader rejects the dtype itself) + FbsBuilder b = new FbsBuilder(64); + int nul = FbsNull.createFbsNull(b); + b.finish(FbsDType.createFbsDType(b, (byte) 99, nul)); + MemorySegment seg = MemorySegment.ofArray(b.sizedByteArray()); + + // When / Then + assertThatCode(() -> FbsDType.getRootAsFbsDType(seg)).doesNotThrowAnyException(); + } + } + + @Nested + class Limits { + + @Test + void nestingAtDepthLimit_passes() { + // Given + MemorySegment seg = nestedLayouts(FbsVerifier.MAX_DEPTH); + + // When / Then + assertThatCode(() -> FbsLayout.getRootAsFbsLayout(seg)).doesNotThrowAnyException(); + } + + @Test + void nestingPastDepthLimit_rejected() { + // Given + MemorySegment seg = nestedLayouts(FbsVerifier.MAX_DEPTH + 1); + + // When / Then + assertMalformed(seg, "nest deeper than 64"); + } + + @Test + void tableCountPastLimit_rejected() { + // Given — one child table referenced MAX_TABLES times: a 4 MB buffer that a walk without + // the table cap would visit a million times over (each visit counts, as in Rust) + FbsBuilder b = new FbsBuilder(8 * 1024 * 1024); + int leaf = FbsLayout.createFbsLayout(b, 0, 1L, 0, 0, 0); + int[] children = new int[FbsVerifier.MAX_TABLES]; + Arrays.fill(children, leaf); + int childrenVector = FbsLayout.createChildrenVector(b, children); + b.finish(FbsLayout.createFbsLayout(b, 0, 1L, 0, childrenVector, 0)); + MemorySegment seg = b.dataSegment(); + + // When / Then + assertMalformed(seg, "more than 1000000 tables"); + } + } + + private static void assertMalformed(MemorySegment seg, String reason) { + assertThatThrownBy(() -> FbsLayout.getRootAsFbsLayout(seg)) + .isInstanceOf(VortexException.class) + .hasMessageContaining("malformed flatbuffer: ") + .hasMessageContaining(reason); + } + + /// A layout with the given segment ids, a 3-byte metadata vector and `children` leaf children. + private static MemorySegment layout(long[] segments, int children) { + FbsBuilder b = new FbsBuilder(256); + int[] leaves = new int[children]; + for (int i = 0; i < children; i++) { + leaves[i] = FbsLayout.createFbsLayout(b, 0, 1L, 0, 0, 0); + } + int childrenVector = FbsLayout.createChildrenVector(b, leaves); + int metadata = FbsLayout.createMetadataVector(b, new byte[]{1, 2, 3}); + int segmentsVector = FbsLayout.createSegmentsVector(b, segments); + b.finish(FbsLayout.createFbsLayout(b, 0, 7L, metadata, childrenVector, segmentsVector)); + return MemorySegment.ofArray(b.sizedByteArray()); + } + + /// A chain of `tables` layouts, each the single child of the next. + private static MemorySegment nestedLayouts(int tables) { + FbsBuilder b = new FbsBuilder(tables * 32 + 64); + int current = FbsLayout.createFbsLayout(b, 0, 1L, 0, 0, 0); + for (int i = 1; i < tables; i++) { + current = FbsLayout.createFbsLayout(b, 0, 1L, 0, FbsLayout.createChildrenVector(b, new int[]{current}), 0); + } + b.finish(current); + return MemorySegment.ofArray(b.sizedByteArray()); + } + + private static long rootPosition(MemorySegment seg) { + return seg.get(LE_INT, 0); + } + + private static long vtable(MemorySegment seg) { + long table = rootPosition(seg); + return table - seg.get(LE_INT, table); + } + + /// Position of the length prefix of the vector (or string) in `table`'s `slot`. + private static long vectorStart(MemorySegment seg, FbsTable table, int slot) { + long at = table.pos + table.fieldOffset(slot); + return at + seg.get(LE_INT, at); + } +} diff --git a/fbs-gen/src/main/java/io/github/dfa1/vortex/fbsgen/CodeGen.java b/fbs-gen/src/main/java/io/github/dfa1/vortex/fbsgen/CodeGen.java index bd3e37d30..2dddfee76 100644 --- a/fbs-gen/src/main/java/io/github/dfa1/vortex/fbsgen/CodeGen.java +++ b/fbs-gen/src/main/java/io/github/dfa1/vortex/fbsgen/CodeGen.java @@ -83,14 +83,16 @@ private String emitTable(Ast.TableDecl table) { sb.append("public final class ").append(name).append(" extends FbsTable {\n\n"); sb.append(""" - /// Positions a reader at the root table of a finished buffer. + /// Verifies a finished buffer and positions a reader at its root table. /// @param seg the buffer /// @return a reader positioned at the root + /// @throws io.github.dfa1.vortex.core.error.VortexException if the buffer is malformed public static %1$s getRootAs%1$s(MemorySegment seg) { - return new %1$s().assign(seg, FbsTable.rootPosition(seg, 0)); + return new %1$s().assign(seg, FbsVerifier.verifyRoot(seg, %1$s::verify)); } """.formatted(name)); + emitVerify(sb, table); sb.append(""" /// Positions this reader at a table. @@ -112,6 +114,96 @@ private String emitTable(Ast.TableDecl table) { return sb.toString(); } + /// Emits the table's `verify(FbsVerifier, long)`: one verifier call per field, recursing into + /// child tables, mirroring flatc's generated `Verifiable` impls. + private void emitVerify(StringBuilder sb, Ast.TableDecl table) { + sb.append(""" + /// Checks this table and everything reachable from it. + /// @param v the verifier walking the buffer + /// @param pos the table's absolute position + static void verify(FbsVerifier v, long pos) { + v.startTable(pos); + """); + int vtableIndex = 0; + for (Ast.Field field : table.fields()) { + int vt = 4 + vtableIndex * 2; + vtableIndex += emitVerifyField(sb, field, vt); + } + sb.append(""" + v.endTable(); + } + + """); + } + + /// Emits the verifier call for one field; returns the vtable slots it uses (two for a union). + private int emitVerifyField(StringBuilder sb, Ast.Field field, int vt) { + String name = field.name(); + boolean required = field.required(); + String call = switch (field.type()) { + case Ast.Scalar scalar when scalar == Ast.Scalar.STRING -> "v.string(pos, %d, \"%s\", %b);" + .formatted(vt, name, required); + case Ast.Scalar scalar -> inlineCall(vt, TypeRegistry.sizeOf(scalar), name, required); + case Ast.Vector vector -> vectorCall(vector.element(), vt, name, required); + case Ast.Ref ref -> switch (registry.resolve(ref)) { + case Ast.EnumDecl e -> inlineCall(vt, TypeRegistry.sizeOf(e.underlying()), name, required); + case Ast.StructDecl s -> inlineCall(vt, registry.layoutOf(s).size(), name, required); + case Ast.TableDecl t -> "v.table(pos, %d, \"%s\", %b, %s::verify);" + .formatted(vt, name, required, className(t.name())); + case Ast.UnionDecl u -> unionCall(u, vt, name); + }; + }; + sb.append(" ").append(call).append('\n'); + return field.type() instanceof Ast.Ref ref && registry.resolve(ref) instanceof Ast.UnionDecl ? 2 : 1; + } + + private static String inlineCall(int vt, int size, String name, boolean required) { + return "v.inline(pos, %d, %d, \"%s\", %b);".formatted(vt, size, name, required); + } + + private String vectorCall(Ast.FieldType element, int vt, String name, boolean required) { + return switch (element) { + case Ast.Scalar scalar when scalar == Ast.Scalar.STRING -> "v.stringVector(pos, %d, \"%s\", %b);" + .formatted(vt, name, required); + case Ast.Scalar scalar -> "v.vector(pos, %d, %d, \"%s\", %b);" + .formatted(vt, TypeRegistry.sizeOf(scalar), name, required); + case Ast.Vector _ -> throw new FbsParseException("nested vectors are not supported ('" + name + "')"); + case Ast.Ref ref -> switch (registry.resolve(ref)) { + case Ast.EnumDecl e -> "v.vector(pos, %d, %d, \"%s\", %b);" + .formatted(vt, TypeRegistry.sizeOf(e.underlying()), name, required); + case Ast.StructDecl s -> "v.vector(pos, %d, %d, \"%s\", %b);" + .formatted(vt, registry.layoutOf(s).size(), name, required); + case Ast.TableDecl t -> "v.tableVector(pos, %d, \"%s\", %b, %s::verify);" + .formatted(vt, name, required, className(t.name())); + case Ast.UnionDecl _ -> throw new FbsParseException("vectors of unions are not supported ('" + name + "')"); + }; + }; + } + + /// A union is a discriminator slot plus a value slot, consistent or rejected. Only a member this + /// schema knows is verified: like flatc's verifiers, an unknown discriminator is accepted, since + /// no accessor reads its value. + private static String unionCall(Ast.UnionDecl union, int vt, String name) { + StringBuilder sb = new StringBuilder(); + String type = camelCase(name) + "Type"; + sb.append("int ").append(type).append(" = v.unionType(pos, ").append(vt).append(", ").append(vt + 2) + .append(", \"").append(name).append("\");\n"); + sb.append(" if (").append(type).append(" != 0) {\n"); + sb.append(" long member = v.unionValue(pos, ").append(vt + 2).append(", \"").append(name).append("\");\n"); + sb.append(" switch (").append(type).append(") {\n"); + long next = 1; + for (Ast.UnionMember m : union.members()) { + long value = m.number().isPresent() ? m.number().getAsInt() : next; + next = value + 1; + sb.append(" case ").append(value).append(" -> ") + .append(className(simpleName(m.typeName()))).append(".verify(v, member);\n"); + } + sb.append(" default -> {\n }\n"); + sb.append(" }\n"); + sb.append(" }"); + return sb.toString(); + } + /// Emits accessors for one table field; returns the next vtable index (a union /// consumes two slots: discriminator + value). private int emitTableField(StringBuilder sb, Ast.Field field, int vtableIndex) { @@ -222,7 +314,13 @@ private void emitVectorField(StringBuilder sb, String accessor, Ast.FieldType el int o = fieldOffset(%2$d); return o != 0 ? vectorLength(o) : 0; } - """.formatted(accessor, vt)); + + /// Tells an absent `%1$s` vector from an empty one, which [#%1$sLength()] cannot. + /// @return whether the `%1$s` vector is present + public boolean has%3$s() { + return fieldOffset(%2$d) != 0; + } + """.formatted(accessor, vt, capitalize(accessor))); if (element instanceof Ast.Scalar scalar) { emitScalarVectorElements(sb, accessor, scalar, vt); diff --git a/fbs-gen/src/test/java/io/github/dfa1/vortex/fbsgen/CodeGenTest.java b/fbs-gen/src/test/java/io/github/dfa1/vortex/fbsgen/CodeGenTest.java index c465d95b1..199f554dc 100644 --- a/fbs-gen/src/test/java/io/github/dfa1/vortex/fbsgen/CodeGenTest.java +++ b/fbs-gen/src/test/java/io/github/dfa1/vortex/fbsgen/CodeGenTest.java @@ -65,6 +65,26 @@ void emitsScalarVectorAndChildAccessorsForTable() throws IOException { .contains("readInt(vectorElements(o) + (long) j * 4) & 0xFFFFFFFFL"); } + @Test + void emitsVerifierRunBeforeEveryRootAccess() throws IOException { + // Given — DType is a union holder, ArraySpec has a required string, Footer has table and + // struct vectors: between them every verifier call shape + String dtype = generateOne("FbsDType.java"); + String arraySpec = generateOne("FbsArraySpec.java"); + String footer = generateOne("FbsFooter.java"); + + // Then — getRootAs verifies first, and verify walks each field with its wire shape + assertThat(dtype) + .contains("return new FbsDType().assign(seg, FbsVerifier.verifyRoot(seg, FbsDType::verify));") + .contains("int typeType = v.unionType(pos, 4, 6, \"type\");") + .contains("long member = v.unionValue(pos, 6, \"type\");") + .contains("case 7 -> FbsStruct.verify(v, member);"); + assertThat(arraySpec).contains("v.string(pos, 4, \"id\", true);"); + assertThat(footer) + .contains("v.tableVector(pos, 4, \"array_specs\", false, FbsArraySpec::verify);") + .contains("v.vector(pos, 8, 16, \"segment_specs\", false);"); + } + @Test void emitsInlineOffsetsForStruct() throws IOException { // Given — Buffer struct: padding u16 @0, alignment u8 @2, compression(enum u8) @3, diff --git a/fuzz/oss-fuzz/Dockerfile b/fuzz/oss-fuzz/Dockerfile new file mode 100644 index 000000000..31bd2d18f --- /dev/null +++ b/fuzz/oss-fuzz/Dockerfile @@ -0,0 +1,9 @@ +# OSS-Fuzz builder for vortex-java (#493). The base image's JDK is 17; the project needs 25, which +# build.sh uses to build and ships next to the fuzzers for them to run on. +FROM gcr.io/oss-fuzz-base/base-builder-jvm +RUN curl -sSL -o jdk.tar.gz \ + "https://api.adoptium.net/v3/binary/latest/25/ga/linux/x64/jdk/hotspot/normal/eclipse" && \ + mkdir $SRC/jdk25 && tar -xzf jdk.tar.gz -C $SRC/jdk25 --strip-components=1 && rm jdk.tar.gz +RUN git clone --depth 1 https://github.com/dfa1/vortex-java vortex-java +COPY build.sh $SRC/ +WORKDIR $SRC/vortex-java diff --git a/fuzz/oss-fuzz/README.md b/fuzz/oss-fuzz/README.md new file mode 100644 index 000000000..da18b930c --- /dev/null +++ b/fuzz/oss-fuzz/README.md @@ -0,0 +1,25 @@ +# OSS-Fuzz integration (prepared, not submitted) + +The files an [OSS-Fuzz](https://google.github.io/oss-fuzz/) project needs, to copy into +`projects/vortex-java/` of a google/oss-fuzz pull request (#493): + +- `project.yaml`: fill in `primary_contact` before submitting. +- `Dockerfile`: the JVM builder image, plus JDK 25 (the image ships 17) and a clone of this repo. +- `build.sh`: builds the `fuzz` module and writes one fuzzer per `@FuzzTest` (reader open, scan, + filtered scan, metadata blobs; FSST round trip), each seeded with its saved inputs. The + `integration` targets need the vortex-jni native library and are left out. + +Checked locally without Docker: `build.sh` (with a local JDK 25 as `$SRC/jdk25`) packages the +fuzzers, and a generated wrapper ran `scanMalformedFile` 20,000 times on Jazzer 0.30's standalone +driver. Not yet run through OSS-Fuzz's own tooling; before submitting, from a google/oss-fuzz +checkout with these files in `projects/vortex-java/`: + +```bash +python3 infra/helper.py build_image vortex-java +python3 infra/helper.py build_fuzzers vortex-java +python3 infra/helper.py check_build vortex-java +python3 infra/helper.py run_fuzzer vortex-java VortexReaderFuzzTest_scanMalformedFile +``` + +OSS-Fuzz accepts projects that are widely used or critical infrastructure; the weekly `Fuzz` +GitHub workflow covers the same targets meanwhile. diff --git a/fuzz/oss-fuzz/build.sh b/fuzz/oss-fuzz/build.sh new file mode 100755 index 000000000..9ff330a2d --- /dev/null +++ b/fuzz/oss-fuzz/build.sh @@ -0,0 +1,45 @@ +#!/bin/bash -eu +# OSS-Fuzz build for vortex-java (#493): every JUnit @FuzzTest of the `fuzz` module becomes one +# fuzzer, run by Jazzer's driver with --target_class/--target_method, seeded with its saved inputs. +# The `integration` targets are left out: they need the vortex-jni native library. + +export JAVA_HOME=$SRC/jdk25 +export PATH=$JAVA_HOME/bin:$PATH + +# One reactor invocation: the sibling modules are packaged, never installed, so copy-dependencies +# must resolve them from this same build. Jazzer comes from OSS-Fuzz, not Maven. +./mvnw -B -q package dependency:copy-dependencies -pl fuzz -am -DskipTests \ + -DincludeScope=test -DexcludeGroupIds=com.code-intelligence -DoutputDirectory="$OUT/libs" +cp -r fuzz/target/test-classes "$OUT/classes" +cp -r "$JAVA_HOME" "$OUT/jdk" + +INPUTS=fuzz/src/test/resources/io/github/dfa1/vortex/fuzz +for target in \ + VortexReaderFuzzTest:openMalformedFile \ + VortexReaderFuzzTest:scanMalformedFile \ + VortexReaderFuzzTest:filterMalformedFile \ + VortexReaderFuzzTest:parseMetadataBlobs \ + FsstFuzzTest:compressDecompressRoundTrip; do + class=${target%%:*} + method=${target##*:} + fuzzer=${class}_${method} + + if compgen -G "$INPUTS/${class}Inputs/$method/*" > /dev/null; then + zip -jq "$OUT/${fuzzer}_seed_corpus.zip" "$INPUTS/${class}Inputs/$method"/* + fi + + # The JDK 25 shipped above runs the fuzzer; OSS-Fuzz's own jazzer jars are on the classpath. + cat > "$OUT/$fuzzer" <vortex-fuzz Coverage-guided fuzz testing (ADR 0020) for the Vortex columnar file format Java implementation. - Opt-in only — see CLAUDE.md. + Saved inputs replay in every build; JAZZER_FUZZ=1 explores new ones — see CLAUDE.md. true - - fuzz @@ -30,6 +25,11 @@ vortex-reader test + + io.github.dfa1.vortex + vortex-writer + test + org.junit.jupiter junit-jupiter @@ -42,23 +42,4 @@ - - - - org.apache.maven.plugins - maven-surefire-plugin - - - ${vortex.fuzz.excludedGroups} - - - - diff --git a/fuzz/src/test/java/io/github/dfa1/vortex/fuzz/ArrayWalker.java b/fuzz/src/test/java/io/github/dfa1/vortex/fuzz/ArrayWalker.java new file mode 100644 index 000000000..d5880700d --- /dev/null +++ b/fuzz/src/test/java/io/github/dfa1/vortex/fuzz/ArrayWalker.java @@ -0,0 +1,165 @@ +package io.github.dfa1.vortex.fuzz; + +import java.lang.foreign.Arena; +import java.util.function.Supplier; + +import io.github.dfa1.vortex.core.error.VortexException; +import io.github.dfa1.vortex.reader.Chunk; +import io.github.dfa1.vortex.reader.array.Array; +import io.github.dfa1.vortex.reader.array.BoolArray; +import io.github.dfa1.vortex.reader.array.ByteArray; +import io.github.dfa1.vortex.reader.array.DecimalArray; +import io.github.dfa1.vortex.reader.array.DoubleArray; +import io.github.dfa1.vortex.reader.array.FixedSizeListArray; +import io.github.dfa1.vortex.reader.array.Float16Array; +import io.github.dfa1.vortex.reader.array.FloatArray; +import io.github.dfa1.vortex.reader.array.IntArray; +import io.github.dfa1.vortex.reader.array.ListArray; +import io.github.dfa1.vortex.reader.array.ListViewArray; +import io.github.dfa1.vortex.reader.array.LongArray; +import io.github.dfa1.vortex.reader.array.MapArray; +import io.github.dfa1.vortex.reader.array.MaskedArray; +import io.github.dfa1.vortex.reader.array.ShortArray; +import io.github.dfa1.vortex.reader.array.StructArray; +import io.github.dfa1.vortex.reader.array.UnionArray; +import io.github.dfa1.vortex.reader.array.VarBinArray; +import io.github.dfa1.vortex.reader.array.VariantArray; + +/// Reads every value of a decoded chunk, so a fuzz target reaches the code that only runs on value +/// access: `Lazy*` arrays decode when read, not when the chunk is built, and the reader's security +/// contract holds there too. +/// +/// Each leaf is read twice over, through the two decode paths it has: bulk +/// [Array#materialize(java.lang.foreign.SegmentAllocator)] and the per-row getters, which are separate +/// code in the lazy arrays. Every column is walked again limited to half its rows, for row-range +/// slicing (where a buffer-alignment abort once hid). A [VortexException] ends only the walk of the +/// array that threw, so one rejected column does not hide the others from the fuzzer. +final class ArrayWalker { + + private ArrayWalker() { + } + + /// Reads every value of every column of `chunk`, whole and limited to half its rows. + /// + /// @param chunk the decoded chunk + /// @param arena allocator for materialized values + static void walk(Chunk chunk, Arena arena) { + for (Chunk.Column column : chunk.columns().values()) { + Array array = column.array(); + walk(array, arena); + walk(() -> Array.limited(array, array.length() / 2), arena); + } + } + + private static void walk(Supplier array, Arena arena) { + try { + walk(array.get(), arena); + } catch (VortexException _) { + // rejected as malformed: the contract holds + } + } + + private static void walk(Array array, Arena arena) { + switch (array) { + case MaskedArray masked -> { + walk(masked::validity, arena); + walk(masked::inner, arena); + } + case StructArray struct -> { + for (int i = 0; i < struct.fieldCount(); i++) { + int field = i; + walk(() -> struct.field(field), arena); + } + } + case ListArray list -> { + walk(list::offsets, arena); + walk(list::elements, arena); + } + case ListViewArray list -> { + walk(list::offsets, arena); + walk(list::sizes, arena); + walk(list::elements, arena); + } + case FixedSizeListArray list -> walk(list::elements, arena); + case MapArray map -> walk(map::entries, arena); + case UnionArray union -> { + walk(union::typeIds, arena); + for (int i = 0; i < union.variantCount(); i++) { + int variant = i; + walk(() -> union.variant(variant), arena); + } + } + case VariantArray variant -> { + walk(variant::coreStorage, arena); + if (variant.shredded() != null) { + walk(variant::shredded, arena); + } + } + default -> leaf(array, arena); + } + } + + private static void leaf(Array array, Arena arena) { + try { + array.materialize(arena); + } catch (VortexException _) { + // rejected as malformed: the per-row path below is separate code, so still walk it + } + long rows = array.length(); + switch (array) { + case LongArray a -> { + for (long i = 0; i < rows; i++) { + a.getLong(i); + } + } + case IntArray a -> { + for (long i = 0; i < rows; i++) { + a.getInt(i); + } + } + case ShortArray a -> { + for (long i = 0; i < rows; i++) { + a.getShort(i); + } + } + case ByteArray a -> { + for (long i = 0; i < rows; i++) { + a.getByte(i); + } + } + case DoubleArray a -> { + for (long i = 0; i < rows; i++) { + a.getDouble(i); + } + } + case FloatArray a -> { + for (long i = 0; i < rows; i++) { + a.getFloat(i); + } + } + case Float16Array a -> { + for (long i = 0; i < rows; i++) { + a.getFloat(i); + } + } + case BoolArray a -> { + for (long i = 0; i < rows; i++) { + a.getBoolean(i); + } + } + case DecimalArray a -> { + for (long i = 0; i < rows; i++) { + a.getDecimal(i); + } + } + case VarBinArray a -> { + for (long i = 0; i < rows; i++) { + a.getBytes(i); + } + } + default -> { + // NullArray, UnknownArray, GenericArray: nothing per row beyond materialize + } + } + } +} diff --git a/fuzz/src/test/java/io/github/dfa1/vortex/fuzz/FsstFuzzTest.java b/fuzz/src/test/java/io/github/dfa1/vortex/fuzz/FsstFuzzTest.java new file mode 100644 index 000000000..3cd245389 --- /dev/null +++ b/fuzz/src/test/java/io/github/dfa1/vortex/fuzz/FsstFuzzTest.java @@ -0,0 +1,59 @@ +package io.github.dfa1.vortex.fuzz; + +import static org.junit.jupiter.api.Assertions.assertArrayEquals; +import static org.junit.jupiter.api.Assertions.assertEquals; + +import java.lang.foreign.MemorySegment; +import java.util.Arrays; + +import com.code_intelligence.jazzer.api.FuzzedDataProvider; +import com.code_intelligence.jazzer.junit.FuzzTest; +import io.github.dfa1.vortex.fsst.Compressor; +import io.github.dfa1.vortex.fsst.CompressorBuilder; +import io.github.dfa1.vortex.fsst.Decompressor; +import org.junit.jupiter.api.Tag; + +/// FSST compressor correctness (#493): a table trained on fuzzed rows must compress every row, +/// fuzzed or not, to codes that decompress back to it, through both the `byte[]` and the +/// `MemorySegment` paths, which must emit the same codes. Not crash fuzzing of [Decompressor]: +/// it documents itself as an internal boundary, not an untrusted-input one; the reader's FSST +/// validation of untrusted files is covered by [VortexReaderFuzzTest]. +@Tag("fuzz") +class FsstFuzzTest { + + @FuzzTest + void compressDecompressRoundTrip(FuzzedDataProvider data) { + // Given + byte[][] rows = new byte[data.consumeInt(0, 64)][]; + for (int i = 0; i < rows.length; i++) { + rows[i] = data.consumeBytes(data.consumeInt(0, 128)); + } + Compressor compressor = new CompressorBuilder().seed(data.consumeLong()).train(rows); + Decompressor decompressor = compressor.toDecompressor(); + // A row the table was not trained on: all escapes is still a valid encoding. + byte[] unseen = data.consumeRemainingAsBytes(); + + // When / Then — every row round-trips (asserted per row) + for (byte[] row : rows) { + roundTrip(compressor, decompressor, row); + } + roundTrip(compressor, decompressor, unseen); + } + + private static void roundTrip(Compressor compressor, Decompressor decompressor, byte[] row) { + // Worst case: one escape pair per input byte; a code expands to at most 8 bytes. + byte[] codes = new byte[2 * row.length]; + int codesEnd = (int) compressor.compress(row, 0, row.length, codes, 0); + + MemorySegment segmentCodes = MemorySegment.ofArray(new byte[2 * row.length]); + long segmentEnd = compressor.compress(MemorySegment.ofArray(row), 0, row.length, segmentCodes, 0); + assertEquals(codesEnd, segmentEnd, "byte[] and MemorySegment code lengths"); + assertArrayEquals(Arrays.copyOf(codes, codesEnd), + segmentCodes.asSlice(0, segmentEnd).toArray(java.lang.foreign.ValueLayout.JAVA_BYTE), + "byte[] and MemorySegment codes"); + + byte[] decoded = new byte[8 * codesEnd]; + int decodedEnd = (int) decompressor.decompress(codes, 0, codesEnd, decoded, 0); + assertArrayEquals(row, Arrays.copyOf(decoded, decodedEnd), "decompress(compress(row))"); + } +} diff --git a/fuzz/src/test/java/io/github/dfa1/vortex/fuzz/MetadataBlobs.java b/fuzz/src/test/java/io/github/dfa1/vortex/fuzz/MetadataBlobs.java new file mode 100644 index 000000000..4f4d3b32d --- /dev/null +++ b/fuzz/src/test/java/io/github/dfa1/vortex/fuzz/MetadataBlobs.java @@ -0,0 +1,67 @@ +package io.github.dfa1.vortex.fuzz; + +import java.nio.ByteBuffer; +import java.nio.ByteOrder; + +import io.github.dfa1.vortex.core.fbs.FbsBuilder; +import io.github.dfa1.vortex.core.fbs.FbsPostscript; +import io.github.dfa1.vortex.core.fbs.FbsPostscriptSegment; +import io.github.dfa1.vortex.core.io.VortexFormat; + +/// Moves a Vortex file's three metadata blobs (footer, layout, dtype FlatBuffers) between their +/// place in a file and the flat form [VortexReaderFuzzTest#parseMetadataBlobs(byte[])] fuzzes. +final class MetadataBlobs { + + private MetadataBlobs() { + } + + /// Lays the blobs `data[from..]` (footer, layout, then dtype) out as a file: the blobs, a + /// postscript pointing at them, and the trailer. + /// + /// @param data the bytes holding the blobs + /// @param from where the footer starts in `data` + /// @param footerLength footer length + /// @param layoutLength layout length + /// @param dtypeLength dtype length (0 for none) + /// @return the file's bytes + static byte[] assemble(byte[] data, int from, int footerLength, int layoutLength, int dtypeLength) { + int body = footerLength + layoutLength + dtypeLength; + FbsBuilder builder = new FbsBuilder(256); + int footer = FbsPostscriptSegment.createFbsPostscriptSegment(builder, 0, footerLength, 0, 0, 0); + int layout = FbsPostscriptSegment.createFbsPostscriptSegment(builder, footerLength, layoutLength, 0, 0, 0); + int dtype = FbsPostscriptSegment.createFbsPostscriptSegment(builder, footerLength + layoutLength, + dtypeLength, 0, 0, 0); + FbsPostscript.finishFbsPostscriptBuffer(builder, FbsPostscript.createFbsPostscript(builder, dtype, layout, 0, footer)); + byte[] postscript = builder.sizedByteArray(); + + ByteBuffer file = ByteBuffer.allocate(body + postscript.length + VortexFormat.TRAILER_SIZE) + .order(ByteOrder.LITTLE_ENDIAN); + file.put(data, from, body).put(postscript) + .putShort((short) VortexFormat.VERSION).putShort((short) postscript.length) + .put(VortexFormat.MAGIC.asByteBuffer()); + return file.array(); + } + + /// The flat fuzz input for a valid file: the blob-length header, then its footer, layout and + /// dtype blobs as its postscript locates them. + /// + /// @param file a Vortex file's bytes + /// @return the header and the three blobs + static byte[] extract(byte[] file) { + ByteBuffer trailer = ByteBuffer.wrap(file, file.length - VortexFormat.TRAILER_SIZE, VortexFormat.TRAILER_SIZE) + .order(ByteOrder.LITTLE_ENDIAN); + trailer.getShort(); + int postscriptLength = Short.toUnsignedInt(trailer.getShort()); + int postscriptStart = file.length - VortexFormat.TRAILER_SIZE - postscriptLength; + var postscript = FbsPostscript.getRootAsFbsPostscript( + java.lang.foreign.MemorySegment.ofArray(file).asSlice(postscriptStart, postscriptLength)); + FbsPostscriptSegment[] segments = {postscript.footer(), postscript.layout(), postscript.dtype()}; + ByteBuffer out = ByteBuffer.allocate(VortexReaderFuzzTest.BLOB_HEADER_BYTES + + (int) (segments[0].length() + segments[1].length() + segments[2].length())); + out.putShort((short) segments[0].length()).putShort((short) segments[1].length()); + for (FbsPostscriptSegment segment : segments) { + out.put(file, (int) segment.offset(), (int) segment.length()); + } + return out.array(); + } +} diff --git a/fuzz/src/test/java/io/github/dfa1/vortex/fuzz/SeedCorpusGenerator.java b/fuzz/src/test/java/io/github/dfa1/vortex/fuzz/SeedCorpusGenerator.java new file mode 100644 index 000000000..66606ae6e --- /dev/null +++ b/fuzz/src/test/java/io/github/dfa1/vortex/fuzz/SeedCorpusGenerator.java @@ -0,0 +1,142 @@ +package io.github.dfa1.vortex.fuzz; + +import static org.junit.jupiter.api.Assertions.assertTrue; + +import java.io.IOException; +import java.nio.ByteBuffer; +import java.nio.channels.FileChannel; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.StandardOpenOption; +import java.util.Arrays; +import java.util.List; +import java.util.Map; +import java.util.Random; +import java.util.function.LongFunction; +import java.util.stream.Stream; + +import io.github.dfa1.vortex.core.model.ColumnName; +import io.github.dfa1.vortex.core.model.DType; +import io.github.dfa1.vortex.core.model.EncodingId; +import io.github.dfa1.vortex.writer.ColumnEncoding; +import io.github.dfa1.vortex.writer.VortexWriter; +import io.github.dfa1.vortex.writer.WriteOptions; +import org.junit.jupiter.api.AfterAll; +import org.junit.jupiter.api.condition.EnabledIfSystemProperty; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.Arguments; +import org.junit.jupiter.params.provider.MethodSource; + +/// Writes the per-encoding seed corpus of [VortexReaderFuzzTest#scanMalformedFile(byte[])] (#493), +/// [VortexReaderFuzzTest#filterMalformedFile(byte[])]'s copy of every scan seed behind a +/// filter-choice header, and [VortexReaderFuzzTest#parseMetadataBlobs(byte[])]'s metadata blobs of +/// every scan seed. +/// +/// A byte mutator never synthesizes a valid FSST symbol table or Pco chunk header from the +/// hand-made seeds, so each encoding gets a small file of its own that already decodes: the +/// fuzzer then mutates from inside the decoder instead of bouncing off its first check. Each +/// column is forced through one encoding with [ColumnEncoding#candidates(EncodingId...)], and +/// shaped so that encoding wins (a candidate that does not pay is silently left canonical, which +/// the assertion on the written bytes catches). Every 13th row is null so validity is exercised. +/// +/// Not part of any build; regenerate (then commit) with +/// `./mvnw test -pl fuzz -am -Dtest=SeedCorpusGenerator -Dsurefire.failIfNoSpecifiedTests=false -Dvortex.fuzz.regenerateSeeds=true` +@EnabledIfSystemProperty(named = "vortex.fuzz.regenerateSeeds", matches = "true") +class SeedCorpusGenerator { + + private static final Path INPUTS = Path.of("src/test/resources/io/github/dfa1/vortex/fuzz/VortexReaderFuzzTestInputs"); + private static final Path OUT = INPUTS.resolve("scanMalformedFile"); + private static final Path FILTER_OUT = INPUTS.resolve("filterMalformedFile"); + private static final Path BLOBS_OUT = INPUTS.resolve("parseMetadataBlobs"); + private static final ColumnName VALUE = ColumnName.of("v"); + // One full 1024-value FastLanes block plus a partial one, still only a few KB per file. + private static final int ROWS = 1_100; + + static Stream seeds() { + Random random = new Random(42); + double[] noise = random.doubles(ROWS).toArray(); + return Stream.of( + seed(DType.I32, i -> (int) (i % 100), EncodingId.FASTLANES_BITPACKED), + seed(DType.I64, i -> 1_000_000L + i % 100, EncodingId.FASTLANES_FOR), + seed(DType.I64, i -> i * 3 + i % 2, EncodingId.FASTLANES_DELTA), + seed(DType.I32, i -> (int) (i / 50), EncodingId.FASTLANES_RLE), + seed(DType.I64, i -> i / 100, EncodingId.VORTEX_RUNEND), + seed(DType.I32, i -> (int) (i % 2 == 0 ? i % 50 : -(i % 50)), EncodingId.VORTEX_ZIGZAG), + seed(DType.I64, i -> i % 97 == 0 ? i : 0L, EncodingId.VORTEX_SPARSE), + seed(DType.I64, i -> (i * 2_654_435_761L) % 1_000_003, EncodingId.VORTEX_PCO), + seed(DType.F64, i -> (i % 1000) / 100.0, EncodingId.VORTEX_ALP), + seed(DType.F64, i -> noise[(int) i], EncodingId.VORTEX_ALPRD, EncodingId.FASTLANES_BITPACKED), + seed(DType.UTF8, i -> "category-" + i % 5, EncodingId.VORTEX_DICT), + seed(DType.UTF8, i -> "https://example.com/items/" + i % 300 + "/detail", EncodingId.VORTEX_FSST), + seed(DType.UTF8, i -> "https://example.com/items/" + i % 300 + "/detail", EncodingId.VORTEX_ONPAIR), + seed(DType.UTF8, i -> "row " + i % 40 + " of the zstd seed", EncodingId.VORTEX_ZSTD)); + } + + /// Mirrors every scan seed (not the crash inputs) into the filter target's corpus behind a + /// header choosing column 0 and a predicate kind that cycles across seeds, against 3. + @AfterAll + static void writeFilterSeeds() throws IOException { + Files.createDirectories(FILTER_OUT); + List seeds = scanSeeds(); + for (int i = 0; i < seeds.size(); i++) { + ByteBuffer header = ByteBuffer.allocate(VortexReaderFuzzTest.FILTER_HEADER_BYTES) + .putInt(0).put((byte) i).putLong(3L); + byte[] file = Files.readAllBytes(seeds.get(i)); + byte[] seed = Arrays.copyOf(header.array(), header.capacity() + file.length); + System.arraycopy(file, 0, seed, header.capacity(), file.length); + Files.write(FILTER_OUT.resolve(seeds.get(i).getFileName()), seed); + } + } + + /// Mirrors every scan seed's footer, layout and dtype blobs into the metadata-blob target's + /// corpus, in the flat form it fuzzes. + @AfterAll + static void writeBlobSeeds() throws IOException { + Files.createDirectories(BLOBS_OUT); + for (Path seed : scanSeeds()) { + Files.write(BLOBS_OUT.resolve(seed.getFileName()), MetadataBlobs.extract(Files.readAllBytes(seed))); + } + } + + private static List scanSeeds() throws IOException { + try (Stream files = Files.list(OUT)) { + return files.filter(f -> f.getFileName().toString().startsWith("seed-")).sorted().toList(); + } + } + + @ParameterizedTest(name = "{0}") + @MethodSource("seeds") + void writeSeed(String name, DType dtype, Object values, EncodingId[] candidates) throws IOException { + // Given + Path file = OUT.resolve("seed-" + name); + DType.Struct schema = new DType.Struct(List.of(VALUE), List.of(dtype.asNullable()), false); + WriteOptions options = WriteOptions.defaults().withoutEditions() + .withColumnEncoding(VALUE, ColumnEncoding.candidates(candidates)); + + // When + try (var ch = FileChannel.open(file, StandardOpenOption.CREATE, StandardOpenOption.WRITE, + StandardOpenOption.TRUNCATE_EXISTING); + var writer = VortexWriter.create(ch, schema, options)) { + writer.writeChunk(Map.of(VALUE, values)); + } + + // Then + String result = new String(Files.readAllBytes(file), StandardCharsets.ISO_8859_1); + assertTrue(result.contains(candidates[0].toString()), candidates[0] + " lost to the canonical encoding"); + } + + private static Arguments seed(DType dtype, LongFunction value, EncodingId... candidates) { + Object[] values = switch (dtype) { + case DType.Primitive p when p.equals(DType.I32) -> new Integer[ROWS]; + case DType.Primitive p when p.equals(DType.I64) -> new Long[ROWS]; + case DType.Primitive p when p.equals(DType.F64) -> new Double[ROWS]; + default -> new String[ROWS]; + }; + for (int i = 0; i < ROWS; i++) { + values[i] = i % 13 == 0 ? null : value.apply(i); + } + String name = candidates[0].toString().replace('.', '-'); + return Arguments.of(name, dtype, values, candidates); + } +} diff --git a/fuzz/src/test/java/io/github/dfa1/vortex/fuzz/VortexReaderFuzzTest.java b/fuzz/src/test/java/io/github/dfa1/vortex/fuzz/VortexReaderFuzzTest.java index d93aa7a2f..5ae339fd5 100644 --- a/fuzz/src/test/java/io/github/dfa1/vortex/fuzz/VortexReaderFuzzTest.java +++ b/fuzz/src/test/java/io/github/dfa1/vortex/fuzz/VortexReaderFuzzTest.java @@ -1,32 +1,46 @@ package io.github.dfa1.vortex.fuzz; import java.io.IOException; +import java.lang.foreign.Arena; +import java.nio.ByteBuffer; import java.nio.file.Files; import java.nio.file.Path; +import java.util.Arrays; import com.code_intelligence.jazzer.junit.FuzzTest; import io.github.dfa1.vortex.core.error.VortexException; +import io.github.dfa1.vortex.core.model.ColumnName; +import io.github.dfa1.vortex.core.model.DType; +import io.github.dfa1.vortex.reader.Chunk; +import io.github.dfa1.vortex.reader.RowFilter; +import io.github.dfa1.vortex.reader.ScanIterator; +import io.github.dfa1.vortex.reader.ScanOptions; import io.github.dfa1.vortex.reader.VortexReader; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Tag; import org.junit.jupiter.api.io.TempDir; -/// Coverage-guided fuzzing of the full-file open path (ADR 0020). The reader's security contract -/// requires every malformed input to surface as a [VortexException] — anything else escaping this -/// driver (an [Error], a raw `IndexOutOfBoundsException`, a FlatBuffer/Protobuf parser exception) -/// is a real finding, so the fuzzer's pass/fail oracle is simply "no other throwable escapes". +/// Coverage-guided fuzzing of the reader (ADR 0020). The reader's security contract requires every +/// malformed input to surface as a [VortexException] — anything else escaping a target (an [Error], +/// a raw `IndexOutOfBoundsException`, a FlatBuffer/Protobuf parser exception) is a real finding, so +/// the fuzzer's pass/fail oracle is simply "no other throwable escapes". /// /// There is no `VortexReader.open(byte[])`: the public API memory-maps a real file, so the fuzzer /// bytes are written to a temp file and fed through the actual [VortexReader#open(Path)] entry -/// point — exactly the trailer / postscript / footer / layout / dtype parse path a malformed file -/// on disk would hit. +/// point — exactly the path a malformed file on disk would hit. /// -/// Tagged `fuzz` so a routine build skips it; run with -/// `./mvnw test -pl fuzz -am -Dvortex.fuzz.excludedGroups=` -/// (add `JAZZER_FUZZ=1` to actually explore new inputs rather than replay the corpus). +/// Every target replays its saved inputs in a routine `./mvnw test`; run with `JAZZER_FUZZ=1` to +/// explore new inputs instead (see CLAUDE.md). @Tag("fuzz") class VortexReaderFuzzTest { + /// Length of the filter-choice header in front of the file in + /// [#filterMalformedFile(byte[])]'s inputs. + static final int FILTER_HEADER_BYTES = 16; + + /// Length of the blob-length header in front of [#parseMetadataBlobs(byte[])]'s inputs. + static final int BLOB_HEADER_BYTES = 4; + // @FuzzTest defaults to PER_TEST lifecycle: the test instance — and therefore this injected // temp directory — is created once and reused across every fuzzing execution. @TempDir @@ -39,9 +53,14 @@ void setUp() { fuzzFile = tempDir.resolve("fuzz.vortex"); } + /// The open path alone: trailer, postscript, footer, layout and dtype. Fastest target, so it + /// explores the file-level metadata deepest. @FuzzTest void openMalformedFile(byte[] data) throws IOException { + // Given Files.write(fuzzFile, data); + + // When / Then — anything but a VortexException escaping is a finding try (var _ = VortexReader.open(fuzzFile)) { // Successfully opened: fine. The security contract only requires that MALFORMED input // fails as VortexException, not that every fuzzer-generated byte string is rejected. @@ -49,4 +68,110 @@ void openMalformedFile(byte[] data) throws IOException { // Malformed input correctly rejected per the reader's security contract. } } + + /// Past the open into the data: file statistics, then every value of every chunk through + /// [ArrayWalker], so the encoding decoders and the `Lazy*` arrays see fuzzed input. Seeded with + /// one small file per encoding ([SeedCorpusGenerator]). + @FuzzTest + void scanMalformedFile(byte[] data) throws IOException { + // Given + Files.write(fuzzFile, data); + + // When / Then — anything but a VortexException escaping is a finding + try (VortexReader reader = VortexReader.open(fuzzFile)) { + reader.columnStats(); + scan(reader, ScanOptions.all()); + } catch (VortexException _) { + // Malformed input correctly rejected per the reader's security contract. + } + } + + /// The scan again, filtered: zone-map tables are parsed and chunks pruned against a filter the + /// fuzzer chooses. The input is a [#FILTER_HEADER_BYTES]-byte header of filter choices followed + /// by the file, rather than numbers drawn from a `FuzzedDataProvider`, which takes them from the + /// end of the input, where the file's trailer is: seeds stay valid files. + @FuzzTest + void filterMalformedFile(byte[] data) throws IOException { + if (data.length < FILTER_HEADER_BYTES) { + return; + } + // Given + ByteBuffer header = ByteBuffer.wrap(data, 0, FILTER_HEADER_BYTES); + Files.write(fuzzFile, Arrays.copyOfRange(data, FILTER_HEADER_BYTES, data.length)); + + // When / Then — anything but a VortexException escaping is a finding + try (VortexReader reader = VortexReader.open(fuzzFile)) { + if (reader.dtype() instanceof DType.Struct schema && !schema.fieldNames().isEmpty()) { + int column = Math.floorMod(header.getInt(), schema.fieldNames().size()); + RowFilter filter = filter(schema.fieldNames().get(column), schema.fieldTypes().get(column), + header.get(), header.getLong()); + scan(reader, ScanOptions.all().withFilter(filter)); + } + } catch (VortexException _) { + // Malformed input correctly rejected per the reader's security contract. + } + } + + /// `PostscriptParser.parseBlobs` reached directly: the input is the footer, layout and dtype + /// FlatBuffers themselves (behind a 4-byte header of the first two lengths), wrapped here in a + /// valid postscript and trailer, so every mutation lands in the metadata parse instead of being + /// spent breaking the file framing [#openMalformedFile(byte[])] has to get past first. Built + /// through the public API: no access seam into the reader's package. + @FuzzTest + void parseMetadataBlobs(byte[] data) throws IOException { + // Given + if (data.length < BLOB_HEADER_BYTES) { + return; + } + ByteBuffer header = ByteBuffer.wrap(data, 0, BLOB_HEADER_BYTES); + int footerLength = Short.toUnsignedInt(header.getShort()); + int layoutLength = Short.toUnsignedInt(header.getShort()); + int dtypeLength = data.length - BLOB_HEADER_BYTES - footerLength - layoutLength; + if (dtypeLength < 0) { + return; + } + Files.write(fuzzFile, MetadataBlobs.assemble(data, BLOB_HEADER_BYTES, footerLength, layoutLength, dtypeLength)); + + // When / Then — anything but a VortexException escaping is a finding + try (var _ = VortexReader.open(fuzzFile)) { + // Parsed: fine, as for openMalformedFile. + } catch (VortexException _) { + // Malformed input correctly rejected per the reader's security contract. + } + } + + private static void scan(VortexReader reader, ScanOptions options) { + try (ScanIterator chunks = reader.scan(options)) { + while (chunks.hasNext()) { + try (Chunk chunk = chunks.next(); Arena arena = Arena.ofConfined()) { + ArrayWalker.walk(chunk, arena); + } + } + } + } + + /// A predicate on `column` with a value of the type its dtype compares against: a mismatched + /// value is a caller error, not malformed input, so it would only produce false findings. + private static RowFilter filter(ColumnName column, DType dtype, byte kind, long bits) { + Comparable value = switch (dtype) { + case DType.Primitive p when p.ptype().isFloating() -> Double.longBitsToDouble(bits); + case DType.Primitive _ -> bits; + case DType.Utf8 _ -> Long.toString(bits); + case DType.Bool _ -> (bits & 1) != 0; + default -> null; + }; + if (value == null) { + return (kind & 1) == 0 ? RowFilter.isNull(column) : RowFilter.isNotNull(column); + } + return switch (Math.floorMod(kind, 8)) { + case 0 -> RowFilter.eq(column, value); + case 1 -> RowFilter.neq(column, value); + case 2 -> RowFilter.gt(column, value); + case 3 -> RowFilter.gte(column, value); + case 4 -> RowFilter.lt(column, value); + case 5 -> RowFilter.lte(column, value); + case 6 -> RowFilter.isNull(column); + default -> RowFilter.isNotNull(column); + }; + } } diff --git a/fuzz/src/test/resources/io/github/dfa1/vortex/fuzz/VortexReaderFuzzTestInputs/filterMalformedFile/crash-materialized-empty-buffer b/fuzz/src/test/resources/io/github/dfa1/vortex/fuzz/VortexReaderFuzzTestInputs/filterMalformedFile/crash-materialized-empty-buffer new file mode 100644 index 000000000..1bd68d664 Binary files /dev/null and b/fuzz/src/test/resources/io/github/dfa1/vortex/fuzz/VortexReaderFuzzTestInputs/filterMalformedFile/crash-materialized-empty-buffer differ diff --git a/fuzz/src/test/resources/io/github/dfa1/vortex/fuzz/VortexReaderFuzzTestInputs/filterMalformedFile/seed-booleans b/fuzz/src/test/resources/io/github/dfa1/vortex/fuzz/VortexReaderFuzzTestInputs/filterMalformedFile/seed-booleans new file mode 100644 index 000000000..6486211d6 Binary files /dev/null and b/fuzz/src/test/resources/io/github/dfa1/vortex/fuzz/VortexReaderFuzzTestInputs/filterMalformedFile/seed-booleans differ diff --git a/fuzz/src/test/resources/io/github/dfa1/vortex/fuzz/VortexReaderFuzzTestInputs/filterMalformedFile/seed-chunked b/fuzz/src/test/resources/io/github/dfa1/vortex/fuzz/VortexReaderFuzzTestInputs/filterMalformedFile/seed-chunked new file mode 100644 index 000000000..13c6d63ab Binary files /dev/null and b/fuzz/src/test/resources/io/github/dfa1/vortex/fuzz/VortexReaderFuzzTestInputs/filterMalformedFile/seed-chunked differ diff --git a/fuzz/src/test/resources/io/github/dfa1/vortex/fuzz/VortexReaderFuzzTestInputs/filterMalformedFile/seed-fastlanes-bitpacked b/fuzz/src/test/resources/io/github/dfa1/vortex/fuzz/VortexReaderFuzzTestInputs/filterMalformedFile/seed-fastlanes-bitpacked new file mode 100644 index 000000000..dcbe8b2a4 Binary files /dev/null and b/fuzz/src/test/resources/io/github/dfa1/vortex/fuzz/VortexReaderFuzzTestInputs/filterMalformedFile/seed-fastlanes-bitpacked differ diff --git a/fuzz/src/test/resources/io/github/dfa1/vortex/fuzz/VortexReaderFuzzTestInputs/filterMalformedFile/seed-fastlanes-delta b/fuzz/src/test/resources/io/github/dfa1/vortex/fuzz/VortexReaderFuzzTestInputs/filterMalformedFile/seed-fastlanes-delta new file mode 100644 index 000000000..6f4dda829 Binary files /dev/null and b/fuzz/src/test/resources/io/github/dfa1/vortex/fuzz/VortexReaderFuzzTestInputs/filterMalformedFile/seed-fastlanes-delta differ diff --git a/fuzz/src/test/resources/io/github/dfa1/vortex/fuzz/VortexReaderFuzzTestInputs/filterMalformedFile/seed-fastlanes-for b/fuzz/src/test/resources/io/github/dfa1/vortex/fuzz/VortexReaderFuzzTestInputs/filterMalformedFile/seed-fastlanes-for new file mode 100644 index 000000000..81a5ab6e2 Binary files /dev/null and b/fuzz/src/test/resources/io/github/dfa1/vortex/fuzz/VortexReaderFuzzTestInputs/filterMalformedFile/seed-fastlanes-for differ diff --git a/fuzz/src/test/resources/io/github/dfa1/vortex/fuzz/VortexReaderFuzzTestInputs/filterMalformedFile/seed-fastlanes-rle b/fuzz/src/test/resources/io/github/dfa1/vortex/fuzz/VortexReaderFuzzTestInputs/filterMalformedFile/seed-fastlanes-rle new file mode 100644 index 000000000..d577f1920 Binary files /dev/null and b/fuzz/src/test/resources/io/github/dfa1/vortex/fuzz/VortexReaderFuzzTestInputs/filterMalformedFile/seed-fastlanes-rle differ diff --git a/fuzz/src/test/resources/io/github/dfa1/vortex/fuzz/VortexReaderFuzzTestInputs/filterMalformedFile/seed-null b/fuzz/src/test/resources/io/github/dfa1/vortex/fuzz/VortexReaderFuzzTestInputs/filterMalformedFile/seed-null new file mode 100644 index 000000000..71ee7668c Binary files /dev/null and b/fuzz/src/test/resources/io/github/dfa1/vortex/fuzz/VortexReaderFuzzTestInputs/filterMalformedFile/seed-null differ diff --git a/fuzz/src/test/resources/io/github/dfa1/vortex/fuzz/VortexReaderFuzzTestInputs/filterMalformedFile/seed-primitives b/fuzz/src/test/resources/io/github/dfa1/vortex/fuzz/VortexReaderFuzzTestInputs/filterMalformedFile/seed-primitives new file mode 100644 index 000000000..51eac6cfd Binary files /dev/null and b/fuzz/src/test/resources/io/github/dfa1/vortex/fuzz/VortexReaderFuzzTestInputs/filterMalformedFile/seed-primitives differ diff --git a/fuzz/src/test/resources/io/github/dfa1/vortex/fuzz/VortexReaderFuzzTestInputs/filterMalformedFile/seed-varbin b/fuzz/src/test/resources/io/github/dfa1/vortex/fuzz/VortexReaderFuzzTestInputs/filterMalformedFile/seed-varbin new file mode 100644 index 000000000..c3587d9c1 Binary files /dev/null and b/fuzz/src/test/resources/io/github/dfa1/vortex/fuzz/VortexReaderFuzzTestInputs/filterMalformedFile/seed-varbin differ diff --git a/fuzz/src/test/resources/io/github/dfa1/vortex/fuzz/VortexReaderFuzzTestInputs/filterMalformedFile/seed-vortex-alp b/fuzz/src/test/resources/io/github/dfa1/vortex/fuzz/VortexReaderFuzzTestInputs/filterMalformedFile/seed-vortex-alp new file mode 100644 index 000000000..fbde4e3d9 Binary files /dev/null and b/fuzz/src/test/resources/io/github/dfa1/vortex/fuzz/VortexReaderFuzzTestInputs/filterMalformedFile/seed-vortex-alp differ diff --git a/fuzz/src/test/resources/io/github/dfa1/vortex/fuzz/VortexReaderFuzzTestInputs/filterMalformedFile/seed-vortex-alprd b/fuzz/src/test/resources/io/github/dfa1/vortex/fuzz/VortexReaderFuzzTestInputs/filterMalformedFile/seed-vortex-alprd new file mode 100644 index 000000000..78b83648d Binary files /dev/null and b/fuzz/src/test/resources/io/github/dfa1/vortex/fuzz/VortexReaderFuzzTestInputs/filterMalformedFile/seed-vortex-alprd differ diff --git a/fuzz/src/test/resources/io/github/dfa1/vortex/fuzz/VortexReaderFuzzTestInputs/filterMalformedFile/seed-vortex-dict b/fuzz/src/test/resources/io/github/dfa1/vortex/fuzz/VortexReaderFuzzTestInputs/filterMalformedFile/seed-vortex-dict new file mode 100644 index 000000000..2ecb1c994 Binary files /dev/null and b/fuzz/src/test/resources/io/github/dfa1/vortex/fuzz/VortexReaderFuzzTestInputs/filterMalformedFile/seed-vortex-dict differ diff --git a/fuzz/src/test/resources/io/github/dfa1/vortex/fuzz/VortexReaderFuzzTestInputs/filterMalformedFile/seed-vortex-fsst b/fuzz/src/test/resources/io/github/dfa1/vortex/fuzz/VortexReaderFuzzTestInputs/filterMalformedFile/seed-vortex-fsst new file mode 100644 index 000000000..dccab9983 Binary files /dev/null and b/fuzz/src/test/resources/io/github/dfa1/vortex/fuzz/VortexReaderFuzzTestInputs/filterMalformedFile/seed-vortex-fsst differ diff --git a/fuzz/src/test/resources/io/github/dfa1/vortex/fuzz/VortexReaderFuzzTestInputs/filterMalformedFile/seed-vortex-onpair b/fuzz/src/test/resources/io/github/dfa1/vortex/fuzz/VortexReaderFuzzTestInputs/filterMalformedFile/seed-vortex-onpair new file mode 100644 index 000000000..405618c31 Binary files /dev/null and b/fuzz/src/test/resources/io/github/dfa1/vortex/fuzz/VortexReaderFuzzTestInputs/filterMalformedFile/seed-vortex-onpair differ diff --git a/fuzz/src/test/resources/io/github/dfa1/vortex/fuzz/VortexReaderFuzzTestInputs/filterMalformedFile/seed-vortex-pco b/fuzz/src/test/resources/io/github/dfa1/vortex/fuzz/VortexReaderFuzzTestInputs/filterMalformedFile/seed-vortex-pco new file mode 100644 index 000000000..e2a9ced7b Binary files /dev/null and b/fuzz/src/test/resources/io/github/dfa1/vortex/fuzz/VortexReaderFuzzTestInputs/filterMalformedFile/seed-vortex-pco differ diff --git a/fuzz/src/test/resources/io/github/dfa1/vortex/fuzz/VortexReaderFuzzTestInputs/filterMalformedFile/seed-vortex-runend b/fuzz/src/test/resources/io/github/dfa1/vortex/fuzz/VortexReaderFuzzTestInputs/filterMalformedFile/seed-vortex-runend new file mode 100644 index 000000000..0d759c71f Binary files /dev/null and b/fuzz/src/test/resources/io/github/dfa1/vortex/fuzz/VortexReaderFuzzTestInputs/filterMalformedFile/seed-vortex-runend differ diff --git a/fuzz/src/test/resources/io/github/dfa1/vortex/fuzz/VortexReaderFuzzTestInputs/filterMalformedFile/seed-vortex-sparse b/fuzz/src/test/resources/io/github/dfa1/vortex/fuzz/VortexReaderFuzzTestInputs/filterMalformedFile/seed-vortex-sparse new file mode 100644 index 000000000..fbac83fa1 Binary files /dev/null and b/fuzz/src/test/resources/io/github/dfa1/vortex/fuzz/VortexReaderFuzzTestInputs/filterMalformedFile/seed-vortex-sparse differ diff --git a/fuzz/src/test/resources/io/github/dfa1/vortex/fuzz/VortexReaderFuzzTestInputs/filterMalformedFile/seed-vortex-zigzag b/fuzz/src/test/resources/io/github/dfa1/vortex/fuzz/VortexReaderFuzzTestInputs/filterMalformedFile/seed-vortex-zigzag new file mode 100644 index 000000000..d54d19f7d Binary files /dev/null and b/fuzz/src/test/resources/io/github/dfa1/vortex/fuzz/VortexReaderFuzzTestInputs/filterMalformedFile/seed-vortex-zigzag differ diff --git a/fuzz/src/test/resources/io/github/dfa1/vortex/fuzz/VortexReaderFuzzTestInputs/filterMalformedFile/seed-vortex-zstd b/fuzz/src/test/resources/io/github/dfa1/vortex/fuzz/VortexReaderFuzzTestInputs/filterMalformedFile/seed-vortex-zstd new file mode 100644 index 000000000..4292eda65 Binary files /dev/null and b/fuzz/src/test/resources/io/github/dfa1/vortex/fuzz/VortexReaderFuzzTestInputs/filterMalformedFile/seed-vortex-zstd differ diff --git a/fuzz/src/test/resources/io/github/dfa1/vortex/fuzz/VortexReaderFuzzTestInputs/filterMalformedFile/seed-zstd-buffers b/fuzz/src/test/resources/io/github/dfa1/vortex/fuzz/VortexReaderFuzzTestInputs/filterMalformedFile/seed-zstd-buffers new file mode 100644 index 000000000..71ea034b0 Binary files /dev/null and b/fuzz/src/test/resources/io/github/dfa1/vortex/fuzz/VortexReaderFuzzTestInputs/filterMalformedFile/seed-zstd-buffers differ diff --git a/fuzz/src/test/resources/io/github/dfa1/vortex/fuzz/VortexReaderFuzzTestInputs/parseMetadataBlobs/seed-booleans b/fuzz/src/test/resources/io/github/dfa1/vortex/fuzz/VortexReaderFuzzTestInputs/parseMetadataBlobs/seed-booleans new file mode 100644 index 000000000..a0b6fdfdc Binary files /dev/null and b/fuzz/src/test/resources/io/github/dfa1/vortex/fuzz/VortexReaderFuzzTestInputs/parseMetadataBlobs/seed-booleans differ diff --git a/fuzz/src/test/resources/io/github/dfa1/vortex/fuzz/VortexReaderFuzzTestInputs/parseMetadataBlobs/seed-chunked b/fuzz/src/test/resources/io/github/dfa1/vortex/fuzz/VortexReaderFuzzTestInputs/parseMetadataBlobs/seed-chunked new file mode 100644 index 000000000..654c620cd Binary files /dev/null and b/fuzz/src/test/resources/io/github/dfa1/vortex/fuzz/VortexReaderFuzzTestInputs/parseMetadataBlobs/seed-chunked differ diff --git a/fuzz/src/test/resources/io/github/dfa1/vortex/fuzz/VortexReaderFuzzTestInputs/parseMetadataBlobs/seed-fastlanes-bitpacked b/fuzz/src/test/resources/io/github/dfa1/vortex/fuzz/VortexReaderFuzzTestInputs/parseMetadataBlobs/seed-fastlanes-bitpacked new file mode 100644 index 000000000..4f1dd0337 Binary files /dev/null and b/fuzz/src/test/resources/io/github/dfa1/vortex/fuzz/VortexReaderFuzzTestInputs/parseMetadataBlobs/seed-fastlanes-bitpacked differ diff --git a/fuzz/src/test/resources/io/github/dfa1/vortex/fuzz/VortexReaderFuzzTestInputs/parseMetadataBlobs/seed-fastlanes-delta b/fuzz/src/test/resources/io/github/dfa1/vortex/fuzz/VortexReaderFuzzTestInputs/parseMetadataBlobs/seed-fastlanes-delta new file mode 100644 index 000000000..843e4fc09 Binary files /dev/null and b/fuzz/src/test/resources/io/github/dfa1/vortex/fuzz/VortexReaderFuzzTestInputs/parseMetadataBlobs/seed-fastlanes-delta differ diff --git a/fuzz/src/test/resources/io/github/dfa1/vortex/fuzz/VortexReaderFuzzTestInputs/parseMetadataBlobs/seed-fastlanes-for b/fuzz/src/test/resources/io/github/dfa1/vortex/fuzz/VortexReaderFuzzTestInputs/parseMetadataBlobs/seed-fastlanes-for new file mode 100644 index 000000000..9beda946f Binary files /dev/null and b/fuzz/src/test/resources/io/github/dfa1/vortex/fuzz/VortexReaderFuzzTestInputs/parseMetadataBlobs/seed-fastlanes-for differ diff --git a/fuzz/src/test/resources/io/github/dfa1/vortex/fuzz/VortexReaderFuzzTestInputs/parseMetadataBlobs/seed-fastlanes-rle b/fuzz/src/test/resources/io/github/dfa1/vortex/fuzz/VortexReaderFuzzTestInputs/parseMetadataBlobs/seed-fastlanes-rle new file mode 100644 index 000000000..17de94658 Binary files /dev/null and b/fuzz/src/test/resources/io/github/dfa1/vortex/fuzz/VortexReaderFuzzTestInputs/parseMetadataBlobs/seed-fastlanes-rle differ diff --git a/fuzz/src/test/resources/io/github/dfa1/vortex/fuzz/VortexReaderFuzzTestInputs/parseMetadataBlobs/seed-null b/fuzz/src/test/resources/io/github/dfa1/vortex/fuzz/VortexReaderFuzzTestInputs/parseMetadataBlobs/seed-null new file mode 100644 index 000000000..4ecb7310a Binary files /dev/null and b/fuzz/src/test/resources/io/github/dfa1/vortex/fuzz/VortexReaderFuzzTestInputs/parseMetadataBlobs/seed-null differ diff --git a/fuzz/src/test/resources/io/github/dfa1/vortex/fuzz/VortexReaderFuzzTestInputs/parseMetadataBlobs/seed-primitives b/fuzz/src/test/resources/io/github/dfa1/vortex/fuzz/VortexReaderFuzzTestInputs/parseMetadataBlobs/seed-primitives new file mode 100644 index 000000000..623abe14a Binary files /dev/null and b/fuzz/src/test/resources/io/github/dfa1/vortex/fuzz/VortexReaderFuzzTestInputs/parseMetadataBlobs/seed-primitives differ diff --git a/fuzz/src/test/resources/io/github/dfa1/vortex/fuzz/VortexReaderFuzzTestInputs/parseMetadataBlobs/seed-varbin b/fuzz/src/test/resources/io/github/dfa1/vortex/fuzz/VortexReaderFuzzTestInputs/parseMetadataBlobs/seed-varbin new file mode 100644 index 000000000..7c1ed66bf Binary files /dev/null and b/fuzz/src/test/resources/io/github/dfa1/vortex/fuzz/VortexReaderFuzzTestInputs/parseMetadataBlobs/seed-varbin differ diff --git a/fuzz/src/test/resources/io/github/dfa1/vortex/fuzz/VortexReaderFuzzTestInputs/parseMetadataBlobs/seed-vortex-alp b/fuzz/src/test/resources/io/github/dfa1/vortex/fuzz/VortexReaderFuzzTestInputs/parseMetadataBlobs/seed-vortex-alp new file mode 100644 index 000000000..df685fbef Binary files /dev/null and b/fuzz/src/test/resources/io/github/dfa1/vortex/fuzz/VortexReaderFuzzTestInputs/parseMetadataBlobs/seed-vortex-alp differ diff --git a/fuzz/src/test/resources/io/github/dfa1/vortex/fuzz/VortexReaderFuzzTestInputs/parseMetadataBlobs/seed-vortex-alprd b/fuzz/src/test/resources/io/github/dfa1/vortex/fuzz/VortexReaderFuzzTestInputs/parseMetadataBlobs/seed-vortex-alprd new file mode 100644 index 000000000..781d7a268 Binary files /dev/null and b/fuzz/src/test/resources/io/github/dfa1/vortex/fuzz/VortexReaderFuzzTestInputs/parseMetadataBlobs/seed-vortex-alprd differ diff --git a/fuzz/src/test/resources/io/github/dfa1/vortex/fuzz/VortexReaderFuzzTestInputs/parseMetadataBlobs/seed-vortex-dict b/fuzz/src/test/resources/io/github/dfa1/vortex/fuzz/VortexReaderFuzzTestInputs/parseMetadataBlobs/seed-vortex-dict new file mode 100644 index 000000000..3e8ebaf83 Binary files /dev/null and b/fuzz/src/test/resources/io/github/dfa1/vortex/fuzz/VortexReaderFuzzTestInputs/parseMetadataBlobs/seed-vortex-dict differ diff --git a/fuzz/src/test/resources/io/github/dfa1/vortex/fuzz/VortexReaderFuzzTestInputs/parseMetadataBlobs/seed-vortex-fsst b/fuzz/src/test/resources/io/github/dfa1/vortex/fuzz/VortexReaderFuzzTestInputs/parseMetadataBlobs/seed-vortex-fsst new file mode 100644 index 000000000..0983b9a84 Binary files /dev/null and b/fuzz/src/test/resources/io/github/dfa1/vortex/fuzz/VortexReaderFuzzTestInputs/parseMetadataBlobs/seed-vortex-fsst differ diff --git a/fuzz/src/test/resources/io/github/dfa1/vortex/fuzz/VortexReaderFuzzTestInputs/parseMetadataBlobs/seed-vortex-onpair b/fuzz/src/test/resources/io/github/dfa1/vortex/fuzz/VortexReaderFuzzTestInputs/parseMetadataBlobs/seed-vortex-onpair new file mode 100644 index 000000000..bd2bba527 Binary files /dev/null and b/fuzz/src/test/resources/io/github/dfa1/vortex/fuzz/VortexReaderFuzzTestInputs/parseMetadataBlobs/seed-vortex-onpair differ diff --git a/fuzz/src/test/resources/io/github/dfa1/vortex/fuzz/VortexReaderFuzzTestInputs/parseMetadataBlobs/seed-vortex-pco b/fuzz/src/test/resources/io/github/dfa1/vortex/fuzz/VortexReaderFuzzTestInputs/parseMetadataBlobs/seed-vortex-pco new file mode 100644 index 000000000..ba326dd06 Binary files /dev/null and b/fuzz/src/test/resources/io/github/dfa1/vortex/fuzz/VortexReaderFuzzTestInputs/parseMetadataBlobs/seed-vortex-pco differ diff --git a/fuzz/src/test/resources/io/github/dfa1/vortex/fuzz/VortexReaderFuzzTestInputs/parseMetadataBlobs/seed-vortex-runend b/fuzz/src/test/resources/io/github/dfa1/vortex/fuzz/VortexReaderFuzzTestInputs/parseMetadataBlobs/seed-vortex-runend new file mode 100644 index 000000000..e1fb13b91 Binary files /dev/null and b/fuzz/src/test/resources/io/github/dfa1/vortex/fuzz/VortexReaderFuzzTestInputs/parseMetadataBlobs/seed-vortex-runend differ diff --git a/fuzz/src/test/resources/io/github/dfa1/vortex/fuzz/VortexReaderFuzzTestInputs/parseMetadataBlobs/seed-vortex-sparse b/fuzz/src/test/resources/io/github/dfa1/vortex/fuzz/VortexReaderFuzzTestInputs/parseMetadataBlobs/seed-vortex-sparse new file mode 100644 index 000000000..a9d831dad Binary files /dev/null and b/fuzz/src/test/resources/io/github/dfa1/vortex/fuzz/VortexReaderFuzzTestInputs/parseMetadataBlobs/seed-vortex-sparse differ diff --git a/fuzz/src/test/resources/io/github/dfa1/vortex/fuzz/VortexReaderFuzzTestInputs/parseMetadataBlobs/seed-vortex-zigzag b/fuzz/src/test/resources/io/github/dfa1/vortex/fuzz/VortexReaderFuzzTestInputs/parseMetadataBlobs/seed-vortex-zigzag new file mode 100644 index 000000000..fdcb551f8 Binary files /dev/null and b/fuzz/src/test/resources/io/github/dfa1/vortex/fuzz/VortexReaderFuzzTestInputs/parseMetadataBlobs/seed-vortex-zigzag differ diff --git a/fuzz/src/test/resources/io/github/dfa1/vortex/fuzz/VortexReaderFuzzTestInputs/parseMetadataBlobs/seed-vortex-zstd b/fuzz/src/test/resources/io/github/dfa1/vortex/fuzz/VortexReaderFuzzTestInputs/parseMetadataBlobs/seed-vortex-zstd new file mode 100644 index 000000000..650a30375 Binary files /dev/null and b/fuzz/src/test/resources/io/github/dfa1/vortex/fuzz/VortexReaderFuzzTestInputs/parseMetadataBlobs/seed-vortex-zstd differ diff --git a/fuzz/src/test/resources/io/github/dfa1/vortex/fuzz/VortexReaderFuzzTestInputs/parseMetadataBlobs/seed-zstd-buffers b/fuzz/src/test/resources/io/github/dfa1/vortex/fuzz/VortexReaderFuzzTestInputs/parseMetadataBlobs/seed-zstd-buffers new file mode 100644 index 000000000..22bc9b10f Binary files /dev/null and b/fuzz/src/test/resources/io/github/dfa1/vortex/fuzz/VortexReaderFuzzTestInputs/parseMetadataBlobs/seed-zstd-buffers differ diff --git a/fuzz/src/test/resources/io/github/dfa1/vortex/fuzz/VortexReaderFuzzTestInputs/scanMalformedFile/crash-array-flatbuffer-vector-out-of-bounds b/fuzz/src/test/resources/io/github/dfa1/vortex/fuzz/VortexReaderFuzzTestInputs/scanMalformedFile/crash-array-flatbuffer-vector-out-of-bounds new file mode 100644 index 000000000..8ae602885 Binary files /dev/null and b/fuzz/src/test/resources/io/github/dfa1/vortex/fuzz/VortexReaderFuzzTestInputs/scanMalformedFile/crash-array-flatbuffer-vector-out-of-bounds differ diff --git a/fuzz/src/test/resources/io/github/dfa1/vortex/fuzz/VortexReaderFuzzTestInputs/scanMalformedFile/crash-bitpacked-struct-dtype b/fuzz/src/test/resources/io/github/dfa1/vortex/fuzz/VortexReaderFuzzTestInputs/scanMalformedFile/crash-bitpacked-struct-dtype new file mode 100644 index 000000000..96f508e8a Binary files /dev/null and b/fuzz/src/test/resources/io/github/dfa1/vortex/fuzz/VortexReaderFuzzTestInputs/scanMalformedFile/crash-bitpacked-struct-dtype differ diff --git a/fuzz/src/test/resources/io/github/dfa1/vortex/fuzz/VortexReaderFuzzTestInputs/scanMalformedFile/crash-dict-empty-codes b/fuzz/src/test/resources/io/github/dfa1/vortex/fuzz/VortexReaderFuzzTestInputs/scanMalformedFile/crash-dict-empty-codes new file mode 100644 index 000000000..7b1597c10 Binary files /dev/null and b/fuzz/src/test/resources/io/github/dfa1/vortex/fuzz/VortexReaderFuzzTestInputs/scanMalformedFile/crash-dict-empty-codes differ diff --git a/fuzz/src/test/resources/io/github/dfa1/vortex/fuzz/VortexReaderFuzzTestInputs/scanMalformedFile/crash-no-dtype b/fuzz/src/test/resources/io/github/dfa1/vortex/fuzz/VortexReaderFuzzTestInputs/scanMalformedFile/crash-no-dtype new file mode 100644 index 000000000..372d316a8 Binary files /dev/null and b/fuzz/src/test/resources/io/github/dfa1/vortex/fuzz/VortexReaderFuzzTestInputs/scanMalformedFile/crash-no-dtype differ diff --git a/fuzz/src/test/resources/io/github/dfa1/vortex/fuzz/VortexReaderFuzzTestInputs/scanMalformedFile/crash-struct-dtype-missing-names b/fuzz/src/test/resources/io/github/dfa1/vortex/fuzz/VortexReaderFuzzTestInputs/scanMalformedFile/crash-struct-dtype-missing-names new file mode 100644 index 000000000..6d66e8f17 Binary files /dev/null and b/fuzz/src/test/resources/io/github/dfa1/vortex/fuzz/VortexReaderFuzzTestInputs/scanMalformedFile/crash-struct-dtype-missing-names differ diff --git a/fuzz/src/test/resources/io/github/dfa1/vortex/fuzz/VortexReaderFuzzTestInputs/scanMalformedFile/crash-zstd-buffers-inner-id-control-char b/fuzz/src/test/resources/io/github/dfa1/vortex/fuzz/VortexReaderFuzzTestInputs/scanMalformedFile/crash-zstd-buffers-inner-id-control-char new file mode 100644 index 000000000..6807a3320 Binary files /dev/null and b/fuzz/src/test/resources/io/github/dfa1/vortex/fuzz/VortexReaderFuzzTestInputs/scanMalformedFile/crash-zstd-buffers-inner-id-control-char differ diff --git a/fuzz/src/test/resources/io/github/dfa1/vortex/fuzz/VortexReaderFuzzTestInputs/scanMalformedFile/seed-booleans b/fuzz/src/test/resources/io/github/dfa1/vortex/fuzz/VortexReaderFuzzTestInputs/scanMalformedFile/seed-booleans new file mode 100644 index 000000000..72060d1ea Binary files /dev/null and b/fuzz/src/test/resources/io/github/dfa1/vortex/fuzz/VortexReaderFuzzTestInputs/scanMalformedFile/seed-booleans differ diff --git a/fuzz/src/test/resources/io/github/dfa1/vortex/fuzz/VortexReaderFuzzTestInputs/scanMalformedFile/seed-chunked b/fuzz/src/test/resources/io/github/dfa1/vortex/fuzz/VortexReaderFuzzTestInputs/scanMalformedFile/seed-chunked new file mode 100644 index 000000000..91a8ed5eb Binary files /dev/null and b/fuzz/src/test/resources/io/github/dfa1/vortex/fuzz/VortexReaderFuzzTestInputs/scanMalformedFile/seed-chunked differ diff --git a/fuzz/src/test/resources/io/github/dfa1/vortex/fuzz/VortexReaderFuzzTestInputs/scanMalformedFile/seed-fastlanes-bitpacked b/fuzz/src/test/resources/io/github/dfa1/vortex/fuzz/VortexReaderFuzzTestInputs/scanMalformedFile/seed-fastlanes-bitpacked new file mode 100644 index 000000000..222af1f76 Binary files /dev/null and b/fuzz/src/test/resources/io/github/dfa1/vortex/fuzz/VortexReaderFuzzTestInputs/scanMalformedFile/seed-fastlanes-bitpacked differ diff --git a/fuzz/src/test/resources/io/github/dfa1/vortex/fuzz/VortexReaderFuzzTestInputs/scanMalformedFile/seed-fastlanes-delta b/fuzz/src/test/resources/io/github/dfa1/vortex/fuzz/VortexReaderFuzzTestInputs/scanMalformedFile/seed-fastlanes-delta new file mode 100644 index 000000000..f490a5831 Binary files /dev/null and b/fuzz/src/test/resources/io/github/dfa1/vortex/fuzz/VortexReaderFuzzTestInputs/scanMalformedFile/seed-fastlanes-delta differ diff --git a/fuzz/src/test/resources/io/github/dfa1/vortex/fuzz/VortexReaderFuzzTestInputs/scanMalformedFile/seed-fastlanes-for b/fuzz/src/test/resources/io/github/dfa1/vortex/fuzz/VortexReaderFuzzTestInputs/scanMalformedFile/seed-fastlanes-for new file mode 100644 index 000000000..2cd6c120c Binary files /dev/null and b/fuzz/src/test/resources/io/github/dfa1/vortex/fuzz/VortexReaderFuzzTestInputs/scanMalformedFile/seed-fastlanes-for differ diff --git a/fuzz/src/test/resources/io/github/dfa1/vortex/fuzz/VortexReaderFuzzTestInputs/scanMalformedFile/seed-fastlanes-rle b/fuzz/src/test/resources/io/github/dfa1/vortex/fuzz/VortexReaderFuzzTestInputs/scanMalformedFile/seed-fastlanes-rle new file mode 100644 index 000000000..f4d7aa3bd Binary files /dev/null and b/fuzz/src/test/resources/io/github/dfa1/vortex/fuzz/VortexReaderFuzzTestInputs/scanMalformedFile/seed-fastlanes-rle differ diff --git a/fuzz/src/test/resources/io/github/dfa1/vortex/fuzz/VortexReaderFuzzTestInputs/scanMalformedFile/seed-null b/fuzz/src/test/resources/io/github/dfa1/vortex/fuzz/VortexReaderFuzzTestInputs/scanMalformedFile/seed-null new file mode 100644 index 000000000..dfa85c4ae Binary files /dev/null and b/fuzz/src/test/resources/io/github/dfa1/vortex/fuzz/VortexReaderFuzzTestInputs/scanMalformedFile/seed-null differ diff --git a/fuzz/src/test/resources/io/github/dfa1/vortex/fuzz/VortexReaderFuzzTestInputs/scanMalformedFile/seed-primitives b/fuzz/src/test/resources/io/github/dfa1/vortex/fuzz/VortexReaderFuzzTestInputs/scanMalformedFile/seed-primitives new file mode 100644 index 000000000..145917170 Binary files /dev/null and b/fuzz/src/test/resources/io/github/dfa1/vortex/fuzz/VortexReaderFuzzTestInputs/scanMalformedFile/seed-primitives differ diff --git a/fuzz/src/test/resources/io/github/dfa1/vortex/fuzz/VortexReaderFuzzTestInputs/scanMalformedFile/seed-varbin b/fuzz/src/test/resources/io/github/dfa1/vortex/fuzz/VortexReaderFuzzTestInputs/scanMalformedFile/seed-varbin new file mode 100644 index 000000000..26e525510 Binary files /dev/null and b/fuzz/src/test/resources/io/github/dfa1/vortex/fuzz/VortexReaderFuzzTestInputs/scanMalformedFile/seed-varbin differ diff --git a/fuzz/src/test/resources/io/github/dfa1/vortex/fuzz/VortexReaderFuzzTestInputs/scanMalformedFile/seed-vortex-alp b/fuzz/src/test/resources/io/github/dfa1/vortex/fuzz/VortexReaderFuzzTestInputs/scanMalformedFile/seed-vortex-alp new file mode 100644 index 000000000..14fc67ffe Binary files /dev/null and b/fuzz/src/test/resources/io/github/dfa1/vortex/fuzz/VortexReaderFuzzTestInputs/scanMalformedFile/seed-vortex-alp differ diff --git a/fuzz/src/test/resources/io/github/dfa1/vortex/fuzz/VortexReaderFuzzTestInputs/scanMalformedFile/seed-vortex-alprd b/fuzz/src/test/resources/io/github/dfa1/vortex/fuzz/VortexReaderFuzzTestInputs/scanMalformedFile/seed-vortex-alprd new file mode 100644 index 000000000..44c6ea8c7 Binary files /dev/null and b/fuzz/src/test/resources/io/github/dfa1/vortex/fuzz/VortexReaderFuzzTestInputs/scanMalformedFile/seed-vortex-alprd differ diff --git a/fuzz/src/test/resources/io/github/dfa1/vortex/fuzz/VortexReaderFuzzTestInputs/scanMalformedFile/seed-vortex-dict b/fuzz/src/test/resources/io/github/dfa1/vortex/fuzz/VortexReaderFuzzTestInputs/scanMalformedFile/seed-vortex-dict new file mode 100644 index 000000000..616bf89fe Binary files /dev/null and b/fuzz/src/test/resources/io/github/dfa1/vortex/fuzz/VortexReaderFuzzTestInputs/scanMalformedFile/seed-vortex-dict differ diff --git a/fuzz/src/test/resources/io/github/dfa1/vortex/fuzz/VortexReaderFuzzTestInputs/scanMalformedFile/seed-vortex-fsst b/fuzz/src/test/resources/io/github/dfa1/vortex/fuzz/VortexReaderFuzzTestInputs/scanMalformedFile/seed-vortex-fsst new file mode 100644 index 000000000..f60ef77e1 Binary files /dev/null and b/fuzz/src/test/resources/io/github/dfa1/vortex/fuzz/VortexReaderFuzzTestInputs/scanMalformedFile/seed-vortex-fsst differ diff --git a/fuzz/src/test/resources/io/github/dfa1/vortex/fuzz/VortexReaderFuzzTestInputs/scanMalformedFile/seed-vortex-onpair b/fuzz/src/test/resources/io/github/dfa1/vortex/fuzz/VortexReaderFuzzTestInputs/scanMalformedFile/seed-vortex-onpair new file mode 100644 index 000000000..77a97eed3 Binary files /dev/null and b/fuzz/src/test/resources/io/github/dfa1/vortex/fuzz/VortexReaderFuzzTestInputs/scanMalformedFile/seed-vortex-onpair differ diff --git a/fuzz/src/test/resources/io/github/dfa1/vortex/fuzz/VortexReaderFuzzTestInputs/scanMalformedFile/seed-vortex-pco b/fuzz/src/test/resources/io/github/dfa1/vortex/fuzz/VortexReaderFuzzTestInputs/scanMalformedFile/seed-vortex-pco new file mode 100644 index 000000000..28684c302 Binary files /dev/null and b/fuzz/src/test/resources/io/github/dfa1/vortex/fuzz/VortexReaderFuzzTestInputs/scanMalformedFile/seed-vortex-pco differ diff --git a/fuzz/src/test/resources/io/github/dfa1/vortex/fuzz/VortexReaderFuzzTestInputs/scanMalformedFile/seed-vortex-runend b/fuzz/src/test/resources/io/github/dfa1/vortex/fuzz/VortexReaderFuzzTestInputs/scanMalformedFile/seed-vortex-runend new file mode 100644 index 000000000..02668c533 Binary files /dev/null and b/fuzz/src/test/resources/io/github/dfa1/vortex/fuzz/VortexReaderFuzzTestInputs/scanMalformedFile/seed-vortex-runend differ diff --git a/fuzz/src/test/resources/io/github/dfa1/vortex/fuzz/VortexReaderFuzzTestInputs/scanMalformedFile/seed-vortex-sparse b/fuzz/src/test/resources/io/github/dfa1/vortex/fuzz/VortexReaderFuzzTestInputs/scanMalformedFile/seed-vortex-sparse new file mode 100644 index 000000000..dc64babf1 Binary files /dev/null and b/fuzz/src/test/resources/io/github/dfa1/vortex/fuzz/VortexReaderFuzzTestInputs/scanMalformedFile/seed-vortex-sparse differ diff --git a/fuzz/src/test/resources/io/github/dfa1/vortex/fuzz/VortexReaderFuzzTestInputs/scanMalformedFile/seed-vortex-zigzag b/fuzz/src/test/resources/io/github/dfa1/vortex/fuzz/VortexReaderFuzzTestInputs/scanMalformedFile/seed-vortex-zigzag new file mode 100644 index 000000000..a062f51b9 Binary files /dev/null and b/fuzz/src/test/resources/io/github/dfa1/vortex/fuzz/VortexReaderFuzzTestInputs/scanMalformedFile/seed-vortex-zigzag differ diff --git a/fuzz/src/test/resources/io/github/dfa1/vortex/fuzz/VortexReaderFuzzTestInputs/scanMalformedFile/seed-vortex-zstd b/fuzz/src/test/resources/io/github/dfa1/vortex/fuzz/VortexReaderFuzzTestInputs/scanMalformedFile/seed-vortex-zstd new file mode 100644 index 000000000..6b9f196ff Binary files /dev/null and b/fuzz/src/test/resources/io/github/dfa1/vortex/fuzz/VortexReaderFuzzTestInputs/scanMalformedFile/seed-vortex-zstd differ diff --git a/fuzz/src/test/resources/io/github/dfa1/vortex/fuzz/VortexReaderFuzzTestInputs/scanMalformedFile/seed-zstd-buffers b/fuzz/src/test/resources/io/github/dfa1/vortex/fuzz/VortexReaderFuzzTestInputs/scanMalformedFile/seed-zstd-buffers new file mode 100644 index 000000000..02d49ff77 Binary files /dev/null and b/fuzz/src/test/resources/io/github/dfa1/vortex/fuzz/VortexReaderFuzzTestInputs/scanMalformedFile/seed-zstd-buffers differ diff --git a/inspector/src/main/java/io/github/dfa1/vortex/inspect/InspectorTree.java b/inspector/src/main/java/io/github/dfa1/vortex/inspect/InspectorTree.java index 4b967e2e6..df0074c4d 100644 --- a/inspector/src/main/java/io/github/dfa1/vortex/inspect/InspectorTree.java +++ b/inspector/src/main/java/io/github/dfa1/vortex/inspect/InspectorTree.java @@ -110,8 +110,7 @@ public static InspectorTree buildShallow(VortexHandle handle) { Footer footer = handle.footer(); Layout layout = handle.layout(); DType dtype = handle.dtype(); - List colNames = (dtype instanceof DType.Struct s) - ? s.fieldNames().stream().map(ColumnName::value).toList() : List.of(); + List colNames = rootChildLabels(dtype); Node root = shallowNode(layout, Optional.empty()); if (layout.isStruct()) { List named = new ArrayList<>(root.children().size()); @@ -183,8 +182,7 @@ public static InspectorTree build(VortexHandle handle, Progress progress) { int total = countPeekableSegments(layout, footer); int[] counter = {0}; - List colNames = (dtype instanceof DType.Struct s) - ? s.fieldNames().stream().map(ColumnName::value).toList() : List.of(); + List colNames = rootChildLabels(dtype); Set overallUsed = new LinkedHashSet<>(); Map segmentEncodings = new LinkedHashMap<>(); Node root = buildNode(layout, Optional.empty(), handle, footer.arraySpecs(), @@ -384,6 +382,20 @@ public interface Progress { void update(int current, int total); } + /// Labels for the root struct layout's children: the column names, after a `` child + /// when the struct is nullable (its layout leads with one, as Rust's `validate_children` requires). + private static List rootChildLabels(DType dtype) { + if (!(dtype instanceof DType.Struct s)) { + return List.of(); + } + List labels = new ArrayList<>(); + if (s.nullable()) { + labels.add(""); + } + s.fieldNames().forEach(name -> labels.add(name.value())); + return labels; + } + private static Peek peekFlatRoot(MemorySegment seg, List arraySpecs) { int segLen = (int) seg.byteSize(); int fbLen = seg.get(LE_INT, segLen - 4L); @@ -394,24 +406,16 @@ private static Peek peekFlatRoot(MemorySegment seg, List arraySpecs) return new Peek(null, ArrayStats.empty(), Set.of()); } Set nested = new LinkedHashSet<>(); - collectEncodings(root, arraySpecs, nested, 0, new int[]{0}); + collectEncodings(root, arraySpecs, nested, new int[]{0}); return new Peek(resolveEncodingId(arraySpecs, root.encoding()), ArrayStats.fromFbs(root.stats()), Set.copyOf(nested)); } - /// Hard cap on the `ArrayNode`-tree walk depth in [#collectEncodings(FbsArrayNode, List, Set, int, int[])], - /// mirroring `SerializedArrayDecoder.MAX_ARRAY_TREE_DEPTH` in the reader module: a crafted segment with - /// deeply nested children could otherwise drive unbounded recursion into a `StackOverflowError`, - /// an `Error` that would bypass the [VortexException] contract for malformed untrusted input. - static final int MAX_ENCODING_TREE_DEPTH = 64; - /// Hard cap on the total number of `ArrayNode`s visited by one - /// [#collectEncodings(FbsArrayNode, List, Set, int, int[])] walk. The depth cap alone is not enough: nothing - /// stops two sibling `children` vector slots from resolving to the exact same absolute child position (the - /// underlying FlatBuffers `uoffset` mechanism is not validated for aliasing), so a crafted segment with a - /// handful of levels, each fanning out to two children that both alias the next level's single node, drives - /// the walk to an exponential (up to 2^[#MAX_ENCODING_TREE_DEPTH]) number of visits while depth itself - /// never exceeds the limit. + /// [#collectEncodings(FbsArrayNode, List, Set, int[])] walk. Depth is already bounded by the FlatBuffer + /// verifier `FbsArray.getRootAsFbsArray` runs (64 nested tables, as Rust), and so is the total, but only at + /// a million visits: two sibling `children` slots may alias the same child, so a handful of levels fanning + /// out to two aliases each reaches that many. The inspector peeks every segment, so it stops far sooner. static final int MAX_ENCODING_TREE_NODES = 4096; /// Resolves a wire-supplied encoding index against the footer's array spec table. @@ -435,20 +439,16 @@ private static String resolveEncodingId(List arraySpecs, int encIdx) /// @param node array node to visit /// @param arraySpecs footer's array spec table, indexed by [FbsArrayNode#encoding()] /// @param into accumulator for resolved encoding ids - /// @param depth current recursion depth, starting at `0` for the root /// @param visited single-element counter of nodes visited so far across the whole walk private static void collectEncodings(FbsArrayNode node, List arraySpecs, Set into, - int depth, int[] visited) { - if (depth > MAX_ENCODING_TREE_DEPTH) { - throw new VortexException("array tree depth exceeds limit (" + MAX_ENCODING_TREE_DEPTH + ")"); - } + int[] visited) { if (++visited[0] > MAX_ENCODING_TREE_NODES) { throw new VortexException("array tree node count exceeds limit (" + MAX_ENCODING_TREE_NODES + ")"); } into.add(resolveEncodingId(arraySpecs, node.encoding())); int childCount = node.childrenLength(); for (int i = 0; i < childCount; i++) { - collectEncodings(node.children(i), arraySpecs, into, depth + 1, visited); + collectEncodings(node.children(i), arraySpecs, into, visited); } } diff --git a/inspector/src/test/java/io/github/dfa1/vortex/inspect/InspectorTreeTest.java b/inspector/src/test/java/io/github/dfa1/vortex/inspect/InspectorTreeTest.java index 0399ad11a..2eeaca970 100644 --- a/inspector/src/test/java/io/github/dfa1/vortex/inspect/InspectorTreeTest.java +++ b/inspector/src/test/java/io/github/dfa1/vortex/inspect/InspectorTreeTest.java @@ -299,14 +299,14 @@ void build_flatSegmentWithNestedEncoding_reportsRootAndNestedEncodings() { @Test void build_flatSegmentEncodingTreeAtDepthLimit_clearsGuard() { - // Given — deepest node at exactly MAX_ENCODING_TREE_DEPTH clears the guard - // (`depth > limit` is false there); brackets the boundary with the next test. + // Given — 64 nested tables (the Array table and 63 nodes), exactly the FlatBuffer verifier's + // depth limit, Rust's own; brackets the boundary with the next test. try (Arena arena = Arena.ofConfined()) { Layout root = new Layout(LayoutId.parse("vortex.flat"), 10, null, List.of(), List.of(0)); DType dtype = DType.I32; SegmentSpec spec = new SegmentSpec(0, 100, (byte) 0, CompressionScheme.NONE); givenHandle(dtype, root, List.of("vortex.flat"), List.of(spec)); - byte[] fb = deeplyNestedArrayFlatBuffer(InspectorTree.MAX_ENCODING_TREE_DEPTH); + byte[] fb = deeplyNestedArrayFlatBuffer(62); given(handle.rawSegment(spec)).willReturn(wrapAsSegment(fb, arena)); // When @@ -319,7 +319,7 @@ void build_flatSegmentEncodingTreeAtDepthLimit_clearsGuard() { @Test void build_flatSegmentEncodingTreeOneOverDepthLimit_throwsVortexException() { - // Given — one level deeper than MAX_ENCODING_TREE_DEPTH: a crafted file with a very + // Given — one table deeper than the verifier allows: a crafted file with a very // deep ArrayNode nesting must surface as VortexException, never StackOverflowError // (the untrusted-input contract every reader/inspector parse path must uphold). try (Arena arena = Arena.ofConfined()) { @@ -327,13 +327,13 @@ void build_flatSegmentEncodingTreeOneOverDepthLimit_throwsVortexException() { DType dtype = DType.I32; SegmentSpec spec = new SegmentSpec(0, 100, (byte) 0, CompressionScheme.NONE); givenHandle(dtype, root, List.of("vortex.flat"), List.of(spec)); - byte[] fb = deeplyNestedArrayFlatBuffer(InspectorTree.MAX_ENCODING_TREE_DEPTH + 1); + byte[] fb = deeplyNestedArrayFlatBuffer(63); given(handle.rawSegment(spec)).willReturn(wrapAsSegment(fb, arena)); // When / Then assertThatThrownBy(() -> InspectorTree.build(handle)) .isInstanceOf(VortexException.class) - .hasMessageContaining("depth"); + .hasMessageContaining("nest deeper than 64"); } } @@ -364,7 +364,7 @@ void build_flatSegmentEncodingTreeOverNodeCountLimit_throwsVortexException() { // children, so depth never exceeds 1 but total node count does. Without a separate // node-count cap, two aliased children per level (both children vector slots resolving // to the same table position) could drive collectEncodings to an exponential number of - // visits while depth stays within MAX_ENCODING_TREE_DEPTH; this is the simpler, + // visits while depth stays within the verifier's limit; this is the simpler, // non-aliased way to exercise the same cap. try (Arena arena = Arena.ofConfined()) { Layout root = new Layout(LayoutId.parse("vortex.flat"), 10, null, List.of(), List.of(0)); diff --git a/integration/pom.xml b/integration/pom.xml index 6c915e65e..5badc2461 100644 --- a/integration/pom.xml +++ b/integration/pom.xml @@ -107,6 +107,11 @@ junit-jupiter test + + com.code-intelligence + jazzer-junit + test + org.assertj assertj-core @@ -242,8 +247,12 @@ class selection, so running a raincloud test always requires clearing the property as well. --> ${vortex.it.excludedGroups} + @{argLine} + -javaagent:${net.bytebuddy:byte-buddy-agent:jar} --add-opens java.base/java.nio=ALL-UNNAMED --enable-native-access=ALL-UNNAMED --sun-misc-unsafe-memory-access=allow diff --git a/integration/src/test/java/io/github/dfa1/vortex/integration/MalformedFileInteropIntegrationTest.java b/integration/src/test/java/io/github/dfa1/vortex/integration/MalformedFileInteropIntegrationTest.java new file mode 100644 index 000000000..f301702c3 --- /dev/null +++ b/integration/src/test/java/io/github/dfa1/vortex/integration/MalformedFileInteropIntegrationTest.java @@ -0,0 +1,94 @@ +package io.github.dfa1.vortex.integration; + +import static org.assertj.core.api.Assertions.assertThatThrownBy; + +import dev.vortex.api.DataSource; +import dev.vortex.api.Partition; +import dev.vortex.api.Scan; +import dev.vortex.api.ScanOptions; +import dev.vortex.api.Session; +import dev.vortex.arrow.ArrowAllocation; +import dev.vortex.jni.NativeLoader; +import io.github.dfa1.vortex.core.error.VortexException; +import io.github.dfa1.vortex.reader.Chunk; +import io.github.dfa1.vortex.reader.ScanIterator; +import io.github.dfa1.vortex.reader.VortexReader; +import org.apache.arrow.vector.FieldVector; +import org.apache.arrow.vector.VectorSchemaRoot; +import org.apache.arrow.vector.ipc.ArrowReader; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.ValueSource; + +import java.net.URISyntaxException; +import java.nio.file.Path; +import java.util.Objects; + +/// Files the Jazzer `scanMalformedFile` target produced (#493) that once escaped vortex-java as a +/// raw JDK exception. Both readers must reject them: Rust is the oracle for what is malformed, and +/// vortex-java must throw only [VortexException] doing so (the reader's security contract). +class MalformedFileInteropIntegrationTest { + + static { + NativeLoader.loadJni(); + } + + @ParameterizedTest + @ValueSource(strings = { + // A child-node vector offset in a segment's array FlatBuffer points past its end: + // escaped as IndexOutOfBoundsException from the FlatBuffer runtime. + "array-flatbuffer-vector-out-of-bounds", + // A Struct column whose segment is fastlanes.bitpacked: escaped as ClassCastException. + "bitpacked-struct-dtype", + // A struct dtype with no names vector, which Rust refuses at open: vortex-java read it + // as a 0-field struct and failed the scan with ArrayIndexOutOfBoundsException. + "struct-dtype-missing-names", + // A vortex.zstd_buffers whose wrapped encoding id holds a control character: escaped as + // IllegalArgumentException from EncodingId.parse. + "zstd-buffers-inner-id-control-char", + // A file with no dtype blob, which Rust opens only when the caller supplies the dtype: + // vortex-java opened it with a null dtype and the scan threw NullPointerException. + "no-dtype", + // A dict column whose codes child is empty for a non-empty chunk: the broadcast's + // `i % capacity` divided by zero (ArithmeticException). + "dict-empty-codes", + // A primitive child whose buffer is empty for a non-empty array: the Materialized* + // arrays broadcast with `i % elementCount` and divided by zero on the first read. + "materialized-empty-buffer"}) + void bothReadersReject(String name) throws URISyntaxException { + // Given + Path file = Path.of(Objects.requireNonNull( + getClass().getResource("/fixtures/malformed/" + name + ".vortex")).toURI()); + + // When / Then + assertThatThrownBy(() -> scanJni(file)).as("vortex-jni").isInstanceOf(Exception.class); + assertThatThrownBy(() -> scanJava(file)).as("vortex-java").isInstanceOf(VortexException.class); + } + + private static void scanJava(Path file) throws Exception { + try (VortexReader reader = VortexReader.open(file); + ScanIterator scan = reader.scan(io.github.dfa1.vortex.reader.ScanOptions.all())) { + while (scan.hasNext()) { + try (Chunk _ = scan.next()) { + // decoding the chunk is enough: every file here fails before a value is read + } + } + } + } + + private static void scanJni(Path file) throws Exception { + Scan scan = DataSource.open(Session.create(), file.toUri().toString()).scan(ScanOptions.builder().build()); + while (scan.hasNext()) { + Partition partition = scan.next(); + try (ArrowReader reader = partition.scanArrow(ArrowAllocation.rootAllocator())) { + while (reader.loadNextBatch()) { + VectorSchemaRoot root = reader.getVectorSchemaRoot(); + for (FieldVector vector : root.getFieldVectors()) { + for (int i = 0; i < root.getRowCount(); i++) { + vector.getObject(i); + } + } + } + } + } + } +} diff --git a/integration/src/test/java/io/github/dfa1/vortex/integration/WriteReadFuzzIntegrationTest.java b/integration/src/test/java/io/github/dfa1/vortex/integration/WriteReadFuzzIntegrationTest.java new file mode 100644 index 000000000..8cb2a59ce --- /dev/null +++ b/integration/src/test/java/io/github/dfa1/vortex/integration/WriteReadFuzzIntegrationTest.java @@ -0,0 +1,378 @@ +package io.github.dfa1.vortex.integration; + +import static org.junit.jupiter.api.Assertions.assertEquals; + +import java.io.IOException; +import java.nio.channels.FileChannel; +import java.nio.file.Path; +import java.nio.file.StandardOpenOption; +import java.util.ArrayList; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import java.util.Random; + +import com.code_intelligence.jazzer.api.FuzzedDataProvider; +import com.code_intelligence.jazzer.junit.FuzzTest; +import dev.vortex.api.DataSource; +import dev.vortex.api.Partition; +import dev.vortex.api.Scan; +import dev.vortex.api.ScanOptions; +import dev.vortex.api.Session; +import dev.vortex.arrow.ArrowAllocation; +import dev.vortex.jni.NativeLoader; +import io.github.dfa1.vortex.core.compute.PrimitiveArrays; +import io.github.dfa1.vortex.core.model.ColumnName; +import io.github.dfa1.vortex.core.model.DType; +import io.github.dfa1.vortex.core.model.EncodingId; +import io.github.dfa1.vortex.reader.Chunk; +import io.github.dfa1.vortex.reader.VortexReader; +import io.github.dfa1.vortex.reader.array.Array; +import io.github.dfa1.vortex.reader.array.BoolArray; +import io.github.dfa1.vortex.reader.array.ByteArray; +import io.github.dfa1.vortex.reader.array.DoubleArray; +import io.github.dfa1.vortex.reader.array.FloatArray; +import io.github.dfa1.vortex.reader.array.IntArray; +import io.github.dfa1.vortex.reader.array.LongArray; +import io.github.dfa1.vortex.reader.array.MaskedArray; +import io.github.dfa1.vortex.reader.array.NullArray; +import io.github.dfa1.vortex.reader.array.ShortArray; +import io.github.dfa1.vortex.reader.array.VarBinArray; +import io.github.dfa1.vortex.writer.VortexWriter; +import io.github.dfa1.vortex.writer.WriteOptions; +import io.github.dfa1.vortex.writer.encode.NullableData; +import org.apache.arrow.vector.FieldVector; +import org.apache.arrow.vector.VectorSchemaRoot; +import org.apache.arrow.vector.ipc.ArrowReader; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Tag; +import org.junit.jupiter.api.io.TempDir; + +/// Write-side correctness fuzzing (#493): the fuzzer builds a small table, vortex-java writes it, +/// and a reader must give back exactly the values written. +/// +/// The fuzzer does not choose every value: per column it picks a dtype, nullability and a *shape* +/// (random, constant, small range, runs, sorted, sparse) plus a seed, and the values follow. The +/// cascading compressor chooses its schemes from data statistics, so shapes are what steer it into +/// its edges (dict, RLE, FOR, delta, ALP, FSST, sparse patches, all-null, one-row chunks) while an +/// input stays a few bytes. +/// +/// - [#javaRoundTrip(FuzzedDataProvider)]: vortex-java reads it back. +/// - [#jniReadsJavaWrite(FuzzedDataProvider)]: vortex-jni reads it back, so a file vortex-java +/// writes that Rust reads differently, or refuses, is a finding. A Rust panic aborts the JVM, +/// which ends the fuzzing run but still names the input on stderr. +/// +/// Values compare in one canonical form: integers as `long` (unsigned ones zero-extended), floats +/// as their bit patterns (so `-0.0` and NaN are exact), booleans, strings and `null`. +@Tag("fuzz") +class WriteReadFuzzIntegrationTest { + + static { + NativeLoader.loadJni(); + } + + private static final List TYPES = List.of( + DType.I8, DType.I16, DType.I32, DType.I64, DType.U8, DType.U16, DType.U32, DType.U64, + DType.F32, DType.F64, DType.BOOL, DType.UTF8); + + @TempDir + Path tempDir; + + private Path file; + + @BeforeEach + void setUp() { + file = tempDir.resolve("fuzz.vortex"); + } + + @FuzzTest + void javaRoundTrip(FuzzedDataProvider data) throws IOException { + // Given + Table table = Table.generate(data); + table.write(file); + + // When + Map> result = read(() -> readJava(file, table), table); + + // Then + assertEquals(table.expected(), result, "vortex-java reading " + table); + } + + @FuzzTest + void jniReadsJavaWrite(FuzzedDataProvider data) throws IOException { + // Given + Table table = Table.generate(data); + table.write(file); + + // When + Map> result = read(() -> readJni(file, table), table); + + // Then + assertEquals(table.expected(), result, "vortex-jni reading " + table); + } + + @FunctionalInterface + private interface Read { + Map> values() throws IOException; + } + + /// Names the table in any read failure, so a finding says what was written. + private static Map> read(Read read, Table table) { + try { + return read.values(); + } catch (IOException | RuntimeException e) { + throw new AssertionError("reading " + table, e); + } + } + + /// One generated column: its schema entry, the array the writer takes, and the canonical values. + private record Column(ColumnName name, DType dtype, String shape, Object values, List expected) { + } + + private record Table(List columns, int rows, List chunkRows) { + + static Table generate(FuzzedDataProvider data) { + int rows = data.consumeInt(0, 3_000); + List chunkRows = new ArrayList<>(); + int left = rows; + for (int chunks = data.consumeInt(1, 3); chunks > 1 && left > 0; chunks--) { + int size = data.consumeInt(1, left); + chunkRows.add(size); + left -= size; + } + chunkRows.add(left); + List columns = new ArrayList<>(); + for (int c = data.consumeInt(1, 4); c > 0; c--) { + columns.add(column(data, ColumnName.of("c" + columns.size()), rows)); + } + return new Table(columns, rows, chunkRows); + } + + private static Column column(FuzzedDataProvider data, ColumnName name, int rows) { + DType base = data.pickValue(TYPES); + boolean nullable = data.consumeBoolean(); + Shape shape = data.pickValue(Shape.values()); + int nullEvery = nullable ? data.consumeInt(0, 4) : 0; // 0 = no nulls, 1 = all null + Random random = new Random(data.consumeLong()); + long parameter = data.consumeLong(); + List expected = new ArrayList<>(rows); + for (int i = 0; i < rows; i++) { + boolean isNull = nullEvery == 1 || nullEvery > 1 && random.nextInt(nullEvery) == 0; + expected.add(isNull ? null : canonical(base, shape.raw(random, i, parameter))); + } + DType dtype = nullable ? base.asNullable() : base; + return new Column(name, dtype, shape.name(), writerValues(dtype, expected), expected); + } + + /// The canonical value the raw shape draw stands for in `dtype`. + private static Object canonical(DType dtype, long raw) { + return switch (dtype) { + case DType.Bool _ -> (raw & 1) != 0; + case DType.Utf8 _ -> raw % 97 == 0 ? "" : "row-" + Long.toUnsignedString(raw, 36); + case DType.Primitive p -> switch (p.ptype()) { + case I8 -> (long) (byte) raw; + case I16 -> (long) (short) raw; + case I32 -> (long) (int) raw; + case U8 -> raw & 0xFFL; + case U16 -> raw & 0xFFFFL; + case U32 -> raw & 0xFFFF_FFFFL; + case F32 -> Float.floatToIntBits(floatOf(raw)); + case F64 -> Double.doubleToLongBits(doubleOf(raw)); + default -> raw; + }; + default -> throw new IllegalStateException(dtype.toString()); + }; + } + + /// Small raws become decimals (ALP's case), big ones arbitrary bit patterns (NaN, inf, -0.0). + private static double doubleOf(long raw) { + return Math.abs(raw) < 1_000_000 ? raw / 100.0 : Double.longBitsToDouble(raw); + } + + private static float floatOf(long raw) { + return Math.abs(raw) < 1_000_000 ? raw / 100.0f : Float.intBitsToFloat((int) raw); + } + + /// The array [VortexWriter#writeChunk(Map)] takes: the column type's primitive array (or + /// `String[]`), wrapped with its validity in [NullableData] when the column is nullable. + private static Object writerValues(DType dtype, List canonical) { + int n = canonical.size(); + if (dtype instanceof DType.Utf8) { + return canonical.toArray(new String[0]); + } + boolean[] valid = new boolean[n]; + long[] bits = new long[n]; + boolean[] bools = new boolean[n]; + for (int i = 0; i < n; i++) { + Object v = canonical.get(i); + valid[i] = v != null; + if (v instanceof Boolean b) { + bools[i] = b; + } else if (v != null) { + bits[i] = ((Number) v).longValue(); + } + } + Object values = dtype instanceof DType.Primitive p + ? PrimitiveArrays.fromBitsArray(bits, p.ptype(), EncodingId.VORTEX_PRIMITIVE) + : bools; + return dtype.nullable() ? new NullableData(values, valid) : values; + } + + void write(Path path) throws IOException { + DType.Struct schema = new DType.Struct(columns.stream().map(Column::name).toList(), + columns.stream().map(Column::dtype).toList(), false); + try (var channel = FileChannel.open(path, StandardOpenOption.CREATE, StandardOpenOption.WRITE, + StandardOpenOption.TRUNCATE_EXISTING); + var writer = VortexWriter.create(channel, schema, WriteOptions.defaults())) { + int from = 0; + for (int size : chunkRows) { + Map chunk = new LinkedHashMap<>(); + for (Column column : columns) { + chunk.put(column.name(), writerValues(column.dtype(), column.expected().subList(from, from + size))); + } + writer.writeChunk(chunk); + from += size; + } + } + } + + Map> expected() { + Map> expected = new LinkedHashMap<>(); + columns.forEach(c -> expected.put(c.name(), c.expected())); + return expected; + } + + @Override + public String toString() { + return rows + " rows in chunks " + chunkRows + ", columns " + columns.stream() + .map(c -> c.name() + ":" + c.dtype() + "/" + c.shape()).toList(); + } + } + + /// How a column's raw values follow from its seed and parameter. + private enum Shape { + RANDOM { + @Override + long raw(Random random, int row, long parameter) { + return random.nextLong(); + } + }, + CONSTANT { + @Override + long raw(Random random, int row, long parameter) { + return parameter; + } + }, + SMALL_RANGE { + @Override + long raw(Random random, int row, long parameter) { + return parameter + random.nextInt(1 + (int) Math.floorMod(parameter, 64L)); + } + }, + RUNS { + @Override + long raw(Random random, int row, long parameter) { + return row / (1 + Math.floorMod(parameter, 500L)); + } + }, + SORTED { + @Override + long raw(Random random, int row, long parameter) { + return parameter + (long) row * (1 + Math.floorMod(parameter, 7L)); + } + }, + SPARSE { + @Override + long raw(Random random, int row, long parameter) { + return random.nextInt(100) == 0 ? random.nextLong() : parameter; + } + }; + + abstract long raw(Random random, int row, long parameter); + } + + private static Map> readJava(Path path, Table table) throws IOException { + Map> read = new LinkedHashMap<>(); + table.columns().forEach(c -> read.put(c.name(), new ArrayList<>())); + try (VortexReader reader = VortexReader.open(path)) { + reader.scan(io.github.dfa1.vortex.reader.ScanOptions.all()).forEachRemaining(chunk -> { + for (Column column : table.columns()) { + appendJava(chunk, column, read.get(column.name())); + } + }); + } + return read; + } + + private static void appendJava(Chunk chunk, Column column, List out) { + MaskedArray.Unwrapped unwrapped = MaskedArray.unwrap(chunk.column(column.name())); + Array values = unwrapped.inner(); + BoolArray validity = unwrapped.validity(); + for (long i = 0; i < values.length(); i++) { + boolean isNull = values instanceof NullArray || validity != null && !validity.getBoolean(i); + out.add(isNull ? null : javaValue(values, i, column.dtype())); + } + } + + private static Object javaValue(Array values, long i, DType dtype) { + Object raw = switch (values) { + case LongArray a -> a.getLong(i); + case IntArray a -> (long) a.getInt(i); + case ShortArray a -> (long) a.getShort(i); + case ByteArray a -> (long) a.getByte(i); + case DoubleArray a -> Double.doubleToLongBits(a.getDouble(i)); + case FloatArray a -> Float.floatToIntBits(a.getFloat(i)); + case BoolArray a -> a.getBoolean(i); + case VarBinArray a -> a.getString(i); + default -> throw new IllegalStateException("unexpected " + values.getClass().getSimpleName() + " for " + dtype); + }; + return raw instanceof Long l ? zeroExtend(dtype, l) : raw; + } + + private static Map> readJni(Path path, Table table) throws IOException { + Map> read = new LinkedHashMap<>(); + table.columns().forEach(c -> read.put(c.name(), new ArrayList<>())); + Scan scan = DataSource.open(Session.create(), path.toAbsolutePath().toUri().toString()) + .scan(ScanOptions.builder().build()); + while (scan.hasNext()) { + Partition partition = scan.next(); + try (ArrowReader reader = partition.scanArrow(ArrowAllocation.rootAllocator())) { + while (reader.loadNextBatch()) { + VectorSchemaRoot root = reader.getVectorSchemaRoot(); + for (Column column : table.columns()) { + FieldVector vector = root.getVector(column.name().value()); + for (int i = 0; i < root.getRowCount(); i++) { + read.get(column.name()).add(vector.isNull(i) ? null : jniValue(vector.getObject(i), column.dtype())); + } + } + } + } + } + return read; + } + + private static Object jniValue(Object value, DType dtype) { + return switch (value) { + case Double d -> Double.doubleToLongBits(d); + case Float f -> Float.floatToIntBits(f); + case Boolean b -> b; + case Character c -> (long) c; + case Number n -> zeroExtend(dtype, n.longValue()); + case org.apache.arrow.vector.util.Text text -> text.toString(); + default -> value.toString(); + }; + } + + /// Java's signed readings of an unsigned column back to its canonical, zero-extended form. + private static long zeroExtend(DType dtype, long value) { + if (!(dtype instanceof DType.Primitive p)) { + return value; + } + return switch (p.ptype()) { + case U8 -> value & 0xFFL; + case U16 -> value & 0xFFFFL; + case U32 -> value & 0xFFFF_FFFFL; + default -> value; + }; + } +} diff --git a/integration/src/test/resources/fixtures/malformed/array-flatbuffer-vector-out-of-bounds.vortex b/integration/src/test/resources/fixtures/malformed/array-flatbuffer-vector-out-of-bounds.vortex new file mode 100644 index 000000000..8ae602885 Binary files /dev/null and b/integration/src/test/resources/fixtures/malformed/array-flatbuffer-vector-out-of-bounds.vortex differ diff --git a/integration/src/test/resources/fixtures/malformed/bitpacked-struct-dtype.vortex b/integration/src/test/resources/fixtures/malformed/bitpacked-struct-dtype.vortex new file mode 100644 index 000000000..96f508e8a Binary files /dev/null and b/integration/src/test/resources/fixtures/malformed/bitpacked-struct-dtype.vortex differ diff --git a/integration/src/test/resources/fixtures/malformed/dict-empty-codes.vortex b/integration/src/test/resources/fixtures/malformed/dict-empty-codes.vortex new file mode 100644 index 000000000..7b1597c10 Binary files /dev/null and b/integration/src/test/resources/fixtures/malformed/dict-empty-codes.vortex differ diff --git a/integration/src/test/resources/fixtures/malformed/materialized-empty-buffer.vortex b/integration/src/test/resources/fixtures/malformed/materialized-empty-buffer.vortex new file mode 100644 index 000000000..a09ff0bcd Binary files /dev/null and b/integration/src/test/resources/fixtures/malformed/materialized-empty-buffer.vortex differ diff --git a/integration/src/test/resources/fixtures/malformed/no-dtype.vortex b/integration/src/test/resources/fixtures/malformed/no-dtype.vortex new file mode 100644 index 000000000..372d316a8 Binary files /dev/null and b/integration/src/test/resources/fixtures/malformed/no-dtype.vortex differ diff --git a/integration/src/test/resources/fixtures/malformed/struct-dtype-missing-names.vortex b/integration/src/test/resources/fixtures/malformed/struct-dtype-missing-names.vortex new file mode 100644 index 000000000..6d66e8f17 Binary files /dev/null and b/integration/src/test/resources/fixtures/malformed/struct-dtype-missing-names.vortex differ diff --git a/integration/src/test/resources/fixtures/malformed/zstd-buffers-inner-id-control-char.vortex b/integration/src/test/resources/fixtures/malformed/zstd-buffers-inner-id-control-char.vortex new file mode 100644 index 000000000..6807a3320 Binary files /dev/null and b/integration/src/test/resources/fixtures/malformed/zstd-buffers-inner-id-control-char.vortex differ diff --git a/integration/src/test/resources/io/github/dfa1/vortex/integration/WriteReadFuzzIntegrationTestInputs/jniReadsJavaWrite/crash-constant-u8-sign-extended b/integration/src/test/resources/io/github/dfa1/vortex/integration/WriteReadFuzzIntegrationTestInputs/jniReadsJavaWrite/crash-constant-u8-sign-extended new file mode 100644 index 000000000..ee0d16f0b --- /dev/null +++ b/integration/src/test/resources/io/github/dfa1/vortex/integration/WriteReadFuzzIntegrationTestInputs/jniReadsJavaWrite/crash-constant-u8-sign-extended @@ -0,0 +1 @@ +ûûÿÿ«)÷ \ No newline at end of file diff --git a/integration/src/test/resources/io/github/dfa1/vortex/integration/WriteReadFuzzIntegrationTestInputs/jniReadsJavaWrite/crash-sequence-u16-sign-extended b/integration/src/test/resources/io/github/dfa1/vortex/integration/WriteReadFuzzIntegrationTestInputs/jniReadsJavaWrite/crash-sequence-u16-sign-extended new file mode 100644 index 000000000..da19a258f Binary files /dev/null and b/integration/src/test/resources/io/github/dfa1/vortex/integration/WriteReadFuzzIntegrationTestInputs/jniReadsJavaWrite/crash-sequence-u16-sign-extended differ diff --git a/integration/src/test/resources/junit-platform.properties b/integration/src/test/resources/junit-platform.properties index 188453ce9..d24a88889 100644 --- a/integration/src/test/resources/junit-platform.properties +++ b/integration/src/test/resources/junit-platform.properties @@ -5,3 +5,7 @@ junit.jupiter.execution.parallel.mode.default=same_thread junit.jupiter.execution.parallel.mode.classes.default=same_thread junit.jupiter.execution.parallel.config.strategy=dynamic junit.jupiter.execution.parallel.config.dynamic.factor=1 + +# Jazzer (WriteReadFuzzIntegrationTest): instrument only this project's classes for coverage, +# not Arrow, vortex-jni or the test libraries on the classpath. +jazzer.instrument=io.github.dfa1.vortex.** diff --git a/reader/src/main/java/io/github/dfa1/vortex/reader/PostscriptParser.java b/reader/src/main/java/io/github/dfa1/vortex/reader/PostscriptParser.java index 7e48af281..bfb91ce3a 100644 --- a/reader/src/main/java/io/github/dfa1/vortex/reader/PostscriptParser.java +++ b/reader/src/main/java/io/github/dfa1/vortex/reader/PostscriptParser.java @@ -29,26 +29,12 @@ final class PostscriptParser { - /// Hard cap on layout-tree recursion depth. Real-world layouts are typically four levels - /// (Struct → Zoned → Chunked → Flat); 64 is well past any expected schema and prevents - /// adversarial inputs — deeply nested trees or self-referential FlatBuffer cycles — from - /// blowing the JVM stack during [#convertLayout(io.github.dfa1.vortex.core.fbs.FbsLayout, List, int)]. - static final int MAX_LAYOUT_DEPTH = 64; - /// Hard cap on per-layout metadata size. The FlatBuffer runtime returns an unbounded slice /// from `metadataAsSegment()`; a crafted file can claim a multi-gigabyte metadata /// blob and force later allocators into pathological behavior. 4 MiB is well above any /// real encoding's metadata footprint (the largest is FSST's symbol table at ~32 KiB). static final MemorySize MAX_LAYOUT_METADATA_BYTES = MemorySize.ofMiB(4); - /// Hard cap on DType-tree recursion depth. A `DType` nests through Struct fields, List/ - /// FixedSizeList element types, and Extension storage types; like the layout tree, a crafted - /// or self-referential FlatBuffer can drive [#convertDType(io.github.dfa1.vortex.core.fbs.FbsDType, int)] - /// into unbounded recursion and a [StackOverflowError] — which, being an `Error`, would escape - /// the [VortexException] sanitization and leak the reader's memory-mapped Arena. 64 is well past - /// any real schema's nesting. - static final int MAX_DTYPE_DEPTH = 64; - private PostscriptParser() { } @@ -121,12 +107,15 @@ static ParsedFile parseBlobs(MemorySegment footerBuf, MemorySegment layoutBuf, M var fbsLayout = io.github.dfa1.vortex.core.fbs.FbsLayout.getRootAsFbsLayout(layoutBuf); Footer footer = convertFooter(fbsFooter); - Layout layout = convertLayout(fbsLayout, footer.layoutSpecs(), 0); + Layout layout = convertLayout(fbsLayout, footer.layoutSpecs()); - DType dtype = null; - if (dtypeBuf != null && dtypeBuf.byteSize() > 0) { - dtype = convertDType(io.github.dfa1.vortex.core.fbs.FbsDType.getRootAsFbsDType(dtypeBuf), 0); + // Rust opens a file without an embedded dtype only when the caller supplies one + // (VortexOpenOptions::with_dtype); Java has no such option, so such a file is unreadable. + if (dtypeBuf == null || dtypeBuf.byteSize() == 0) { + throw new VortexException("Vortex file doesn't embed a DType"); } + DType dtype = convertDType(io.github.dfa1.vortex.core.fbs.FbsDType.getRootAsFbsDType(dtypeBuf)); + checkStructLayouts(layout, dtype); return new ParsedFile(footer, dtype, layout); } catch (VortexException e) { @@ -136,6 +125,26 @@ static ParsedFile parseBlobs(MemorySegment footerBuf, MemorySegment layoutBuf, M } } + /// Rejects at open a struct layout whose children do not match its struct dtype, as Rust does + /// while deserializing the layout tree (`Layout::::validate_children`): one child per + /// field, plus a leading validity child when the struct is nullable. Left to the scan, a + /// mismatch indexed the field list by layout child and escaped as a raw JDK exception. + /// Recurses only through directly nested struct layouts, whose dtype is the field's own. + private static void checkStructLayouts(Layout layout, DType dtype) { + if (!layout.isStruct() || !(dtype instanceof DType.Struct struct)) { + return; + } + int validity = struct.nullable() ? 1 : 0; + int expected = struct.fieldTypes().size() + validity; + if (layout.children().size() != expected) { + throw new VortexException("vortex.struct layout has " + layout.children().size() + + " children but its struct dtype needs " + expected); + } + for (int i = 0; i < struct.fieldTypes().size(); i++) { + checkStructLayouts(layout.children().get(validity + i), struct.fieldTypes().get(i)); + } + } + private static MemorySegment slice(MemorySegment seg, long offset, long length) { return IoBounds.slice(seg, offset, length); } @@ -215,11 +224,7 @@ static Footer convertFooter(io.github.dfa1.vortex.core.fbs.FbsFooter f) { List.copyOf(segmentSpecs), List.copyOf(compressionSpecs)); } - private static Layout convertLayout(io.github.dfa1.vortex.core.fbs.FbsLayout l, List layoutSpecs, int depth) { - if (depth > MAX_LAYOUT_DEPTH) { - throw new VortexException( - "layout tree depth exceeds limit (" + MAX_LAYOUT_DEPTH + ")"); - } + private static Layout convertLayout(io.github.dfa1.vortex.core.fbs.FbsLayout l, List layoutSpecs) { int encIdx = l.encoding(); if (encIdx < 0 || encIdx >= layoutSpecs.size()) { throw new VortexException( @@ -238,7 +243,7 @@ private static Layout convertLayout(io.github.dfa1.vortex.core.fbs.FbsLayout l, var children = new ArrayList(l.childrenLength()); for (int i = 0; i < l.childrenLength(); i++) { - children.add(convertLayout(l.children(i), layoutSpecs, depth + 1)); + children.add(convertLayout(l.children(i), layoutSpecs)); } var segments = new ArrayList(l.segmentsLength()); @@ -249,11 +254,7 @@ private static Layout convertLayout(io.github.dfa1.vortex.core.fbs.FbsLayout l, return new Layout(layoutId, l.rowCount(), metadata, List.copyOf(children), List.copyOf(segments)); } - private static DType convertDType(io.github.dfa1.vortex.core.fbs.FbsDType fbs, int depth) { - if (depth > MAX_DTYPE_DEPTH) { - throw new VortexException( - "DType tree depth exceeds limit (" + MAX_DTYPE_DEPTH + ")"); - } + private static DType convertDType(io.github.dfa1.vortex.core.fbs.FbsDType fbs) { int typeType = fbs.typeType(); return switch (typeType) { case FbsType.FbsNull -> new DType.Null(true); @@ -284,6 +285,13 @@ private static DType convertDType(io.github.dfa1.vortex.core.fbs.FbsDType fbs, i case FbsType.FbsBinary -> new DType.Binary(fbs.type(new FbsBinary()).nullable()); case FbsType.FbsStruct -> { var s = fbs.type(new FbsStruct()); + // Rust (StructFields::from_fb) requires both vectors; only an empty one is a 0-field struct. + if (!s.hasNames()) { + throw new VortexException("failed to parse struct names from flatbuffer"); + } + if (!s.hasDtypes()) { + throw new VortexException("failed to parse struct dtypes from flatbuffer"); + } if (s.namesLength() != s.dtypesLength()) { throw new VortexException("struct names/dtypes length mismatch: " + s.namesLength() + " names, " + s.dtypesLength() + " dtypes"); @@ -311,17 +319,17 @@ private static DType convertDType(io.github.dfa1.vortex.core.fbs.FbsDType fbs, i names.add(new io.github.dfa1.vortex.core.model.ColumnName(name)); } for (int i = 0; i < s.dtypesLength(); i++) { - types.add(convertDType(s.dtypes(i), depth + 1)); + types.add(convertDType(s.dtypes(i))); } yield new DType.Struct(List.copyOf(names), List.copyOf(types), s.nullable()); } case FbsType.FbsList -> { var l = fbs.type(new io.github.dfa1.vortex.core.fbs.FbsList()); - yield new DType.List(convertDType(l.elementType(), depth + 1), l.nullable()); + yield new DType.List(convertDType(l.elementType()), l.nullable()); } case FbsType.FbsFixedSizeList -> { var fsl = fbs.type(new FbsFixedSizeList()); - yield new DType.FixedSizeList(convertDType(fsl.elementType(), depth + 1), (int) fsl.size(), fsl.nullable()); + yield new DType.FixedSizeList(convertDType(fsl.elementType()), (int) fsl.size(), fsl.nullable()); } case FbsType.FbsMap -> { var m = fbs.type(new FbsMap()); @@ -330,15 +338,15 @@ private static DType convertDType(io.github.dfa1.vortex.core.fbs.FbsDType fbs, i if (m.keyType() == null || m.valueType() == null) { throw new VortexException("map dtype missing key_type or value_type"); } - DType keyType = convertDType(m.keyType(), depth + 1); - DType valueType = convertDType(m.valueType(), depth + 1); + DType keyType = convertDType(m.keyType()); + DType valueType = convertDType(m.valueType()); // A nullable key type is rejected by DType.Map's own constructor, which raises // VortexException — the security contract holds without a second check here. yield new DType.Map(keyType, valueType, m.keysSorted(), m.nullable()); } case FbsType.FbsExtension -> { var e = fbs.type(new FbsExtension()); - DType storage = convertDType(e.storageDtype(), depth + 1); + DType storage = convertDType(e.storageDtype()); // Rust requires the field, empty when there is no metadata MemorySegment metadata = e.metadataAsSegment(); if (metadata == null) { @@ -365,7 +373,7 @@ private static DType convertDType(io.github.dfa1.vortex.core.fbs.FbsDType fbs, i } var types = new ArrayList(u.dtypesLength()); for (int i = 0; i < u.dtypesLength(); i++) { - types.add(convertDType(u.dtypes(i), depth + 1)); + types.add(convertDType(u.dtypes(i))); } var typeIds = new ArrayList(u.typeIdsLength()); for (int i = 0; i < u.typeIdsLength(); i++) { diff --git a/reader/src/main/java/io/github/dfa1/vortex/reader/ScanIterator.java b/reader/src/main/java/io/github/dfa1/vortex/reader/ScanIterator.java index b21b09584..9abbb82ca 100644 --- a/reader/src/main/java/io/github/dfa1/vortex/reader/ScanIterator.java +++ b/reader/src/main/java/io/github/dfa1/vortex/reader/ScanIterator.java @@ -582,10 +582,18 @@ private Layout findZonedLayout(Layout root, ColumnName column) { return null; } int idx = struct.fieldNames().indexOf(column); - if (idx < 0 || idx >= root.children().size()) { + if (idx < 0) { return null; } - return firstZoned(root.children().get(idx)); + return firstZoned(root.children().get(fieldChildOffset(struct) + idx)); + } + + /// Index of field 0 among the root struct layout's children: a nullable struct's layout leads + /// with its validity child (open checks the count, as Rust's `validate_children` does), so the + /// columns start one later. The root validity itself is not applied: [Chunk] has no row-level + /// nulls, so a null root row reads as its field values. + private static int fieldChildOffset(DType.Struct root) { + return root.nullable() ? 1 : 0; } private static Layout firstZoned(Layout layout) { @@ -711,13 +719,14 @@ private void initialize() { if (rootLayout.isStruct() && rootDtype instanceof DType.Struct structDtype) { List projection = options.columns(); - for (int i = 0; i < rootLayout.children().size(); i++) { + int offset = fieldChildOffset(structDtype); + for (int i = 0; i < structDtype.fieldNames().size(); i++) { ColumnName colName = structDtype.fieldNames().get(i); DType colDtype = structDtype.fieldTypes().get(i); if (!projection.isEmpty() && !projection.contains(colName)) { continue; } - Layout colTop = rootLayout.children().get(i); + Layout colTop = rootLayout.children().get(offset + i); var flats = new ArrayList(); collectFlats(colTop, flats); columnFlats.put(colName, flats); @@ -1192,8 +1201,7 @@ private ArrayStats readFlatStats(Layout flat) { SegmentSpec spec = file.footer().segmentSpecs().get(segIdx); long segLen = spec.length(); // Stats are an optional zone-map pruning optimization: a malformed stats segment - // degrades to "no stats" (empty) and never aborts the scan. This mirrors - // VortexReader.readFlatStats — both stats readers swallow bounds errors here. + // degrades to "no stats" (empty) and never aborts the scan. // The trailing 4-byte fbLen lives in the segment's last bytes; reading the whole // segment as a ByteBuffer would fail for segments larger than 2 GB (ByteBuffer cap). if (segLen < 4) { @@ -1205,13 +1213,17 @@ private ArrayStats readFlatStats(Layout flat) { return ArrayStats.empty(); } long fbStart = segLen - 4L - fbLen; - var fbArray = io.github.dfa1.vortex.core.fbs.FbsArray.getRootAsFbsArray(IoBounds.slice(seg, fbStart, fbLen)); - - io.github.dfa1.vortex.core.fbs.FbsArrayNode root = fbArray.root(); - if (root == null) { + try { + var fbArray = io.github.dfa1.vortex.core.fbs.FbsArray.getRootAsFbsArray(IoBounds.slice(seg, fbStart, fbLen)); + io.github.dfa1.vortex.core.fbs.FbsArrayNode root = fbArray.root(); + if (root == null) { + return ArrayStats.empty(); + } + return ArrayStats.fromFbs(root.stats()); + } catch (VortexException _) { + // The FlatBuffer verifier rejected the stats segment: malformed stats read as none. return ArrayStats.empty(); } - return ArrayStats.fromFbs(root.stats()); } // ── Layout decode context ───────────────────────────────────────────────── diff --git a/reader/src/main/java/io/github/dfa1/vortex/reader/SerializedArrayDecoder.java b/reader/src/main/java/io/github/dfa1/vortex/reader/SerializedArrayDecoder.java index aebb46436..25723b5a6 100644 --- a/reader/src/main/java/io/github/dfa1/vortex/reader/SerializedArrayDecoder.java +++ b/reader/src/main/java/io/github/dfa1/vortex/reader/SerializedArrayDecoder.java @@ -30,13 +30,6 @@ /// FlatBuffer parsing, buffer-offset arithmetic, and encoding-spec lookup. final class SerializedArrayDecoder { - /// Hard cap on array-node recursion depth. The encoded array tree nests through child nodes - /// (validity, patches, run-ends, dictionary codes/values, …); a crafted or self-referential - /// FlatBuffer can drive [#convertArrayNode] into unbounded recursion and a [StackOverflowError] - /// — an `Error`, so it would bypass the [VortexException] - /// contract. 64 is well past any real encoding's nesting. - static final int MAX_ARRAY_TREE_DEPTH = 64; - private final ReadRegistry registry; /// Creates a decoder backed by the given registry. @@ -63,38 +56,40 @@ public Array decode(MemorySegment seg, List encodingSpecs, int fbLen = seg.get(LE_INT, segLen - 4L); long fbStart = segLen - 4L - fbLen; IoBounds.checkRange(fbStart, fbLen, segLen); - var fbArray = io.github.dfa1.vortex.core.fbs.FbsArray.getRootAsFbsArray(seg.asSlice(fbStart, fbLen)); - int numBuffers = IoBounds.checkCount(fbArray.buffersLength()); - MemorySegment[] bufs = new MemorySegment[numBuffers]; + // Verified whole on parse (FbsVerifier), so the accessors below stay inside it, every vector + // fits in it, and its tables nest at most 64 deep. + var fbArray = io.github.dfa1.vortex.core.fbs.FbsArray.getRootAsFbsArray(seg.asSlice(fbStart, fbLen)); + MemorySegment[] bufs = new MemorySegment[fbArray.buffersLength()]; long dataOffset = 0; - for (int i = 0; i < numBuffers; i++) { + for (int i = 0; i < bufs.length; i++) { FbsBuffer bufDesc = fbArray.buffers(i); dataOffset += bufDesc.padding(); bufs[i] = IoBounds.slice(seg, dataOffset, bufDesc.length()); dataOffset += bufDesc.length(); } + var root = fbArray.root(); + if (root == null) { + throw new VortexException("array flatbuffer missing root"); + } - ArrayNode rootNode = convertArrayNode(fbArray.root(), encodingSpecs, 0); - var ctx = new DecodeContext(rootNode, dtype, rowCount, bufs, registry, arena); + var ctx = new DecodeContext(convertArrayNode(root, encodingSpecs), dtype, rowCount, bufs, registry, arena); return registry.decode(ctx); } - private static ArrayNode convertArrayNode( - io.github.dfa1.vortex.core.fbs.FbsArrayNode fbs, - List encodingSpecs, - int depth - ) { - if (depth > MAX_ARRAY_TREE_DEPTH) { - throw new VortexException( - "array tree depth exceeds limit (" + MAX_ARRAY_TREE_DEPTH + ")"); + private static ArrayNode convertArrayNode(io.github.dfa1.vortex.core.fbs.FbsArrayNode fbs, + List encodingSpecs) { + int encoding = fbs.encoding(); + if (encoding >= encodingSpecs.size()) { + throw new VortexException("array encoding index " + encoding + " out of range [0, " + + encodingSpecs.size() + ")"); } // Already parsed (and blank-guarded) at the footer boundary. - EncodingId encodingId = encodingSpecs.get(fbs.encoding()); + EncodingId encodingId = encodingSpecs.get(encoding); ArrayNode[] children = new ArrayNode[fbs.childrenLength()]; for (int i = 0; i < children.length; i++) { - children[i] = convertArrayNode(fbs.children(i), encodingSpecs, depth + 1); + children[i] = convertArrayNode(fbs.children(i), encodingSpecs); } int[] bufferIndices = new int[fbs.buffersLength()]; diff --git a/reader/src/main/java/io/github/dfa1/vortex/reader/array/AbstractMaterializedArray.java b/reader/src/main/java/io/github/dfa1/vortex/reader/array/AbstractMaterializedArray.java index 83f34659c..199cded69 100644 --- a/reader/src/main/java/io/github/dfa1/vortex/reader/array/AbstractMaterializedArray.java +++ b/reader/src/main/java/io/github/dfa1/vortex/reader/array/AbstractMaterializedArray.java @@ -1,6 +1,7 @@ package io.github.dfa1.vortex.reader.array; import io.github.dfa1.vortex.core.model.DType; +import io.github.dfa1.vortex.reader.decode.SegmentBroadcast; import java.lang.foreign.MemorySegment; import java.lang.foreign.SegmentAllocator; @@ -32,8 +33,10 @@ abstract class AbstractMaterializedArray { this.dtype = dtype; this.length = length; this.buffer = buffer; + // Only a full buffer or a single broadcast element: an empty one for a non-empty array made + // the leaves' `i % elementCount` divide by zero (a fuzzer finding), so it is refused here. this.elementCount = switch (dtype) { - case DType.Primitive p -> buffer.byteSize() / p.ptype().byteSize(); + case DType.Primitive p -> SegmentBroadcast.capacity(buffer, p.ptype().byteSize(), length); default -> length; }; } diff --git a/reader/src/main/java/io/github/dfa1/vortex/reader/array/LazyFsstVarBinArray.java b/reader/src/main/java/io/github/dfa1/vortex/reader/array/LazyFsstVarBinArray.java index 79bde5b4f..898e899b6 100644 --- a/reader/src/main/java/io/github/dfa1/vortex/reader/array/LazyFsstVarBinArray.java +++ b/reader/src/main/java/io/github/dfa1/vortex/reader/array/LazyFsstVarBinArray.java @@ -142,13 +142,7 @@ public int getByteLength(long i) { @Override public void forEachByteLength(IntConsumer c) { long n = length; - long cap = SegmentBroadcast.capacity(uncompressedLengths, uncompressedLengthsPType.byteSize()); - if (cap == 0) { - if (n == 0) { - return; - } - throw new VortexException(EncodingId.VORTEX_FSST, "empty uncompressed-lengths child"); - } + long cap = SegmentBroadcast.capacity(uncompressedLengths, uncompressedLengthsPType.byteSize(), n); if (cap >= n) { forEachClaimedLength(c, n); return; diff --git a/reader/src/main/java/io/github/dfa1/vortex/reader/decode/AlpEncodingDecoder.java b/reader/src/main/java/io/github/dfa1/vortex/reader/decode/AlpEncodingDecoder.java index 29a64495a..52e1318bc 100644 --- a/reader/src/main/java/io/github/dfa1/vortex/reader/decode/AlpEncodingDecoder.java +++ b/reader/src/main/java/io/github/dfa1/vortex/reader/decode/AlpEncodingDecoder.java @@ -83,8 +83,7 @@ private static Array decodeF64(DecodeContext ctx, ProtoALPMetadata meta, int exp // the encoder's verify step. double df = F10_F64[expF]; double de = IF10_F64[expE]; - long srcCap = SegmentBroadcast.capacity(src, 8); - checkSource(srcCap, n); + long srcCap = SegmentBroadcast.capacity(src, 8, n); if (meta.patches() == null) { if (srcCap >= n) { @@ -115,8 +114,7 @@ private static Array decodeF32(DecodeContext ctx, ProtoALPMetadata meta, int exp checkExponents(expE, expF, F10_F32.length); float df = F10_F32[expF]; float de = IF10_F32[expE]; - long srcCap = SegmentBroadcast.capacity(src, 4); - checkSource(srcCap, n); + long srcCap = SegmentBroadcast.capacity(src, 4, n); if (meta.patches() == null) { if (srcCap >= n) { @@ -167,13 +165,6 @@ private static void checkExponents(int expE, int expF, int maxExp) { /// /// @param srcCap number of elements physically present in the encoded child /// @param n logical row count - private static void checkSource(long srcCap, long n) { - if (srcCap == 0 && n > 0) { - throw new VortexException(EncodingId.VORTEX_ALP, - "empty encoded child for " + n + " rows"); - } - } - private static void applyPatches(DecodeContext ctx, ProtoPatchesMetadata pm, MemorySegment out, int elemBytes) { long numPatches = pm.len(); if (numPatches == 0) { @@ -188,13 +179,8 @@ private static void applyPatches(DecodeContext ctx, ProtoPatchesMetadata pm, Mem MemorySegment idxSeg = ctx.decodeChildSegment(1, new DType.Primitive(idxPtype, false), numPatches); MemorySegment valSeg = ctx.decodeChildSegment(2, ctx.dtype(), numPatches); - long idxCap = SegmentBroadcast.capacity(idxSeg, idxBytes); - long valCap = SegmentBroadcast.capacity(valSeg, elemBytes); - if (idxCap == 0 || valCap == 0) { - throw new VortexException(EncodingId.VORTEX_ALP, - "empty patch child for " + numPatches + " declared patches (indices=" - + idxSeg.byteSize() + " bytes, values=" + valSeg.byteSize() + " bytes)"); - } + long idxCap = SegmentBroadcast.capacity(idxSeg, idxBytes, numPatches); + long valCap = SegmentBroadcast.capacity(valSeg, elemBytes, numPatches); if (idxCap >= numPatches && valCap >= numPatches) { for (long i = 0; i < numPatches; i++) { long absIdx = PrimitiveArrays.readLong(idxSeg, i * idxBytes, idxPtype, EncodingId.VORTEX_ALP) - offset; diff --git a/reader/src/main/java/io/github/dfa1/vortex/reader/decode/AlpRdEncodingDecoder.java b/reader/src/main/java/io/github/dfa1/vortex/reader/decode/AlpRdEncodingDecoder.java index 06939c43b..26e508ad8 100644 --- a/reader/src/main/java/io/github/dfa1/vortex/reader/decode/AlpRdEncodingDecoder.java +++ b/reader/src/main/java/io/github/dfa1/vortex/reader/decode/AlpRdEncodingDecoder.java @@ -101,7 +101,7 @@ private static Patches decodePatches(DecodeContext ctx, ProtoPatchesMetadata pm) // Pull the small left-values table into a short[] so lookups don't pay an // Array-dispatch per patch hit. Patches are typically <1% of rows. MemorySegment valSeg = ctx.decodeChildSegment(3, DType.U16, numPatches); - long valCap = SegmentBroadcast.capacity(valSeg, 2); + long valCap = SegmentBroadcast.capacity(valSeg, 2, numPatches); short[] leftValues = new short[(int) numPatches]; for (int j = 0; j < numPatches; j++) { leftValues[j] = valSeg.getAtIndex(VortexFormat.LE_SHORT, j % valCap); diff --git a/reader/src/main/java/io/github/dfa1/vortex/reader/decode/BitpackedEncodingDecoder.java b/reader/src/main/java/io/github/dfa1/vortex/reader/decode/BitpackedEncodingDecoder.java index c4feb8c1c..3b4468a79 100644 --- a/reader/src/main/java/io/github/dfa1/vortex/reader/decode/BitpackedEncodingDecoder.java +++ b/reader/src/main/java/io/github/dfa1/vortex/reader/decode/BitpackedEncodingDecoder.java @@ -48,7 +48,10 @@ public Array decode(DecodeContext ctx) { int bitWidth = meta.bit_width(); int offset = meta.offset(); - PType ptype = ((DType.Primitive) ctx.dtype()).ptype(); + if (!(ctx.dtype() instanceof DType.Primitive p)) { + throw new VortexException(EncodingId.FASTLANES_BITPACKED, "expected primitive dtype, got " + ctx.dtype()); + } + PType ptype = p.ptype(); if (ptype.isFloating()) { throw new VortexException(EncodingId.FASTLANES_BITPACKED, "unsupported ptype " + ptype); } diff --git a/reader/src/main/java/io/github/dfa1/vortex/reader/decode/DecodeContext.java b/reader/src/main/java/io/github/dfa1/vortex/reader/decode/DecodeContext.java index 3e437f655..c69176a86 100644 --- a/reader/src/main/java/io/github/dfa1/vortex/reader/decode/DecodeContext.java +++ b/reader/src/main/java/io/github/dfa1/vortex/reader/decode/DecodeContext.java @@ -59,19 +59,27 @@ public Array decodeChild(int i, DType dtype, long rowCount) { /// @param i zero-based child index within this node's children array /// @return the primary [MemorySegment] of the decoded child public MemorySegment decodeChildSegment(int i) { - var childCtx = new DecodeContext(child(i), dtype, rowCount, segmentBuffers, registry, arena); - return registry.decodeAsSegment(childCtx); + return decodeChildSegment(i, dtype, rowCount); } /// Recursively decode child `i` with an explicit dtype and row count, returning its primary segment. /// + /// A primitive child's segment has one of the two legal shapes, checked here once for every + /// decoder ([SegmentBroadcast#capacity(MemorySegment, int, long)]): at least `rowCount` + /// elements, or exactly one (a constant child broadcast across every row). + /// /// @param i zero-based child index within this node's children array /// @param dtype logical type to assign to the child context /// @param rowCount number of logical rows for the child /// @return the primary [MemorySegment] of the decoded child + /// @throws io.github.dfa1.vortex.core.error.VortexException if the segment has neither shape public MemorySegment decodeChildSegment(int i, DType dtype, long rowCount) { var childCtx = new DecodeContext(child(i), dtype, rowCount, segmentBuffers, registry, arena); - return registry.decodeAsSegment(childCtx); + MemorySegment segment = registry.decodeAsSegment(childCtx); + if (dtype instanceof DType.Primitive p) { + SegmentBroadcast.capacity(segment, p.ptype().byteSize(), rowCount); + } + return segment; } /// Returns buffer `bufferPosition` of child `i` without decoding that child. diff --git a/reader/src/main/java/io/github/dfa1/vortex/reader/decode/DeltaEncodingDecoder.java b/reader/src/main/java/io/github/dfa1/vortex/reader/decode/DeltaEncodingDecoder.java index 36d6cdd2e..e5433150f 100644 --- a/reader/src/main/java/io/github/dfa1/vortex/reader/decode/DeltaEncodingDecoder.java +++ b/reader/src/main/java/io/github/dfa1/vortex/reader/decode/DeltaEncodingDecoder.java @@ -44,7 +44,10 @@ public Array decode(DecodeContext ctx) { } } - PType ptype = ((DType.Primitive) ctx.dtype()).ptype(); + if (!(ctx.dtype() instanceof DType.Primitive p)) { + throw new VortexException(EncodingId.FASTLANES_DELTA, "expected primitive dtype, got " + ctx.dtype()); + } + PType ptype = p.ptype(); if (ptype.isFloating()) { throw new VortexException(EncodingId.FASTLANES_DELTA, "unsupported ptype: " + ptype); } @@ -87,8 +90,8 @@ public Array decode(DecodeContext ctx) { MemorySegment basesSeg = ctx.decodeChildSegment(0, dtype, basesLen); MemorySegment deltasSeg = ctx.decodeChildSegment(1, dtype, deltasLen); int elemBytes = ptype.byteSize(); - long basesCap = SegmentBroadcast.capacity(basesSeg, elemBytes); - long deltasCap = SegmentBroadcast.capacity(deltasSeg, elemBytes); + long basesCap = SegmentBroadcast.capacity(basesSeg, elemBytes, basesLen); + long deltasCap = SegmentBroadcast.capacity(deltasSeg, elemBytes, deltasLen); // The only row-scaled allocation: the output itself, off-heap and at the column's own // width. Everything below is fixed-size scratch — one chunk's worth, cache-resident, diff --git a/reader/src/main/java/io/github/dfa1/vortex/reader/decode/DictEncodingDecoder.java b/reader/src/main/java/io/github/dfa1/vortex/reader/decode/DictEncodingDecoder.java index d6f4b85ff..aa40b32ff 100644 --- a/reader/src/main/java/io/github/dfa1/vortex/reader/decode/DictEncodingDecoder.java +++ b/reader/src/main/java/io/github/dfa1/vortex/reader/decode/DictEncodingDecoder.java @@ -306,7 +306,7 @@ private static BoolArray rowValidity(DecodeContext ctx, MemorySegment codesBuf, return codesValidity; } MemorySegment bits = ctx.arena().allocate((rowCount + 7) >>> 3); - long codesCap = SegmentBroadcast.capacity(codesBuf, codePType.byteSize()); + long codesCap = SegmentBroadcast.capacity(codesBuf, codePType.byteSize(), rowCount); if (codesCap >= rowCount) { for (long i = 0; i < rowCount; i++) { boolean valid = (codesValidity == null || codesValidity.getBoolean(i)) diff --git a/reader/src/main/java/io/github/dfa1/vortex/reader/decode/OnPairEncodingDecoder.java b/reader/src/main/java/io/github/dfa1/vortex/reader/decode/OnPairEncodingDecoder.java index 3a34ac418..79e7b2fe2 100644 --- a/reader/src/main/java/io/github/dfa1/vortex/reader/decode/OnPairEncodingDecoder.java +++ b/reader/src/main/java/io/github/dfa1/vortex/reader/decode/OnPairEncodingDecoder.java @@ -190,7 +190,7 @@ private static long[] childLongs(DecodeContext ctx, int i, ProtoPType wirePtype, PType ptype = PType.fromOrdinal(wirePtype.value()); MemorySegment seg = ctx.decodeChildSegment(i, new DType.Primitive(ptype, false), count); int width = ptype.byteSize(); - if (n > 0 && SegmentBroadcast.capacity(seg, width) < n) { + if (SegmentBroadcast.capacity(seg, width, n) < n) { MemorySegment expanded = ctx.arena().allocate((long) n * width, width); SegmentBroadcast.broadcastCopy(seg, expanded, n, width); seg = expanded; diff --git a/reader/src/main/java/io/github/dfa1/vortex/reader/decode/PatchedEncodingDecoder.java b/reader/src/main/java/io/github/dfa1/vortex/reader/decode/PatchedEncodingDecoder.java index 4e2c57fbb..8c262cc7e 100644 --- a/reader/src/main/java/io/github/dfa1/vortex/reader/decode/PatchedEncodingDecoder.java +++ b/reader/src/main/java/io/github/dfa1/vortex/reader/decode/PatchedEncodingDecoder.java @@ -95,25 +95,25 @@ public Array decode(DecodeContext ctx) { private static MemorySegment patchedOutput(DecodeContext ctx, MemorySegment innerSeg, long n, int elemBytes, long nPatches, long nChunks, long nLanes, long offset, MemorySegment laneOffsetsSeg, MemorySegment patchIndicesSeg, MemorySegment patchValuesSeg) { - if (nPatches == 0 && SegmentBroadcast.capacity(innerSeg, elemBytes) >= n) { + if (nPatches == 0 && SegmentBroadcast.capacity(innerSeg, elemBytes, n) >= n) { return innerSeg.asSlice(0, n * elemBytes).asReadOnly(); } MemorySegment out = ctx.arena().allocate(n * elemBytes); SegmentBroadcast.broadcastCopy(innerSeg, out, n, elemBytes); if (nPatches > 0) { - applyPatches(out, n, nChunks, nLanes, offset, elemBytes, + applyPatches(out, n, nPatches, nChunks, nLanes, offset, elemBytes, laneOffsetsSeg, patchIndicesSeg, patchValuesSeg); } return out; } private static void applyPatches( - MemorySegment out, long n, long nChunks, long nLanes, long offset, int elemBytes, + MemorySegment out, long n, long nPatches, long nChunks, long nLanes, long offset, int elemBytes, MemorySegment laneOffsets, MemorySegment patchIndices, MemorySegment patchValues ) { - long laneCap = SegmentBroadcast.capacity(laneOffsets, 4); - long idxCap = SegmentBroadcast.capacity(patchIndices, 2); - long valCap = SegmentBroadcast.capacity(patchValues, elemBytes); + long laneCap = SegmentBroadcast.capacity(laneOffsets, 4, nChunks * nLanes + 1); + long idxCap = SegmentBroadcast.capacity(patchIndices, 2, nPatches); + long valCap = SegmentBroadcast.capacity(patchValues, elemBytes, nPatches); for (long chunk = 0; chunk < nChunks; chunk++) { long start = Integer.toUnsignedLong( laneOffsets.getAtIndex(VortexFormat.LE_INT, (chunk * nLanes) % laneCap)); diff --git a/reader/src/main/java/io/github/dfa1/vortex/reader/decode/RleEncodingDecoder.java b/reader/src/main/java/io/github/dfa1/vortex/reader/decode/RleEncodingDecoder.java index 8f5d6f881..d48a3df1d 100644 --- a/reader/src/main/java/io/github/dfa1/vortex/reader/decode/RleEncodingDecoder.java +++ b/reader/src/main/java/io/github/dfa1/vortex/reader/decode/RleEncodingDecoder.java @@ -9,6 +9,7 @@ import io.github.dfa1.vortex.core.proto.ProtoRLEMetadata; import io.github.dfa1.vortex.reader.array.Array; import io.github.dfa1.vortex.reader.array.BoolArray; +import io.github.dfa1.vortex.reader.array.LazyConstantBoolArray; import io.github.dfa1.vortex.reader.array.LazyConstantByteArray; import io.github.dfa1.vortex.reader.array.LazyConstantDoubleArray; import io.github.dfa1.vortex.reader.array.LazyConstantFloat16Array; @@ -149,8 +150,13 @@ public Array decode(DecodeContext ctx) { private static Array emptyArray(DecodeContext ctx) { DType dt = ctx.dtype(); - PType ptype = ((DType.Primitive) dt).ptype(); - return switch (ptype) { + if (dt instanceof DType.Bool) { + return new LazyConstantBoolArray(dt, 0L, false); + } + if (!(dt instanceof DType.Primitive p)) { + throw new VortexException(EncodingId.FASTLANES_RLE, "expected Primitive dtype, got " + dt); + } + return switch (p.ptype()) { case I64, U64 -> new LazyConstantLongArray(dt, 0L, 0L); case I32, U32 -> new LazyConstantIntArray(dt, 0L, 0); case I16, U16 -> new LazyConstantShortArray(dt, 0L, (short) 0); @@ -188,11 +194,7 @@ static MemorySegment fitElements(SegmentAllocator arena, MemorySegment buf, long if (buf.byteSize() >= bytes) { return IoBounds.slice(buf, 0, bytes); } - long capacity = SegmentBroadcast.capacity(buf, elemBytes); - if (capacity != 1) { - throw new VortexException(EncodingId.FASTLANES_RLE, - "child holds " + capacity + " element(s) for a declared length of " + count); - } + // Shorter than `count`: broadcastCopy admits only the single element of a constant child. MemorySegment out = arena.allocate(bytes, elemBytes); SegmentBroadcast.broadcastCopy(buf, out, count, elemBytes); return out; diff --git a/reader/src/main/java/io/github/dfa1/vortex/reader/decode/RunEndEncodingDecoder.java b/reader/src/main/java/io/github/dfa1/vortex/reader/decode/RunEndEncodingDecoder.java index 7557dd664..c68da298d 100644 --- a/reader/src/main/java/io/github/dfa1/vortex/reader/decode/RunEndEncodingDecoder.java +++ b/reader/src/main/java/io/github/dfa1/vortex/reader/decode/RunEndEncodingDecoder.java @@ -146,10 +146,10 @@ private static Array withRunValidity(Array result, BoolArray valuesValidity, Arr private static void validateEnds(MemorySegment endsSeg, PType endsPtype, long numRuns, long offset, long n) { PrimitiveArrays.requireUnsigned(endsPtype, EncodingId.VORTEX_RUNEND); int endsBytes = endsPtype.byteSize(); - long endsCap = SegmentBroadcast.capacity(endsSeg, endsBytes); - if (endsCap <= 0) { - throw new VortexException(EncodingId.VORTEX_RUNEND, - "runend: empty ends buffer for " + numRuns + " run(s)"); + long endsCap = SegmentBroadcast.capacity(endsSeg, endsBytes, numRuns); + if (numRuns == 0) { + // An empty array (decode already refused zero runs for rows): nothing to validate. + return; } long prev = PrimitiveArrays.readLong(endsSeg, 0, endsPtype, EncodingId.VORTEX_RUNEND); if (offset != 0 && prev < offset) { diff --git a/reader/src/main/java/io/github/dfa1/vortex/reader/decode/SegmentBroadcast.java b/reader/src/main/java/io/github/dfa1/vortex/reader/decode/SegmentBroadcast.java index c7e040bb8..565341d97 100644 --- a/reader/src/main/java/io/github/dfa1/vortex/reader/decode/SegmentBroadcast.java +++ b/reader/src/main/java/io/github/dfa1/vortex/reader/decode/SegmentBroadcast.java @@ -38,16 +38,25 @@ public static long elementOffset(MemorySegment seg, long i, int elemBytes) { return (i % cap) * elemBytes; } - /// Returns the number of elements physically present in `seg` (≥ 1 for non-empty). + /// Returns the number of elements physically present in `seg`, a child declared `count` + /// elements long, after checking it has one of the two legal shapes: at least `count` + /// elements, or exactly one (a constant child, broadcast across every row). /// - /// Used by callers that want to detect the constant case explicitly — e.g. to hoist - /// a single read out of a hot loop. + /// Any other shape is malformed input: an empty child for a non-empty array made the + /// callers' `i % capacity` broadcast divide by zero, and a truncated one silently repeated + /// its values. Checked here, once, for every caller. /// /// @param seg segment returned by DecodeContext#decodeChildSegment /// @param elemBytes element width in bytes (>= 1) - /// @return physical element count - public static long capacity(MemorySegment seg, int elemBytes) { - return seg.byteSize() / elemBytes; + /// @param count the child's declared element count + /// @return physical element count: `>= count`, or 1 + /// @throws VortexException if `seg` holds neither `count` elements nor exactly one + public static long capacity(MemorySegment seg, int elemBytes, long count) { + long capacity = seg.byteSize() / elemBytes; + if (capacity >= count || capacity == 1) { + return capacity; + } + throw new VortexException("child holds " + capacity + " element(s) for a declared length of " + count); } /// Copies `count` elements from `src` to `dst`, broadcasting the lone @@ -59,13 +68,9 @@ public static long capacity(MemorySegment seg, int elemBytes) { /// @param dst destination segment (must hold ≥ `count * elemBytes` bytes) /// @param count number of elements to materialize /// @param elemBytes element width in bytes (>= 1) - /// @throws VortexException if `src` is empty + /// @throws VortexException if `src` holds neither `count` elements nor exactly one public static void broadcastCopy(MemorySegment src, MemorySegment dst, long count, int elemBytes) { - long cap = src.byteSize() / elemBytes; - if (cap == 0) { - throw new VortexException("source segment is empty (byteSize=" + src.byteSize() - + ", elemBytes=" + elemBytes + ")"); - } + long cap = capacity(src, elemBytes, count); if (cap >= count) { MemorySegment.copy(src, 0, dst, 0, count * elemBytes); return; diff --git a/reader/src/main/java/io/github/dfa1/vortex/reader/decode/VarBinEncodingDecoder.java b/reader/src/main/java/io/github/dfa1/vortex/reader/decode/VarBinEncodingDecoder.java index b9ce5e477..cb65b5692 100644 --- a/reader/src/main/java/io/github/dfa1/vortex/reader/decode/VarBinEncodingDecoder.java +++ b/reader/src/main/java/io/github/dfa1/vortex/reader/decode/VarBinEncodingDecoder.java @@ -40,7 +40,7 @@ public Array decode(DecodeContext ctx) { MemorySegment offsets = ctx.decodeChildSegment(0, offsetsDtype, n + 1); int offBytes = offsetsPtype.byteSize(); - long offCap = SegmentBroadcast.capacity(offsets, offBytes); + long offCap = SegmentBroadcast.capacity(offsets, offBytes, n + 1); if (offCap < n + 1) { MemorySegment materialized = ctx.arena().allocate((n + 1) * offBytes, offBytes); SegmentBroadcast.broadcastCopy(offsets, materialized, n + 1, offBytes); diff --git a/reader/src/main/java/io/github/dfa1/vortex/reader/decode/ZigZagEncodingDecoder.java b/reader/src/main/java/io/github/dfa1/vortex/reader/decode/ZigZagEncodingDecoder.java index 177441544..d415b1687 100644 --- a/reader/src/main/java/io/github/dfa1/vortex/reader/decode/ZigZagEncodingDecoder.java +++ b/reader/src/main/java/io/github/dfa1/vortex/reader/decode/ZigZagEncodingDecoder.java @@ -43,7 +43,7 @@ public Array decode(DecodeContext ctx) { MaskedArray.Unwrapped unwrapped = MaskedArray.unwrap(encoded); MemorySegment src = ctx.materialize(unwrapped.inner()); int elemBytes = signed.byteSize(); - long srcCap = SegmentBroadcast.capacity(src, elemBytes); + long srcCap = SegmentBroadcast.capacity(src, elemBytes, n); Array result = decodeUnwrapped(ctx, signed, n, src, srcCap); return MaskedArray.wrapIfPresent(result, unwrapped.validity()); diff --git a/reader/src/main/java/io/github/dfa1/vortex/reader/decode/ZstdBuffersEncodingDecoder.java b/reader/src/main/java/io/github/dfa1/vortex/reader/decode/ZstdBuffersEncodingDecoder.java index 45c81eb4f..07c7030a8 100644 --- a/reader/src/main/java/io/github/dfa1/vortex/reader/decode/ZstdBuffersEncodingDecoder.java +++ b/reader/src/main/java/io/github/dfa1/vortex/reader/decode/ZstdBuffersEncodingDecoder.java @@ -82,11 +82,21 @@ private static DecodeContext unwrap(DecodeContext ctx) { // Empty inner metadata stays an empty segment, not null: an all-default proto (e.g. varbin // with U8 offsets) encodes to zero bytes, and decoders read null as "missing metadata". byte[] innerMetadata = meta.inner_metadata() == null ? new byte[0] : meta.inner_metadata(); - ArrayNode inner = new ArrayNode(EncodingId.parse(meta.inner_encoding_id()), + ArrayNode inner = new ArrayNode(innerEncodingId(meta.inner_encoding_id()), MemorySegment.ofArray(innerMetadata), ctx.node().children(), innerBuffers); return new DecodeContext(inner, ctx.dtype(), ctx.rowCount(), segments, ctx.registry(), ctx.arena()); } + /// The wrapped encoding's id comes from the file's metadata, so an id [EncodingId#parse] refuses + /// (blank, or holding a control character) is corrupt input, as at the footer. + private static EncodingId innerEncodingId(String raw) { + try { + return EncodingId.parse(raw); + } catch (IllegalArgumentException e) { + throw new VortexException(EncodingId.VORTEX_ZSTD_BUFFERS, "invalid inner encoding id: " + e.getMessage()); + } + } + private static long alignment(int declared) { long value = Integer.toUnsignedLong(declared); if (value == 0 || Long.bitCount(value) != 1 || value > MAX_ALIGNMENT) { diff --git a/reader/src/test/java/io/github/dfa1/vortex/reader/ArrayNodeDepthBombSecurityTest.java b/reader/src/test/java/io/github/dfa1/vortex/reader/ArrayNodeDepthBombSecurityTest.java index 252291f55..687f21aa3 100644 --- a/reader/src/test/java/io/github/dfa1/vortex/reader/ArrayNodeDepthBombSecurityTest.java +++ b/reader/src/test/java/io/github/dfa1/vortex/reader/ArrayNodeDepthBombSecurityTest.java @@ -19,13 +19,11 @@ /// [SerializedArrayDecoder]'s `convertArrayNode`. /// /// The decoder walks the array node tree recursively (validity, patches, run-ends, dictionary -/// codes/values, …). Without the [SerializedArrayDecoder#MAX_ARRAY_TREE_DEPTH] cap a crafted segment -/// with thousands of nested children produces a [StackOverflowError] — an `Error` that escapes the -/// "malformed input must surface as [VortexException]" contract (ADR 0003). -/// -/// The two cases bracket the exact `depth > MAX_ARRAY_TREE_DEPTH` boundary: a tree whose deepest -/// node sits at exactly the limit must clear the guard (and then fail later for an unrelated reason -/// — no decoder), while one node deeper must be rejected by the depth guard itself. +/// codes/values, …). A crafted segment with thousands of nested children would overflow the stack +/// with a [StackOverflowError], an `Error` that escapes the "malformed input must surface as +/// [VortexException]" contract (ADR 0003). The FlatBuffer verifier bounds it before the walk: like +/// Rust's, it rejects tables nested deeper than 64, and here the `Array` table plus `depth + 1` +/// nodes are `depth + 2` tables. class ArrayNodeDepthBombSecurityTest { private static final DType DTYPE = DType.I32; @@ -35,11 +33,9 @@ class ArrayNodeDepthBombSecurityTest { @Test void arrayTreeAtDepthLimit_clearsGuard() { try (Arena arena = Arena.ofConfined()) { - // Given — deepest node at exactly MAX_ARRAY_TREE_DEPTH: convertArrayNode is entered with - // depth == limit there, and `limit > limit` is false. The walk completes and only then - // fails because the empty registry has no decoder. Kills `depth >` relaxed to `>=`, - // which would wrongly reject this legal max-depth tree with the depth message. - byte[] fb = deeplyNestedArrayFlatBuffer(SerializedArrayDecoder.MAX_ARRAY_TREE_DEPTH); + // Given — 64 tables, exactly the verifier's limit: the walk completes and only then + // fails because the empty registry has no decoder + byte[] fb = deeplyNestedArrayFlatBuffer(62); MemorySegment seg = wrapAsSegment(fb, arena); List encodings = List.of(EncodingId.parse("vortex.flat")); @@ -53,16 +49,15 @@ void arrayTreeAtDepthLimit_clearsGuard() { @Test void arrayTreeOneOverDepthLimit_throwsVortexException() { try (Arena arena = Arena.ofConfined()) { - // Given — one level deeper: the deepest node reaches limit + 1, tripping the guard - // before any StackOverflowError can escape. - byte[] fb = deeplyNestedArrayFlatBuffer(SerializedArrayDecoder.MAX_ARRAY_TREE_DEPTH + 1); + // Given — one table deeper (65): rejected by the verifier before the walk starts + byte[] fb = deeplyNestedArrayFlatBuffer(63); MemorySegment seg = wrapAsSegment(fb, arena); List encodings = List.of(EncodingId.parse("vortex.flat")); // When / Then — must surface as VortexException, not StackOverflowError assertThatThrownBy(() -> sut.decode(seg, encodings, DTYPE, 1, arena)) .isInstanceOf(VortexException.class) - .hasMessageContaining("depth"); + .hasMessageContaining("nest deeper than 64"); } } diff --git a/reader/src/test/java/io/github/dfa1/vortex/reader/DTypeDepthBombSecurityTest.java b/reader/src/test/java/io/github/dfa1/vortex/reader/DTypeDepthBombSecurityTest.java index f63afb882..752ebddbc 100644 --- a/reader/src/test/java/io/github/dfa1/vortex/reader/DTypeDepthBombSecurityTest.java +++ b/reader/src/test/java/io/github/dfa1/vortex/reader/DTypeDepthBombSecurityTest.java @@ -22,7 +22,7 @@ /// Adversarial test for the DType-tree recursion in [PostscriptParser]'s `convertDType`. /// /// `convertDType` walks Struct fields, List/FixedSizeList element types, and Extension storage -/// types recursively. Without the [PostscriptParser#MAX_DTYPE_DEPTH] cap a crafted file with +/// types recursively. Without the FlatBuffer verifier's 64-table nesting cap a crafted file with /// thousands of nested `List` types produces a [StackOverflowError] during `VortexReader.open` — /// an `Error` that escapes the [VortexException] sanitization and leaks the reader's memory-mapped /// Arena (`open`'s `catch (Exception)` never runs `arena.close()`). This pins the contract: deeply @@ -36,8 +36,8 @@ class DTypeDepthBombSecurityTest { void deeplyNestedDtype_throwsVortexException(NestKind kind, @TempDir Path tmp) throws Exception { // Given — a file whose DType nests 65536 levels of `kind`. Real schemas nest a handful of // levels; 65536 reliably blows the JVM stack on the recursive convertDType walk. Each kind - // drives a different recursion arm (List/Struct/FixedSizeList/Extension), so all four must - // increment the depth counter for the MAX_DTYPE_DEPTH guard to bound them. + // drives a different recursion arm (List/Struct/FixedSizeList/Extension); the verifier + // walks each union member, so all four must be bounded by its table-nesting cap. Path file = buildDeeplyNestedDtypeFile(tmp, 65536, kind); // When / Then — must surface as VortexException, not StackOverflowError diff --git a/reader/src/test/java/io/github/dfa1/vortex/reader/PostscriptParserBlobBoundsTest.java b/reader/src/test/java/io/github/dfa1/vortex/reader/PostscriptParserBlobBoundsTest.java index 9ac67f77e..1e40526e8 100644 --- a/reader/src/test/java/io/github/dfa1/vortex/reader/PostscriptParserBlobBoundsTest.java +++ b/reader/src/test/java/io/github/dfa1/vortex/reader/PostscriptParserBlobBoundsTest.java @@ -14,7 +14,6 @@ import java.lang.foreign.MemorySegment; -import static org.assertj.core.api.Assertions.assertThat; import static org.assertj.core.api.Assertions.assertThatCode; import static org.assertj.core.api.Assertions.assertThatThrownBy; @@ -135,17 +134,16 @@ void parse_dtypeBlobPastEof_throwsNamingDtype() { void parse_dtypeBlobLengthZero_skipsDtypeCheckEvenWithBadOffset() { // Given — dtype length 0 with a wildly out-of-range offset. The `length > 0` gate must // skip both the bounds check and the slice, so a zero-length dtype never trips on its - // offset and the file still parses (dtype absent). Kills the `length > 0` boundary/negate - // mutants that would otherwise run the check on an empty dtype. + // offset: the file fails as one with no dtype, not on the offset. Kills the `length > 0` + // boundary/negate mutants that would otherwise run the check on an empty dtype. Fixture f = validFile(); MemorySegment ps = buildPostscript(f.footerOff, f.footerLen, f.fileSize + 999, 0, f.layoutOff, f.layoutLen); - // When - PostscriptParser.ParsedFile result = PostscriptParser.parse(ps, f.segment, f.fileSize); - - // Then — parsed, with no dtype - assertThat(result.dtype()).isNull(); + // When / Then + assertThatThrownBy(() -> PostscriptParser.parse(ps, f.segment, f.fileSize)) + .isInstanceOf(VortexException.class) + .hasMessageContaining("doesn't embed a DType"); } // ── FlatBuffer blob builders (minimal, just enough to parse) ──────────────── diff --git a/reader/src/test/java/io/github/dfa1/vortex/reader/PostscriptParserDTypeGuardsTest.java b/reader/src/test/java/io/github/dfa1/vortex/reader/PostscriptParserDTypeGuardsTest.java index 9e62e6437..179a9f673 100644 --- a/reader/src/test/java/io/github/dfa1/vortex/reader/PostscriptParserDTypeGuardsTest.java +++ b/reader/src/test/java/io/github/dfa1/vortex/reader/PostscriptParserDTypeGuardsTest.java @@ -64,6 +64,78 @@ void convertDType_structNamesDtypesArityMismatch_throwsVortexException() { .hasMessageContaining("names/dtypes length mismatch"); } + @Test + void convertDType_structWithoutNamesVector_throwsVortexException() { + // Given — a struct with no names vector at all, which Rust refuses (StructFields::from_fb) + // while it accepts an empty one; read as 0 fields, it failed the scan as a raw + // ArrayIndexOutOfBoundsException (a fuzzer finding) + MemorySegment dtype = structDType(false, true); + + // When / Then + MemorySegment footer = minimalFooter(); + MemorySegment layout = flatLayout(); + assertThatThrownBy(() -> PostscriptParser.parseBlobs(footer, layout, dtype)) + .isInstanceOf(VortexException.class) + .hasMessageContaining("failed to parse struct names"); + } + + @Test + void convertDType_structWithoutDtypesVector_throwsVortexException() { + // Given — the same for the dtypes vector + MemorySegment dtype = structDType(true, false); + + // When / Then + MemorySegment footer = minimalFooter(); + MemorySegment layout = flatLayout(); + assertThatThrownBy(() -> PostscriptParser.parseBlobs(footer, layout, dtype)) + .isInstanceOf(VortexException.class) + .hasMessageContaining("failed to parse struct dtypes"); + } + + @Test + void convertDType_structWithEmptyVectors_isAZeroFieldStruct() { + // Given — both vectors present but empty: Rust's legal 0-field struct + MemorySegment dtype = structDType(true, true); + + // When + MemorySegment footer = minimalFooter(); + MemorySegment layout = flatLayout(); + PostscriptParser.ParsedFile result = PostscriptParser.parseBlobs(footer, layout, dtype); + + // Then + assertThat(result.dtype()).isEqualTo(new DType.Struct(List.of(), List.of(), false)); + } + + @org.junit.jupiter.params.ParameterizedTest + @org.junit.jupiter.params.provider.ValueSource(ints = {0, 2}) + void parseBlobs_structLayoutChildCountNotFieldCount_throwsVortexException(int children) { + // Given — a one-field struct dtype over a struct layout with too few or too many children: + // Rust refuses it while deserializing the layout tree; the scan used to index the field + // list by layout child and throw ArrayIndexOutOfBoundsException (a fuzzer finding) + MemorySegment dtype = structDType(new String[]{"a"}, 1); + + // When / Then + MemorySegment footer = flatAndStructFooter(); + MemorySegment layout = structLayout(children); + assertThatThrownBy(() -> PostscriptParser.parseBlobs(footer, layout, dtype)) + .isInstanceOf(VortexException.class) + .hasMessageContaining("vortex.struct layout has " + children + " children but its struct dtype needs 1"); + } + + @Test + void parseBlobs_structLayoutOneChildPerField_isAccepted() { + // Given + MemorySegment dtype = structDType(new String[]{"a"}, 1); + + // When + MemorySegment footer = flatAndStructFooter(); + MemorySegment layout = structLayout(1); + PostscriptParser.ParsedFile result = PostscriptParser.parseBlobs(footer, layout, dtype); + + // Then + assertThat(result.layout().children()).hasSize(1); + } + @org.junit.jupiter.params.ParameterizedTest @org.junit.jupiter.params.provider.ValueSource(strings = {"", " ", " "}) void convertDType_blankFieldName_isAccepted(String name) { @@ -354,6 +426,18 @@ private static MemorySegment structDType(String[] names, int dtypeCount) { return fbb.dataSegment(); } + /// A 0-field struct dtype whose names/dtypes vectors are written empty or left out entirely + /// (a zero offset omits the field). + private static MemorySegment structDType(boolean withNames, boolean withDtypes) { + var fbb = new FbsBuilder(256); + int namesVec = withNames ? FbsStruct.createNamesVector(fbb, new int[0]) : 0; + int dtypesVec = withDtypes ? FbsStruct.createDtypesVector(fbb, new int[0]) : 0; + int struct = FbsStruct.createFbsStruct(fbb, namesVec, dtypesVec, false); + int root = FbsDType.createFbsDType(fbb, FbsType.FbsStruct, struct); + fbb.finish(root); + return fbb.dataSegment(); + } + private static MemorySegment minimalFooter() { var fbb = new FbsBuilder(256); int asv = FbsFooter.createArraySpecsVector(fbb, new int[]{ @@ -367,6 +451,35 @@ private static MemorySegment minimalFooter() { return fbb.dataSegment(); } + /// A footer declaring layout ids 0 = `vortex.flat`, 1 = `vortex.struct`. + private static MemorySegment flatAndStructFooter() { + var fbb = new FbsBuilder(256); + int asv = FbsFooter.createArraySpecsVector(fbb, new int[]{ + FbsArraySpec.createFbsArraySpec(fbb, fbb.createString("vortex.primitive"))}); + int lsv = FbsFooter.createLayoutSpecsVector(fbb, new int[]{ + FbsLayoutSpec.createFbsLayoutSpec(fbb, fbb.createString("vortex.flat")), + FbsLayoutSpec.createFbsLayoutSpec(fbb, fbb.createString("vortex.struct"))}); + FbsFooter.startSegmentSpecsVector(fbb, 0); + int ssv = fbb.endVector(); + int footOff = FbsFooter.createFbsFooter(fbb, asv, lsv, ssv, 0, 0); + fbb.finish(footOff); + return fbb.dataSegment(); + } + + /// A struct layout (id 1 in [#flatAndStructFooter()]) over `children` flat layouts. + private static MemorySegment structLayout(int children) { + var fbb = new FbsBuilder(256); + int[] flats = new int[children]; + for (int i = 0; i < children; i++) { + int segV = FbsLayout.createSegmentsVector(fbb, new long[]{0}); + flats[i] = FbsLayout.createFbsLayout(fbb, 0, 1L, 0, 0, segV); + } + int childV = FbsLayout.createChildrenVector(fbb, flats); + int off = FbsLayout.createFbsLayout(fbb, 1, 1L, 0, childV, 0); + FbsLayout.finishFbsLayoutBuffer(fbb, off); + return fbb.dataSegment(); + } + private static MemorySegment flatLayout() { var fbb = new FbsBuilder(128); int segV = FbsLayout.createSegmentsVector(fbb, new long[]{0}); diff --git a/reader/src/test/java/io/github/dfa1/vortex/reader/PostscriptParserLayoutIdTest.java b/reader/src/test/java/io/github/dfa1/vortex/reader/PostscriptParserLayoutIdTest.java index 3cb4f862e..7be398155 100644 --- a/reader/src/test/java/io/github/dfa1/vortex/reader/PostscriptParserLayoutIdTest.java +++ b/reader/src/test/java/io/github/dfa1/vortex/reader/PostscriptParserLayoutIdTest.java @@ -29,6 +29,9 @@ /// [IllegalArgumentException] that [LayoutId#parse] raises for a blank id. class PostscriptParserLayoutIdTest { + /// Every file must embed a dtype (Rust refuses one without), so layout-only cases pass a minimal one. + private static final MemorySegment I64_DTYPE = MemorySegment.ofBuffer(MalformedFiles.buildI64Dtype()); + @ParameterizedTest @ValueSource(strings = {"vortex.stats", "vortex.zoned"}) void convertLayout_zonedAlias_parsesToZonedLayoutWithPassthroughChild(String zonedSpec) { @@ -38,7 +41,7 @@ void convertLayout_zonedAlias_parsesToZonedLayoutWithPassthroughChild(String zon MemorySegment layout = zonedOverFlatLayout(); // When - Layout result = PostscriptParser.parseBlobs(footer, layout, null).layout(); + Layout result = PostscriptParser.parseBlobs(footer, layout, I64_DTYPE).layout(); // Then — both aliases resolve to a zoned layout with the same shape: isZoned() is true and // the single flat child is preserved for pass-through decoding. @@ -57,7 +60,7 @@ void convertLayout_blankLayoutId_throwsVortexException() { MemorySegment layout = flatLayout(); // When / Then - assertThatThrownBy(() -> PostscriptParser.parseBlobs(footer, layout, null)) + assertThatThrownBy(() -> PostscriptParser.parseBlobs(footer, layout, I64_DTYPE)) .isInstanceOf(VortexException.class) .hasMessageContaining("blank layout id"); } diff --git a/reader/src/test/java/io/github/dfa1/vortex/reader/PostscriptParserParseBlobsBoundsTest.java b/reader/src/test/java/io/github/dfa1/vortex/reader/PostscriptParserParseBlobsBoundsTest.java index 7dea5af77..0f3880677 100644 --- a/reader/src/test/java/io/github/dfa1/vortex/reader/PostscriptParserParseBlobsBoundsTest.java +++ b/reader/src/test/java/io/github/dfa1/vortex/reader/PostscriptParserParseBlobsBoundsTest.java @@ -29,6 +29,9 @@ /// legal value must parse, the first illegal value must throw a [VortexException]. class PostscriptParserParseBlobsBoundsTest { + /// Every file must embed a dtype (Rust refuses one without), so layout-only cases pass a minimal one. + private static final MemorySegment I64_DTYPE = MemorySegment.ofBuffer(MalformedFiles.buildI64Dtype()); + // ── FbsLayout encoding-index bound: encIdx < 0 || encIdx >= layoutSpecs.size() ── @Test @@ -38,7 +41,7 @@ void parseBlobs_layoutEncodingIndex_atLastSpec_parses() { MemorySegment layout = flatLayout(0); // When / Then — the top valid index must be accepted (kills `>=` relaxed to `>`) - assertThatCode(() -> PostscriptParser.parseBlobs(footer, layout, null)) + assertThatCode(() -> PostscriptParser.parseBlobs(footer, layout, I64_DTYPE)) .doesNotThrowAnyException(); } @@ -49,36 +52,35 @@ void parseBlobs_layoutEncodingIndex_equalToSize_throws() { MemorySegment layout = flatLayout(1); // When / Then - assertThatThrownBy(() -> PostscriptParser.parseBlobs(footer, layout, null)) + assertThatThrownBy(() -> PostscriptParser.parseBlobs(footer, layout, I64_DTYPE)) .isInstanceOf(VortexException.class) .hasMessageContaining("encoding index"); } - // ── FbsLayout depth bound: depth > MAX_LAYOUT_DEPTH (64) ──────────────────────── + // ── Layout nesting: the FlatBuffer verifier caps tables at 64 deep, as Rust does ── @Test void parseBlobs_layoutDepth_atLimit_parses() { - // Given — a single-child chain whose deepest node sits at exactly MAX_LAYOUT_DEPTH. - // convertLayout is called with depth == 64 there, and `64 > 64` is false. Kills the - // `depth >` relaxed to `depth >=` mutant, which would reject the legal max-depth tree. + // Given — a single-child chain of 64 layout tables (63 wraps around the leaf): exactly the + // verifier's depth limit, so it parses MemorySegment footer = footerWithLayoutSpecs("vortex.flat"); - MemorySegment layout = nestedLayout(PostscriptParser.MAX_LAYOUT_DEPTH); + MemorySegment layout = nestedLayout(63); // When / Then - assertThatCode(() -> PostscriptParser.parseBlobs(footer, layout, null)) + assertThatCode(() -> PostscriptParser.parseBlobs(footer, layout, I64_DTYPE)) .doesNotThrowAnyException(); } @Test void parseBlobs_layoutDepth_oneOverLimit_throws() { - // Given — one level deeper: the deepest node reaches depth 65, tripping `65 > 64` + // Given — one table deeper (65): rejected before convertLayout recurses at all MemorySegment footer = footerWithLayoutSpecs("vortex.flat"); - MemorySegment layout = nestedLayout(PostscriptParser.MAX_LAYOUT_DEPTH + 1); + MemorySegment layout = nestedLayout(64); // When / Then - assertThatThrownBy(() -> PostscriptParser.parseBlobs(footer, layout, null)) + assertThatThrownBy(() -> PostscriptParser.parseBlobs(footer, layout, I64_DTYPE)) .isInstanceOf(VortexException.class) - .hasMessageContaining("depth"); + .hasMessageContaining("nest deeper than 64"); } // ── FbsDecimal precision bound: precision < 1 || precision > 76 ───────────────── @@ -146,7 +148,7 @@ void parseBlobs_layoutMetadata_atLimit_parses() { MemorySegment layout = flatLayoutWithMetadata((int) PostscriptParser.MAX_LAYOUT_METADATA_BYTES.bytes()); // When / Then - assertThatCode(() -> PostscriptParser.parseBlobs(footer, layout, null)) + assertThatCode(() -> PostscriptParser.parseBlobs(footer, layout, I64_DTYPE)) .doesNotThrowAnyException(); } @@ -157,19 +159,17 @@ void parseBlobs_layoutMetadata_oneOverLimit_throws() { MemorySegment layout = flatLayoutWithMetadata((int) PostscriptParser.MAX_LAYOUT_METADATA_BYTES.bytes() + 1); // When / Then - assertThatThrownBy(() -> PostscriptParser.parseBlobs(footer, layout, null)) + assertThatThrownBy(() -> PostscriptParser.parseBlobs(footer, layout, I64_DTYPE)) .isInstanceOf(VortexException.class) .hasMessageContaining("metadata size"); } - // ── FbsDType depth bound: depth > MAX_DTYPE_DEPTH (64) ────────────────────────── + // ── DType nesting: DType + List per level, so 64 tables allow 31 list wraps ── @Test void parseBlobs_dtypeDepth_atLimit_parses() { - // Given — a single-child List chain whose deepest node sits at exactly MAX_DTYPE_DEPTH. - // convertDType is entered with depth == 64 there, and `64 > 64` is false. Kills the - // `depth >` relaxed to `depth >=` mutant, which would reject the legal max-depth dtype. - MemorySegment dtype = nestedListDtype(PostscriptParser.MAX_DTYPE_DEPTH); + // Given — 31 List wraps: 64 tables, exactly the verifier's depth limit + MemorySegment dtype = nestedListDtype(31); // When DType result = parseDtype(dtype); @@ -180,31 +180,31 @@ void parseBlobs_dtypeDepth_atLimit_parses() { @Test void parseBlobs_dtypeDepth_oneOverLimit_throws() { - // Given — one level deeper: the deepest node reaches depth 65, tripping `65 > 64` - MemorySegment dtype = nestedListDtype(PostscriptParser.MAX_DTYPE_DEPTH + 1); + // Given — one wrap more: 66 tables + MemorySegment dtype = nestedListDtype(32); // When / Then assertThatThrownBy(() -> parseDtype(dtype)) .isInstanceOf(VortexException.class) - .hasMessageContaining("depth"); + .hasMessageContaining("nest deeper than 64"); } // ── parseBlobs dtype-presence guard: dtypeBuf != null && byteSize() > 0 ────────── - @Test - void parseBlobs_emptyDtypeBlob_yieldsNullDtype() { - // Given — a present but zero-length dtype blob. The guard treats an empty blob as "no dtype" - // (`byteSize() > 0`); relaxing `> 0` to `>= 0` would instead feed the empty buffer to - // getRootAsFbsDType and throw. Pins that an empty blob parses to a null dtype. + @org.junit.jupiter.params.ParameterizedTest + @org.junit.jupiter.params.provider.ValueSource(booleans = {true, false}) + void parseBlobs_noDtypeBlob_throws(boolean emptyBlob) { + // Given — no dtype blob, or a present but zero-length one. Rust opens such a file only when + // the caller supplies the dtype (VortexOpenOptions::with_dtype); Java has no such option, + // and a null dtype failed the scan later with a NullPointerException (a fuzzer finding). MemorySegment footer = footerWithLayoutSpecs("vortex.flat"); MemorySegment layout = flatLayout(0); - MemorySegment emptyDtype = MemorySegment.ofArray(new byte[0]); - - // When - DType result = PostscriptParser.parseBlobs(footer, layout, emptyDtype).dtype(); + MemorySegment dtype = emptyBlob ? MemorySegment.ofArray(new byte[0]) : null; - // Then - assertThat(result).isNull(); + // When / Then + assertThatThrownBy(() -> PostscriptParser.parseBlobs(footer, layout, dtype)) + .isInstanceOf(VortexException.class) + .hasMessageContaining("doesn't embed a DType"); } // ── helpers ────────────────────────────────────────────────────────────────── diff --git a/reader/src/test/java/io/github/dfa1/vortex/reader/ScanIteratorMalformedStatsTest.java b/reader/src/test/java/io/github/dfa1/vortex/reader/ScanIteratorMalformedStatsTest.java new file mode 100644 index 000000000..c65baee4e --- /dev/null +++ b/reader/src/test/java/io/github/dfa1/vortex/reader/ScanIteratorMalformedStatsTest.java @@ -0,0 +1,55 @@ +package io.github.dfa1.vortex.reader; + +import io.github.dfa1.vortex.core.io.VortexFormat; +import io.github.dfa1.vortex.core.model.ColumnName; +import io.github.dfa1.vortex.core.model.DType; +import io.github.dfa1.vortex.core.model.LayoutId; +import io.github.dfa1.vortex.reader.layout.Layout; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.mockito.Mock; +import org.mockito.junit.jupiter.MockitoExtension; + +import java.lang.foreign.MemorySegment; +import java.util.List; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.BDDMockito.given; + +/// A malformed stats segment found by the `scanMalformedFile` fuzz target (#493): it escaped the +/// scan as a raw JDK exception instead of degrading to "no stats". +@ExtendWith(MockitoExtension.class) +class ScanIteratorMalformedStatsTest { + + private static final ColumnName COLUMN = ColumnName.of("v"); + private static final DType.Struct SCHEMA = new DType.Struct(List.of(COLUMN), List.of(DType.I64), false); + + @Mock + private VortexHandle file; + + @Test + void statsSegmentWithCorruptFlatBuffer_degradesToEmptyStats() { + // Given — a flat whose segment frames a 4-byte FlatBuffer (the trailing length is sane) + // whose root offset points far outside it; FlatBuffer navigation used to escape as a raw + // IndexOutOfBoundsException although malformed stats are documented to read as "no stats" + MemorySegment seg = MemorySegment.ofArray(new byte[8]); + seg.set(VortexFormat.LE_INT, 0, 1_000_000); + seg.set(VortexFormat.LE_INT, 4, 4); + Layout flat = new Layout(LayoutId.FLAT, 5, null, List.of(), List.of(0)); + given(file.dtype()).willReturn(SCHEMA); + given(file.layout()).willReturn(new Layout(LayoutId.STRUCT, 5, null, List.of(flat), List.of())); + given(file.footer()).willReturn(new Footer(List.of(), List.of(), + List.of(new SegmentSpec(0, 8, (byte) 0, CompressionScheme.NONE)), List.of())); + given(file.rawSegment(any())).willReturn(seg); + + // When + List result; + try (ScanIterator sut = new ScanIterator(file, ScanOptions.all())) { + result = sut.chunkStats(COLUMN); + } + + // Then + assertThat(result).containsExactly(ArrayStats.empty()); + } +} diff --git a/reader/src/test/java/io/github/dfa1/vortex/reader/ScanIteratorNullableRootTest.java b/reader/src/test/java/io/github/dfa1/vortex/reader/ScanIteratorNullableRootTest.java new file mode 100644 index 000000000..54b713334 --- /dev/null +++ b/reader/src/test/java/io/github/dfa1/vortex/reader/ScanIteratorNullableRootTest.java @@ -0,0 +1,57 @@ +package io.github.dfa1.vortex.reader; + +import io.github.dfa1.vortex.core.model.ColumnName; +import io.github.dfa1.vortex.core.model.DType; +import io.github.dfa1.vortex.core.model.LayoutId; +import io.github.dfa1.vortex.reader.layout.Layout; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.mockito.Mock; +import org.mockito.junit.jupiter.MockitoExtension; + +import java.lang.foreign.MemorySegment; +import java.util.List; + +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.BDDMockito.given; +import static org.mockito.BDDMockito.then; + +/// A nullable root struct's layout leads with its validity child, as Rust writes it and as open +/// checks it: the scan must map field `i` to child `i + 1`. Mapping by raw index read column `a` +/// from the validity child, `b` from `a`'s data, and ran off the field list on the last child. +@ExtendWith(MockitoExtension.class) +class ScanIteratorNullableRootTest { + + private static final ColumnName A = ColumnName.of("a"); + private static final ColumnName B = ColumnName.of("b"); + + @Mock + private VortexHandle file; + + @Test + void columnsSkipTheRootValidityChild() { + // Given — layout children [validity (segment 0), a (segment 1), b (segment 2)] + DType.Struct schema = new DType.Struct(List.of(A, B), List.of(DType.I64, DType.I64), true); + List children = List.of(flat(0), flat(1), flat(2)); + SegmentSpec bSegment = new SegmentSpec(200, 8, (byte) 0, CompressionScheme.NONE); + given(file.dtype()).willReturn(schema); + given(file.layout()).willReturn(new Layout(LayoutId.STRUCT, 5, null, children, List.of())); + given(file.footer()).willReturn(new Footer(List.of(), List.of(), List.of( + new SegmentSpec(0, 8, (byte) 0, CompressionScheme.NONE), + new SegmentSpec(100, 8, (byte) 0, CompressionScheme.NONE), + bSegment), List.of())); + given(file.rawSegment(any())).willReturn(MemorySegment.ofArray(new byte[8])); + + // When + try (ScanIterator sut = new ScanIterator(file, ScanOptions.all())) { + sut.chunkStats(B); + } + + // Then — column b's stats come from b's own segment + then(file).should().rawSegment(bSegment); + } + + private static Layout flat(int segment) { + return new Layout(LayoutId.FLAT, 5, null, List.of(), List.of(segment)); + } +} diff --git a/reader/src/test/java/io/github/dfa1/vortex/reader/SerializedArrayBoundsSecurityTest.java b/reader/src/test/java/io/github/dfa1/vortex/reader/SerializedArrayBoundsSecurityTest.java index fdecfce85..962009a2a 100644 --- a/reader/src/test/java/io/github/dfa1/vortex/reader/SerializedArrayBoundsSecurityTest.java +++ b/reader/src/test/java/io/github/dfa1/vortex/reader/SerializedArrayBoundsSecurityTest.java @@ -85,6 +85,76 @@ void bufferDescriptorLengthPastSegment_throwsVortexException() { } } + @Test + void rootOffsetPastFlatBuffer_throwsVortexException() { + try (Arena arena = Arena.ofConfined()) { + // Given a valid FlatBuffer whose root uoffset is rewritten to land 2 bytes before its + // end: the table's soffset there runs off the slice (the first fuzzer finding) + byte[] fb = arrayFlatBufferWithOneBuffer(0L); + MemorySegment seg = segmentOf(arena, fb); + seg.set(VortexFormat.LE_INT, 0, fb.length - 2); + + // When / Then + assertThatThrownBy(() -> sut.decode(seg, FLAT_SPEC, DTYPE, 1, arena)) + .isInstanceOf(VortexException.class) + .hasMessageContaining("malformed flatbuffer: table"); + } + } + + @Test + void encodingIndexPastSpecTable_throwsVortexException() { + try (Arena arena = Arena.ofConfined()) { + // Given a root node naming encoding #5 while the footer declares a single encoding + FbsBuilder b = new FbsBuilder(); + int children = FbsArrayNode.createChildrenVector(b, new int[0]); + int buffers = FbsArrayNode.createBuffersVector(b, new int[0]); + int root = FbsArrayNode.createFbsArrayNode(b, 5, 0, children, buffers, 0); + MemorySegment seg = segmentOf(arena, finish(b, root)); + + // When / Then + assertThatThrownBy(() -> sut.decode(seg, FLAT_SPEC, DTYPE, 1, arena)) + .isInstanceOf(VortexException.class) + .hasMessageContaining("encoding index 5 out of range"); + } + } + + @Test + void childrenVectorClaimingMoreThanTheFlatBuffer_throwsVortexException() { + try (Arena arena = Arena.ofConfined()) { + // Given a children vector whose length prefix claims 100 000 entries it does not hold. + // At Integer.MAX_VALUE the `new ArrayNode[n]` it sizes would be an OutOfMemoryError; + // the verifier rejects the vector before anything is allocated from its length. + FbsBuilder b = new FbsBuilder(); + FbsArrayNode.startChildrenVector(b, 100_000); + int children = b.endVector(); + int buffers = FbsArrayNode.createBuffersVector(b, new int[0]); + int root = FbsArrayNode.createFbsArrayNode(b, 0, 0, children, buffers, 0); + MemorySegment seg = segmentOf(arena, finish(b, root)); + + // When / Then + assertThatThrownBy(() -> sut.decode(seg, FLAT_SPEC, DTYPE, 1, arena)) + .isInstanceOf(VortexException.class) + .hasMessageContaining("vector of 100000 elements"); + } + } + + /// Lays out `fb` as a flat segment: the FlatBuffer followed by its u32 LE length. + private static MemorySegment segmentOf(Arena arena, byte[] fb) { + MemorySegment seg = arena.allocate(fb.length + 4L); + MemorySegment.copy(MemorySegment.ofArray(fb), 0, seg, 0, fb.length); + seg.set(VortexFormat.LE_INT, fb.length, fb.length); + return seg; + } + + /// Wraps `root` in an `FbsArray` with no buffer descriptors. + private static byte[] finish(FbsBuilder b, int root) { + FbsArray.startBuffersVector(b, 0); + int buffers = b.endVector(); + int array = FbsArray.createFbsArray(b, root, buffers); + FbsArray.finishFbsArrayBuffer(b, array); + return b.sizedByteArray(); + } + /// Builds a minimal valid `FbsArray` FlatBuffer with one buffer descriptor of the given length. private static byte[] arrayFlatBufferWithOneBuffer(long bufferLength) { FbsBuilder b = new FbsBuilder(); diff --git a/reader/src/test/java/io/github/dfa1/vortex/reader/decode/AlpEncodingDecoderTest.java b/reader/src/test/java/io/github/dfa1/vortex/reader/decode/AlpEncodingDecoderTest.java index 31725e190..179633a4b 100644 --- a/reader/src/test/java/io/github/dfa1/vortex/reader/decode/AlpEncodingDecoderTest.java +++ b/reader/src/test/java/io/github/dfa1/vortex/reader/decode/AlpEncodingDecoderTest.java @@ -273,7 +273,7 @@ void emptyPatchIndexChild_throws() { // When / Then assertThatThrownBy(() -> SUT.decode(ctx)) .isInstanceOf(VortexException.class) - .hasMessageContaining("empty patch child"); + .hasMessageContaining("child holds 0 element(s)"); } private DecodeContext ctx(DType dtype, ProtoALPMetadata meta, MemorySegment encoded, long rowCount) { diff --git a/reader/src/test/java/io/github/dfa1/vortex/reader/decode/BitpackedEncodingDecoderTest.java b/reader/src/test/java/io/github/dfa1/vortex/reader/decode/BitpackedEncodingDecoderTest.java index 0fbecb5a2..c4df5ef1c 100644 --- a/reader/src/test/java/io/github/dfa1/vortex/reader/decode/BitpackedEncodingDecoderTest.java +++ b/reader/src/test/java/io/github/dfa1/vortex/reader/decode/BitpackedEncodingDecoderTest.java @@ -16,6 +16,7 @@ import java.lang.foreign.Arena; import java.lang.foreign.MemorySegment; +import java.util.List; import static org.assertj.core.api.Assertions.assertThat; import static org.assertj.core.api.Assertions.assertThatThrownBy; @@ -259,6 +260,22 @@ void packedBufferTooSmallForRowCount_throws() { .hasMessageContaining("too small for 1024 rows at bit width 8"); } + @Test + void nonPrimitiveDtype_throws() { + // Given a struct column whose segment claims bit-packing (a fuzzer finding: the + // decoder cast the dtype blindly and escaped as ClassCastException) + ArrayNode node = new ArrayNode(BITPACKED, MemorySegment.ofArray(new ProtoBitPackedMetadata(8, 0, null).encode()), + new ArrayNode[0], new int[]{0}); + DType struct = new DType.Struct(List.of(), List.of(), false); + DecodeContext ctx = new DecodeContext(node, struct, 8, new MemorySegment[]{packedBytes(64)}, REGISTRY, + Arena.ofAuto()); + + // When / Then + assertThatThrownBy(() -> SUT.decode(ctx)) + .isInstanceOf(VortexException.class) + .hasMessageContaining("expected primitive dtype"); + } + private DecodeContext ctxWithMeta(ProtoBitPackedMetadata meta, MemorySegment packed, long rowCount) { ArrayNode node = new ArrayNode(BITPACKED, MemorySegment.ofArray(meta.encode()), new ArrayNode[0], new int[]{0}); diff --git a/reader/src/test/java/io/github/dfa1/vortex/reader/decode/ChunkedEncodingDecoderTest.java b/reader/src/test/java/io/github/dfa1/vortex/reader/decode/ChunkedEncodingDecoderTest.java index fa9b55bd4..d47b97ae7 100644 --- a/reader/src/test/java/io/github/dfa1/vortex/reader/decode/ChunkedEncodingDecoderTest.java +++ b/reader/src/test/java/io/github/dfa1/vortex/reader/decode/ChunkedEncodingDecoderTest.java @@ -54,10 +54,10 @@ void decode_twoChunks_viewsBothWithoutConcatenating() { /// comment is the raw JDK exception the reader leaked before the guards landed. /// /// The TODO's other Chunked item — a maliciously deep child tree — is guarded and tested one - /// layer up, at the point the `ArrayNode` tree is built from the file's FlatBuffer: see - /// [io.github.dfa1.vortex.reader.SerializedArrayDecoder] `MAX_ARRAY_TREE_DEPTH` and - /// `ArrayNodeDepthBombSecurityTest`. FlatBuffers cannot encode a true cycle, so the depth cap - /// is the whole defense and nothing here can reproduce it from a hand-built node. + /// layer up, when the segment's FlatBuffer is verified (`FbsVerifier`, at most 64 nested tables) + /// before the `ArrayNode` tree is built: see `ArrayNodeDepthBombSecurityTest`. FlatBuffers cannot + /// encode a true cycle, so the depth cap is the whole defense and nothing here can reproduce it + /// from a hand-built node. @Nested class AdversarialInput { @@ -102,7 +102,7 @@ void emptyOffsetsChild_throws() { // When / Then assertThatThrownBy(() -> decode(DType.I64, 3, segs, offsetsChild, chunkChild)) .isInstanceOf(VortexException.class) - .hasMessageContaining("decoded child segment is empty"); + .hasMessageContaining("child holds 0 element(s)"); } /// `IndexOutOfBoundsException` — a decreasing offsets pair yields a negative chunk length. diff --git a/reader/src/test/java/io/github/dfa1/vortex/reader/decode/DeltaEncodingDecoderTest.java b/reader/src/test/java/io/github/dfa1/vortex/reader/decode/DeltaEncodingDecoderTest.java index 25379a2e2..e56499ca5 100644 --- a/reader/src/test/java/io/github/dfa1/vortex/reader/decode/DeltaEncodingDecoderTest.java +++ b/reader/src/test/java/io/github/dfa1/vortex/reader/decode/DeltaEncodingDecoderTest.java @@ -304,6 +304,19 @@ void decode_absurdDeltasLenButSmallWindow_decodesWithoutAllocatingForIt() { assertThat(result.getLong(3)).isZero(); } + @Test + void decode_nonPrimitiveDtype_throws() { + // Given a Utf8 column whose segment claims delta encoding: a crafted file, which escaped + // as a raw ClassCastException before the dtype was checked + ArrayNode node = new ArrayNode(EncodingId.FASTLANES_DELTA, null, new ArrayNode[0], new int[0]); + DecodeContext ctx = new DecodeContext(node, DType.UTF8, 0, new MemorySegment[0], REGISTRY, Arena.ofAuto()); + + // When / Then + assertThatThrownBy(() -> SUT.decode(ctx)) + .isInstanceOf(VortexException.class) + .hasMessageContaining("expected primitive dtype"); + } + @Test void decode_constantBases_nonZeroOffsetAndBase() { // Given a constant base of 5 broadcast across all 16 lanes with zero deltas: diff --git a/reader/src/test/java/io/github/dfa1/vortex/reader/decode/DictEncodingDecoderTest.java b/reader/src/test/java/io/github/dfa1/vortex/reader/decode/DictEncodingDecoderTest.java index dae08734f..09dfd2384 100644 --- a/reader/src/test/java/io/github/dfa1/vortex/reader/decode/DictEncodingDecoderTest.java +++ b/reader/src/test/java/io/github/dfa1/vortex/reader/decode/DictEncodingDecoderTest.java @@ -289,20 +289,16 @@ void valuesBufferShorterThanPool_repeatsTheSingleEntry(PType codePType) { @ParameterizedTest(name = "codes={0}") @EnumSource(value = PType.class, names = {"U8", "U16", "U32"}) - void codesBufferShorterThanRowCount_repeatsTheCodes(PType codePType) { - // Given — 4 rows declared, 2 codes on the wire + void codesBufferShorterThanRowCount_throws(PType codePType) { + // Given — 4 rows declared, 2 codes on the wire: neither every code nor one broadcast + // code. It used to wrap around and repeat the pair; Rust has no such shape. MemorySegment values = TestSegments.leInts(10, 20); MemorySegment codes = codeSegment(codePType, new long[]{1, 0}); - // When - Array result = decodeProtoSegments(DType.I32, codePType, codes, values, 2, 4); - - // Then — the code pair repeats across the row count - IntArray ints = (IntArray) result; - assertThat(ints.getInt(0)).isEqualTo(20); - assertThat(ints.getInt(1)).isEqualTo(10); - assertThat(ints.getInt(2)).isEqualTo(20); - assertThat(ints.getInt(3)).isEqualTo(10); + // When / Then + assertThatThrownBy(() -> decodeProtoSegments(DType.I32, codePType, codes, values, 2, 4)) + .isInstanceOf(VortexException.class) + .hasMessageContaining("child holds 2 element(s) for a declared length of 4"); } } @@ -731,7 +727,7 @@ void emptyValuesPool_throws() { // When / Then assertThatThrownBy(() -> decodeProtoSegments(DType.I32, PType.U8, codes, values, 1, 2)) .isInstanceOf(VortexException.class) - .hasMessageContaining("empty dict child"); + .hasMessageContaining("child holds 0 element(s)"); } @Test @@ -743,7 +739,7 @@ void emptyCodesChild_throws() { // When / Then assertThatThrownBy(() -> decodeProtoSegments(DType.I32, PType.U8, codes, values, 2, 2)) .isInstanceOf(VortexException.class) - .hasMessageContaining("empty dict child"); + .hasMessageContaining("child holds 0 element(s)"); } @Test diff --git a/reader/src/test/java/io/github/dfa1/vortex/reader/decode/FsstEncodingDecoderTest.java b/reader/src/test/java/io/github/dfa1/vortex/reader/decode/FsstEncodingDecoderTest.java index 3dc2018e2..c71f2c313 100644 --- a/reader/src/test/java/io/github/dfa1/vortex/reader/decode/FsstEncodingDecoderTest.java +++ b/reader/src/test/java/io/github/dfa1/vortex/reader/decode/FsstEncodingDecoderTest.java @@ -197,34 +197,27 @@ void forEachByteLength_emptyChildWithNonZeroRows_throwsFromBulkPath() { byte[] compressed = {}; long[] uncompLengths = {}; long[] codeOffsets = {0}; - VarBinArray sut = decodeFsst(1, symbols, symbolLengths, compressed, - PType.U8, uncompLengths, PType.U8, codeOffsets); - - // When / Then + // When / Then — refused when decoded, before any row is read assertThatExceptionOfType(VortexException.class) - .isThrownBy(() -> sut.forEachByteLength(len -> { })) - .withMessageContaining("empty"); + .isThrownBy(() -> decodeFsst(1, symbols, symbolLengths, compressed, + PType.U8, uncompLengths, PType.U8, codeOffsets)) + .withMessageContaining("child holds 0 element(s) for a declared length of 1"); } @Test - void forEachByteLength_broadcastCapacityBetweenOneAndRowCount_readsWithModulo() { - // Given — cap (2) is neither 0 nor >= n (5), so only the per-row modulo path can - // produce the result; existing broadcast fixtures elsewhere use capacity 1 or - // capacity == n, neither of which reaches this branch. + void forEachByteLength_childShorterThanRowCountButNotConstant_throws() { + // Given — 2 lengths for 5 rows: neither a full child nor a constant one. It used to wrap + // around and read 5, 7, 5, 7, 5; Rust has no such shape, so it is malformed input. long[] symbols = {}; byte[] symbolLengths = {}; byte[] compressed = {}; long[] uncompLengths = {5, 7}; long[] codeOffsets = {0}; - VarBinArray sut = decodeFsst(5, symbols, symbolLengths, compressed, - PType.U8, uncompLengths, PType.U8, codeOffsets); - - // When - List lengths = new ArrayList<>(); - sut.forEachByteLength(lengths::add); - - // Then - assertThat(lengths).containsExactly(5, 7, 5, 7, 5); + // When / Then — refused when decoded, before any row is read + assertThatExceptionOfType(VortexException.class) + .isThrownBy(() -> decodeFsst(5, symbols, symbolLengths, compressed, + PType.U8, uncompLengths, PType.U8, codeOffsets)) + .withMessageContaining("child holds 2 element(s) for a declared length of 5"); } @ParameterizedTest(name = "{0}") @@ -260,7 +253,7 @@ void zeroRows_decodesToNothing() { byte[] symbolLengths = {}; byte[] compressed = {}; long[] uncompLengths = {}; - long[] codeOffsets = {}; + long[] codeOffsets = {0}; // n + 1 offsets, as Rust writes them // When VarBinArray result = decodeFsst(0, symbols, symbolLengths, compressed, @@ -463,13 +456,12 @@ void emptyUncompressedLengthsChild_throws() { byte[] compressed = {0x00}; long[] uncompLengths = {}; // empty length child with n == 1 long[] codeOffsets = {0, 1}; - VarBinArray result = decodeFsst(1, symbols, symbolLengths, compressed, - PType.U8, uncompLengths, PType.U8, codeOffsets); - // When / Then — decode() succeeded; reading row 0's length is where this surfaces. + // When / Then — refused when decoded: the child is neither full nor one broadcast value assertThatExceptionOfType(VortexException.class) - .isThrownBy(() -> result.getByteLength(0)) - .withMessageContaining("empty"); + .isThrownBy(() -> decodeFsst(1, symbols, symbolLengths, compressed, + PType.U8, uncompLengths, PType.U8, codeOffsets)) + .withMessageContaining("child holds 0 element(s)"); } @Test @@ -480,13 +472,12 @@ void emptyCodesOffsetsChild_throws() { byte[] compressed = {0x00}; long[] uncompLengths = {2}; long[] codeOffsets = {}; // empty offsets child with n == 1 - VarBinArray result = decodeFsst(1, symbols, symbolLengths, compressed, - PType.U8, uncompLengths, PType.U8, codeOffsets); - // When / Then + // When / Then — refused when decoded: the child is neither full nor one broadcast value assertThatExceptionOfType(VortexException.class) - .isThrownBy(() -> result.getByteLength(0)) - .withMessageContaining("empty"); + .isThrownBy(() -> decodeFsst(1, symbols, symbolLengths, compressed, + PType.U8, uncompLengths, PType.U8, codeOffsets)) + .withMessageContaining("child holds 0 element(s)"); } @Test diff --git a/reader/src/test/java/io/github/dfa1/vortex/reader/decode/PatchedEncodingDecoderTest.java b/reader/src/test/java/io/github/dfa1/vortex/reader/decode/PatchedEncodingDecoderTest.java index bf55e6dc2..4e3ed6095 100644 --- a/reader/src/test/java/io/github/dfa1/vortex/reader/decode/PatchedEncodingDecoderTest.java +++ b/reader/src/test/java/io/github/dfa1/vortex/reader/decode/PatchedEncodingDecoderTest.java @@ -38,7 +38,7 @@ private static Array decode(int n, int[] innerI32, int[] laneOffsets, short[] pa return decode(DType.I32, n, TestSegments.leInts(innerI32), TestSegments.leInts(laneOffsets), TestSegments.leShorts(patchIndices), TestSegments.leInts(patchValues), - laneOffsets.length - 1); + (laneOffsets.length - 1) / Math.max(1, (n + 1023) / 1024)); } private static Array decode(DType dtype, int n, @@ -69,7 +69,8 @@ void decode_noPatches_returnsInnerUnchanged() { int[] inner = {10, 20, 30, 40}; // When - Array result = decode(n, inner, new int[]{0, 0}, new short[]{}, new int[]{}); + // One lane, so one offset per 1024-row chunk plus the end: n_chunks * n_lanes + 1 + Array result = decode(n, inner, new int[(n + 1023) / 1024 + 1], new short[]{}, new int[]{}); // Then assertThat(result).isInstanceOf(IntArray.class); @@ -172,7 +173,8 @@ void decode_variousLengths_noPatches(int n) { int[] inner = new int[n]; // When - Array result = decode(n, inner, new int[]{0, 0}, new short[]{}, new int[]{}); + // One lane, so one offset per 1024-row chunk plus the end: n_chunks * n_lanes + 1 + Array result = decode(n, inner, new int[(n + 1023) / 1024 + 1], new short[]{}, new int[]{}); // Then IntArray ints = (IntArray) result; diff --git a/reader/src/test/java/io/github/dfa1/vortex/reader/decode/RleEncodingDecoderTest.java b/reader/src/test/java/io/github/dfa1/vortex/reader/decode/RleEncodingDecoderTest.java index b8853cb06..93c66e432 100644 --- a/reader/src/test/java/io/github/dfa1/vortex/reader/decode/RleEncodingDecoderTest.java +++ b/reader/src/test/java/io/github/dfa1/vortex/reader/decode/RleEncodingDecoderTest.java @@ -341,6 +341,34 @@ void unsupportedIndicesPtype_isRejected() { } } + @Nested + class EmptyChunk { + + @Test + void boolDtype_decodesEmpty() { + // Given a zero-row bool chunk: RLE covers bools too, but the empty-array shortcut + // cast every dtype to Primitive and threw ClassCastException + MemorySegment empty = MemorySegment.ofArray(new byte[0]); + + // When + Array result = decodeRle(DType.BOOL, 0, 0, empty, empty, empty, 0); + + // Then + assertThat(result.length()).isZero(); + } + + @Test + void nonPrimitiveDtype_isRejected() { + // Given a zero-row chunk of a Utf8 column whose segment claims RLE (a crafted file) + MemorySegment empty = MemorySegment.ofArray(new byte[0]); + + // When / Then + assertThatThrownBy(() -> decodeRle(DType.UTF8, 0, 0, empty, empty, empty, 0)) + .isInstanceOf(VortexException.class) + .hasMessageContaining("expected Primitive dtype"); + } + } + // ── decode harness ───────────────────────────────────────────────────────── /// Builds an RLE node with a `u8` index table: child[0] = values, child[1] = indices, @@ -355,7 +383,11 @@ private static Array decodeRle(DType dtype, long valuesLen, long rowCount, private static Array decodeRle(DType dtype, long valuesLen, long rowCount, MemorySegment indices, ProtoPType indicesPtype, MemorySegment offsets, MemorySegment values, int offset) { - long indicesLen = indices.byteSize() / (indicesPtype == ProtoPType.U8 ? 1 : 2); + long indicesLen = indices.byteSize() / switch (indicesPtype) { + case U8 -> 1; + case U32 -> 4; + default -> 2; + }; long offsetsLen = offsets.byteSize() / 8; // U64 offsets byte[] metaBytes = new ProtoRLEMetadata( valuesLen, indicesLen, indicesPtype, offsetsLen, ProtoPType.U64, offset).encode(); diff --git a/reader/src/test/java/io/github/dfa1/vortex/reader/decode/RunEndEncodingDecoderTest.java b/reader/src/test/java/io/github/dfa1/vortex/reader/decode/RunEndEncodingDecoderTest.java index d0974ca87..d4c56e4d3 100644 --- a/reader/src/test/java/io/github/dfa1/vortex/reader/decode/RunEndEncodingDecoderTest.java +++ b/reader/src/test/java/io/github/dfa1/vortex/reader/decode/RunEndEncodingDecoderTest.java @@ -243,6 +243,20 @@ void utf8WithNonVarBinValuesChild_throwsVortexException() { .hasMessageContaining("expected a VarBin values child"); } + @Test + void zeroRuns_withZeroRows_decodesEmpty() { + // Given — an empty run-end array, which Rust writes and reads; its empty ends buffer was + // refused as "empty ends buffer for 0 run(s)" + MemorySegment empty = MemorySegment.ofArray(new byte[0]); + MemorySegment[] segs = {empty, empty}; + + // When + Array result = decode(DType.I32, PType.U8, 0, 0, 0, segs, primitiveNode(0), primitiveNode(1)); + + // Then + assertThat(result.length()).isZero(); + } + private static Array decode(DType dtype, PType endsPtype, long numRuns, long offset, long n, MemorySegment[] segs, ArrayNode endsNode, ArrayNode valuesNode) { MemorySegment meta = MemorySegment.ofArray( diff --git a/reader/src/test/java/io/github/dfa1/vortex/reader/decode/SegmentBroadcastTest.java b/reader/src/test/java/io/github/dfa1/vortex/reader/decode/SegmentBroadcastTest.java index 7080baeab..adf48f370 100644 --- a/reader/src/test/java/io/github/dfa1/vortex/reader/decode/SegmentBroadcastTest.java +++ b/reader/src/test/java/io/github/dfa1/vortex/reader/decode/SegmentBroadcastTest.java @@ -1,18 +1,50 @@ package io.github.dfa1.vortex.reader.decode; +import io.github.dfa1.vortex.core.error.VortexException; import io.github.dfa1.vortex.core.io.VortexFormat; import org.junit.jupiter.api.Test; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.CsvSource; import java.lang.foreign.Arena; import java.lang.foreign.MemorySegment; import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; /// Unit tests for [SegmentBroadcast]. These verify the wrap-on-broadcast /// semantics that protect downstream encoders/decoders from IOOB when a child /// segment was produced by [ConstantEncoding] (single-element buffer). class SegmentBroadcastTest { + @ParameterizedTest(name = "{0} elements for {1} rows") + @CsvSource({"4, 4", "5, 4", "1, 1000", "0, 0"}) + void capacity_fullOrConstantChild_isAccepted(long elements, long count) { + // Given — a full child, a longer one, a constant child broadcast, and an empty child of an + // empty array: the shapes a decoder may legally meet + MemorySegment seg = MemorySegment.ofArray(new long[(int) elements]); + + // When + long result = SegmentBroadcast.capacity(seg, 8, count); + + // Then + assertThat(result).isEqualTo(elements); + } + + @ParameterizedTest(name = "{0} elements for {1} rows") + @CsvSource({"0, 1", "0, 1000", "2, 3", "999, 1000"}) + void capacity_emptyOrTruncatedChild_throws(long elements, long count) { + // Given — an empty child of a non-empty array made every caller's `i % capacity` broadcast + // divide by zero (a fuzzer finding in the dict decoder); a truncated one silently repeated + // its values + MemorySegment seg = MemorySegment.ofArray(new long[(int) elements]); + + // When / Then + assertThatThrownBy(() -> SegmentBroadcast.capacity(seg, 8, count)) + .isInstanceOf(VortexException.class) + .hasMessageContaining("child holds " + elements + " element(s) for a declared length of " + count); + } + @Test void elementOffset_wrapsConstantSegment() { try (Arena arena = Arena.ofConfined()) { @@ -24,7 +56,7 @@ void elementOffset_wrapsConstantSegment() { assertThat(SegmentBroadcast.elementOffset(seg, 0, 8)).isZero(); assertThat(SegmentBroadcast.elementOffset(seg, 5, 8)).isZero(); assertThat(SegmentBroadcast.elementOffset(seg, 999_999, 8)).isZero(); - assertThat(SegmentBroadcast.capacity(seg, 8)).isEqualTo(1L); + assertThat(SegmentBroadcast.capacity(seg, 8, 1_000_000)).isEqualTo(1L); } } @@ -39,7 +71,7 @@ void elementOffset_passesThroughFullSegment() { assertThat(SegmentBroadcast.elementOffset(seg, i, 8)).isEqualTo(i * 8); } assertThat(SegmentBroadcast.elementOffset(seg, 4, 8)).isZero(); - assertThat(SegmentBroadcast.capacity(seg, 8)).isEqualTo(4L); + assertThat(SegmentBroadcast.capacity(seg, 8, 4)).isEqualTo(4L); } } diff --git a/reader/src/test/java/io/github/dfa1/vortex/reader/decode/SparseEncodingDecoderTest.java b/reader/src/test/java/io/github/dfa1/vortex/reader/decode/SparseEncodingDecoderTest.java index 07309aa45..742bfc0e7 100644 --- a/reader/src/test/java/io/github/dfa1/vortex/reader/decode/SparseEncodingDecoderTest.java +++ b/reader/src/test/java/io/github/dfa1/vortex/reader/decode/SparseEncodingDecoderTest.java @@ -368,7 +368,8 @@ void missingPatchesMetadata_throws() { /// error and used to blow up at the unchecked `(LongArray)` cast. @Test void patchValuesChildWrongType_throws() { - // Given — an i64 sparse array whose values child is `vortex.bool`, not primitive i64 + // Given — an i64 sparse array whose values child is `vortex.bool`, not primitive i64: its + // one-byte bitmap holds no i64, so it is refused as soon as the child is built ArrayNode boolValuesNode = new ArrayNode(EncodingId.VORTEX_BOOL, null, new ArrayNode[0], new int[]{2}); MemorySegment[] segs = {f64Fill(0.0), TestSegments.leInts(1), boolBitmap(true)}; ArrayNode indicesNode = primitiveNode(1); @@ -377,7 +378,7 @@ void patchValuesChildWrongType_throws() { assertThatThrownBy(() -> decode(DType.I64, 1, 0, PType.U32, 5, segs, indicesNode, boolValuesNode)) .isInstanceOf(VortexException.class) - .hasMessageContaining("patch values child decoded to unexpected type"); + .hasMessageContaining("child holds 0 element(s)"); } /// `ArithmeticException: / by zero` — the `Materialized*` accessors broadcast an @@ -394,7 +395,7 @@ void emptyPatchIndicesChild_throws() { assertThatThrownBy(() -> decode(DType.F64, 2, 0, PType.U32, 5, segs, indicesNode, valuesNode)) .isInstanceOf(VortexException.class) - .hasMessageContaining("empty patch indices child"); + .hasMessageContaining("child holds 0 element(s)"); } /// Same divide-by-zero, reached through the values child instead of the indices child. @@ -409,7 +410,7 @@ void emptyPatchValuesChild_throws() { assertThatThrownBy(() -> decode(DType.F64, 2, 0, PType.U32, 5, segs, indicesNode, valuesNode)) .isInstanceOf(VortexException.class) - .hasMessageContaining("empty patch values child"); + .hasMessageContaining("child holds 0 element(s)"); } /// `OutOfMemoryError` — a null fill makes the decoder build a per-patch validity bitmap @@ -460,7 +461,7 @@ void negativePatchCount_throws() { /// child broadcasts instead of failing; the contract this test can hold the sparse /// decoder to is ADR 0003's: no raw JDK exception, no read outside the value buffer. @Test - void varBinPatchCountBeyondValueOffsets_broadcastsWithoutRawException() { + void varBinPatchCountBeyondValueOffsets_throws() { // Given — 4 declared patches but only 3 offsets, covering 2 values ("b", "d") MemorySegment[] segs = { nullFill(), @@ -469,12 +470,12 @@ void varBinPatchCountBeyondValueOffsets_broadcastsWithoutRawException() { TestSegments.leInts(0, 1, 2) // only 3 offsets = 2 values }; - // When — row 3 resolves to patch 3, past what the offsets child covers - VarBinArray result = assertVarBin(decode(nullableUtf8(), 4, 0, PType.U32, 5, segs, - primitiveNode(1), varBinNode(2, 3))); - - // Then — the broadcast wraps to offsets [0, 1), still inside the 2-byte payload - assertThat(result.getString(3)).isEqualTo("b"); + // When / Then — 3 offsets for 4 values is neither a full nor a constant child: it used + // to wrap around and read patch 3 as "b" + assertThatThrownBy(() -> decode(nullableUtf8(), 4, 0, PType.U32, 5, segs, + primitiveNode(1), varBinNode(2, 3))) + .isInstanceOf(VortexException.class) + .hasMessageContaining("child holds 3 element(s) for a declared length of 5"); } /// `IndexOutOfBoundsException` from `MemorySegment.copy` under the eager merge — diff --git a/reader/src/test/java/io/github/dfa1/vortex/reader/decode/ZstdBuffersEncodingDecoderTest.java b/reader/src/test/java/io/github/dfa1/vortex/reader/decode/ZstdBuffersEncodingDecoderTest.java index 2926919f4..400056e8a 100644 --- a/reader/src/test/java/io/github/dfa1/vortex/reader/decode/ZstdBuffersEncodingDecoderTest.java +++ b/reader/src/test/java/io/github/dfa1/vortex/reader/decode/ZstdBuffersEncodingDecoderTest.java @@ -90,6 +90,18 @@ void missingMetadata_throws() { .hasMessageContaining("missing metadata"); } + @Test + void innerEncodingIdWithControlCharacter_throws() { + // Given — an inner id EncodingId.parse refuses (a fuzzer finding: it escaped as a raw + // IllegalArgumentException) + ArrayNode node = zstdBuffers(meta("vortex.\u000Fprimitive", VALUES.length, 4), 0); + + // When / Then + assertThatThrownBy(() -> SUT.decode(ctx(node, compress(VALUES)))) + .isInstanceOf(VortexException.class) + .hasMessageContaining("invalid inner encoding id"); + } + @Test void sizeCountDisagreesWithBuffers_throws() { // Given — one compressed buffer, no declared sizes diff --git a/writer/src/main/java/io/github/dfa1/vortex/writer/VortexWriter.java b/writer/src/main/java/io/github/dfa1/vortex/writer/VortexWriter.java index 6a6b2255a..a7e755efe 100644 --- a/writer/src/main/java/io/github/dfa1/vortex/writer/VortexWriter.java +++ b/writer/src/main/java/io/github/dfa1/vortex/writer/VortexWriter.java @@ -196,6 +196,12 @@ private VortexWriter( WritableByteChannel channel, DType.Struct schema, WriteOptions options, List encodings, WriteRegistry extensionRegistry ) { + // A nullable struct layout carries a leading validity child (Rust's validate_children: + // nfields + 1 children); the root layout written here has one child per column and no way + // to mark a row null, so Rust refused every such file ("nchildren == expected"). + if (schema.nullable()) { + throw new IllegalArgumentException("the root struct cannot be nullable: rows cannot be null"); + } // Wire contract, enforced by the reference writer ("StructLayout must have unique field // names"): duplicate-name schemas are constructible via the DType.Struct record (only // StructBuilder validates), so guard here — colChunks below is name-keyed and would diff --git a/writer/src/main/java/io/github/dfa1/vortex/writer/encode/ConstantEncodingEncoder.java b/writer/src/main/java/io/github/dfa1/vortex/writer/encode/ConstantEncodingEncoder.java index d6e841380..d642226ee 100644 --- a/writer/src/main/java/io/github/dfa1/vortex/writer/encode/ConstantEncodingEncoder.java +++ b/writer/src/main/java/io/github/dfa1/vortex/writer/encode/ConstantEncodingEncoder.java @@ -137,11 +137,16 @@ private static OptionalLong constantBits(Object data, PType ptype) { if (!SimdOperationsSupport.preferred().allEqual(data, ptype)) { return OptionalLong.empty(); } - // No default: every PType is handled, so a new one fails to compile here instead of at write time + // No default: every PType is handled, so a new one fails to compile here instead of at write + // time. Unsigned values widen unsigned: a sign-extended u8 230 became the u64 scalar + // 2^64 - 26, which Rust refuses to read ("out of range for dtype u8"). return OptionalLong.of(switch (ptype) { - case I8, U8 -> ((byte[]) data)[0]; - case I16, U16, F16 -> ((short[]) data)[0]; - case I32, U32 -> ((int[]) data)[0]; + case I8 -> ((byte[]) data)[0]; + case U8 -> Byte.toUnsignedLong(((byte[]) data)[0]); + case I16, F16 -> ((short[]) data)[0]; + case U16 -> Short.toUnsignedLong(((short[]) data)[0]); + case I32 -> ((int[]) data)[0]; + case U32 -> Integer.toUnsignedLong(((int[]) data)[0]); case I64, U64 -> ((long[]) data)[0]; case F32 -> Float.floatToRawIntBits(((float[]) data)[0]); case F64 -> Double.doubleToRawLongBits(((double[]) data)[0]); diff --git a/writer/src/main/java/io/github/dfa1/vortex/writer/encode/SequenceEncodingEncoder.java b/writer/src/main/java/io/github/dfa1/vortex/writer/encode/SequenceEncodingEncoder.java index e9b7131ad..988293568 100644 --- a/writer/src/main/java/io/github/dfa1/vortex/writer/encode/SequenceEncodingEncoder.java +++ b/writer/src/main/java/io/github/dfa1/vortex/writer/encode/SequenceEncodingEncoder.java @@ -1,6 +1,8 @@ package io.github.dfa1.vortex.writer.encode; import io.github.dfa1.vortex.core.model.DType; +import io.github.dfa1.vortex.core.simd.SimdOperations; +import io.github.dfa1.vortex.core.simd.SimdOperationsSupport; import io.github.dfa1.vortex.core.model.PType; import io.github.dfa1.vortex.core.error.VortexException; import io.github.dfa1.vortex.core.model.EncodingId; @@ -14,6 +16,10 @@ /// Write-only encoder for `vortex.sequence` — arithmetic sequences as (base, multiplier). public final class SequenceEncodingEncoder implements EncodingEncoder { + /// Elements widened per step of [#encodeInteger(PType, Object)]'s scan. + private static final int WIDEN_BLOCK = 1024; + + @Override public EncodingId encodingId() { return EncodingId.VORTEX_SEQUENCE; @@ -67,34 +73,103 @@ public EncodeResult encode(DType dtype, Object data, EncodeContext ctx) { return encodeInteger(((DType.Primitive) dtype).ptype(), data); } + /// Rust's `encode_primitive_array`: `base` is the first value and the multiplier + /// `slice[1].checked_sub(base)` in the column's own type, so a step that does not fit it (a + /// decreasing unsigned column, an `i8` step of 255) is no sequence; nor is an empty array or a + /// zero step (constant data, left to the constant encoding); every later step is a + /// `checked_add` in that type too. Values widen zero-extended for unsigned types: a + /// sign-extended `u16` became a `u64` scalar Rust refuses ("out of range for dtype u16"). private static EncodeResult encodeInteger(PType pt, Object data) { int n = Array.getLength(data); + if (n == 0) { + throw new VortexException(EncodingId.VORTEX_SEQUENCE, "an empty array is not a sequence"); + } + // Widened a block at a time: the cascade tries this on every integer column, and most fail + // by their third value, so neither a column-sized copy nor a full scan is spent on them. + long[] block = new long[Math.min(n, WIDEN_BLOCK)]; + SimdOperations ops = SimdOperationsSupport.preferred(); long base = 0; long multiplier = 0; - if (n > 0) { - base = readLong(pt, data, 0); - multiplier = n > 1 ? readLong(pt, data, 1) - base : 0; - for (int i = 2; i < n; i++) { - long expected = base + i * multiplier; - if (readLong(pt, data, i) != expected) { + long previous = 0; + for (int from = 0; from < n; from += block.length) { + int count = Math.min(block.length, n - from); + ops.widenArrayInto(data, from, count, pt, block); + for (int j = 0; j < count; j++) { + int i = from + j; + long value = block[j]; + if (i == 0) { + base = value; + } else if (i == 1) { + multiplier = checkedSub(pt, value, base); + if (multiplier == 0) { + throw new VortexException(EncodingId.VORTEX_SEQUENCE, "a zero step is not a sequence"); + } + } else if (value != checkedAdd(pt, previous, multiplier)) { throw new VortexException(EncodingId.VORTEX_SEQUENCE, "not an arithmetic sequence at index " + i); } + previous = value; } } - ProtoScalarValue baseScalar = buildIntScalar(pt, base); - ProtoScalarValue mulScalar = buildIntScalar(pt, multiplier); - // A perfect arithmetic sequence is monotonic (or constant) end to end, so its extremes are - // always the first and last element -- no separate scan needed. - boolean unsign = pt.isUnsigned(); - ProtoScalarValue statsMin = null; - ProtoScalarValue statsMax = null; - if (n > 0) { - long last = base + (n - 1) * multiplier; - boolean baseIsMin = unsign ? Long.compareUnsigned(base, last) <= 0 : base <= last; - statsMin = buildIntScalar(pt, baseIsMin ? base : last); - statsMax = buildIntScalar(pt, baseIsMin ? last : base); + long last = previous; + boolean baseIsMin = pt.isUnsigned() ? Long.compareUnsigned(base, last) <= 0 : base <= last; + return buildResult(buildIntScalar(pt, base), buildIntScalar(pt, multiplier), + buildIntScalar(pt, baseIsMin ? base : last), buildIntScalar(pt, baseIsMin ? last : base)); + } + + /// `a - b` in `pt`, or not applicable when the result does not fit `pt`. + private static long checkedSub(PType pt, long a, long b) { + return switch (pt) { + case U64 -> { + if (Long.compareUnsigned(a, b) < 0) { + throw notInType(pt); + } + yield a - b; + } + case I64 -> { + try { + yield Math.subtractExact(a, b); + } catch (ArithmeticException _) { + throw notInType(pt); + } + } + default -> fitting(pt, a - b); + }; + } + + /// `a + b` in `pt`, or not applicable when the result does not fit `pt`. + private static long checkedAdd(PType pt, long a, long b) { + return switch (pt) { + case U64 -> { + long sum = a + b; + if (Long.compareUnsigned(sum, a) < 0) { + throw notInType(pt); + } + yield sum; + } + case I64 -> { + try { + yield Math.addExact(a, b); + } catch (ArithmeticException _) { + throw notInType(pt); + } + } + default -> fitting(pt, a + b); + }; + } + + /// `value` (exact in a `long` for widths below 64 bits) if it lies in `pt`'s range. + private static long fitting(PType pt, long value) { + long bits = pt.bits(); + long min = pt.isUnsigned() ? 0 : -(1L << (bits - 1)); + long max = pt.isUnsigned() ? (1L << bits) - 1 : (1L << (bits - 1)) - 1; + if (value < min || value > max) { + throw notInType(pt); } - return buildResult(baseScalar, mulScalar, statsMin, statsMax); + return value; + } + + private static VortexException notInType(PType pt) { + return new VortexException(EncodingId.VORTEX_SEQUENCE, "sequence step overflows " + pt); } private static EncodeResult buildResult(ProtoScalarValue base, ProtoScalarValue mul, ProtoScalarValue statsMin, ProtoScalarValue statsMax) { @@ -111,13 +186,4 @@ private static ProtoScalarValue buildIntScalar(PType pt, long value) { }; } - private static long readLong(PType pt, Object data, int i) { - return switch (pt) { - case I8, U8 -> ((byte[]) data)[i]; - case I16, U16 -> ((short[]) data)[i]; - case I32, U32 -> ((int[]) data)[i]; - case I64, U64 -> ((long[]) data)[i]; - default -> throw new VortexException(EncodingId.VORTEX_SEQUENCE, "unsupported ptype: " + pt); - }; - } } diff --git a/writer/src/test/java/io/github/dfa1/vortex/writer/VortexWriterTest.java b/writer/src/test/java/io/github/dfa1/vortex/writer/VortexWriterTest.java index 2c24b0759..6da4455a5 100644 --- a/writer/src/test/java/io/github/dfa1/vortex/writer/VortexWriterTest.java +++ b/writer/src/test/java/io/github/dfa1/vortex/writer/VortexWriterTest.java @@ -111,6 +111,23 @@ void create_duplicateFieldNames_throwsIllegalArgumentException(@TempDir Path tmp } } + @Test + void create_nullableRootStruct_isRejected(@TempDir Path tmp) throws IOException { + // Given — a nullable root: Rust reads a nullable struct layout only with a leading validity + // child, which this writer never writes, so vortex-jni refused every such file + // ("nchildren == expected") and the reader now refuses it at open, as Rust does + var schema = new DType.Struct(List.of(ColumnName.of("c")), List.of(new DType.Primitive(PType.I64, false)), true); + Path file = tmp.resolve("nullable-root.vtx"); + + // When / Then + WriteOptions opts = WriteOptions.defaults(); + try (var ch = FileChannel.open(file, StandardOpenOption.CREATE, StandardOpenOption.WRITE)) { + assertThatThrownBy(() -> VortexWriter.create(ch, schema, opts)) + .isInstanceOf(IllegalArgumentException.class) + .hasMessageContaining("root struct cannot be nullable"); + } + } + @Test void schema_fieldNameWithNulByte_isUnbuildable() { // Given a NUL byte inside a field name: legal in our pure-Java stack, but it aborts the diff --git a/writer/src/test/java/io/github/dfa1/vortex/writer/encode/ConstantEncodingEncoderTest.java b/writer/src/test/java/io/github/dfa1/vortex/writer/encode/ConstantEncodingEncoderTest.java index 92fbe891c..1772e5c05 100644 --- a/writer/src/test/java/io/github/dfa1/vortex/writer/encode/ConstantEncodingEncoderTest.java +++ b/writer/src/test/java/io/github/dfa1/vortex/writer/encode/ConstantEncodingEncoderTest.java @@ -1,6 +1,7 @@ package io.github.dfa1.vortex.writer.encode; import io.github.dfa1.vortex.core.model.DType; +import io.github.dfa1.vortex.core.model.PType; import io.github.dfa1.vortex.core.model.EncodingId; import io.github.dfa1.vortex.core.error.VortexException; import io.github.dfa1.vortex.core.proto.ProtoNullValue; @@ -179,6 +180,23 @@ void encode_f16_writesRustsF16ScalarField() throws java.io.IOException { assertThat(scalar.f16_value()).isEqualTo(0x3E00L); } + @org.junit.jupiter.params.ParameterizedTest(name = "{0}") + @org.junit.jupiter.params.provider.MethodSource("io.github.dfa1.vortex.writer.encode.ConstantEncodingEncoderTest#highUnsignedConstants") + void encode_unsignedAboveSignedRange_writesZeroExtendedScalar(PType ptype, Object data, long expected) + throws java.io.IOException { + // Given — a constant past the signed range of its width: it was sign-extended, so a u8 230 + // became the u64 scalar 2^64 - 26, which vortex-jni refuses ("out of range for dtype u8"; + // found by the differential fuzz target) + + // When + EncodeResult result = ENCODER.encode(new DType.Primitive(ptype, false), data, EncodeTestHelper.testCtx()); + + // Then + MemorySegment seg = result.encodedBuffers().getFirst().data(); + ProtoScalarValue scalar = ProtoScalarValue.decode(seg, 0, seg.byteSize()); + assertThat(scalar.uint64_value()).isEqualTo(expected); + } + @Test void decode_f16_acceptsTheLegacyUint64Scalar() { // Given — older Rust writers stored an f16 as uint64_value (its bits) @@ -475,4 +493,11 @@ void decode_nullScalar_returnsNullArray(DType dtype) { assertThat(result.length()).isEqualTo(rowCount); } } + + static Stream highUnsignedConstants() { + return Stream.of( + Arguments.of(PType.U8, new byte[]{(byte) 230, (byte) 230}, 230L), + Arguments.of(PType.U16, new short[]{(short) 0xFFF0, (short) 0xFFF0}, 0xFFF0L), + Arguments.of(PType.U32, new int[]{0xFFFF_FFF0, 0xFFFF_FFF0}, 0xFFFF_FFF0L)); + } } diff --git a/writer/src/test/java/io/github/dfa1/vortex/writer/encode/SequenceEncodingEncoderTest.java b/writer/src/test/java/io/github/dfa1/vortex/writer/encode/SequenceEncodingEncoderTest.java index 9acce2a66..3800078e0 100644 --- a/writer/src/test/java/io/github/dfa1/vortex/writer/encode/SequenceEncodingEncoderTest.java +++ b/writer/src/test/java/io/github/dfa1/vortex/writer/encode/SequenceEncodingEncoderTest.java @@ -174,17 +174,50 @@ void encode_u16_roundTrips() { } @Test - void encode_emptyArray_roundTripsToZeroLength() { - // Given the n==0 branch: base and multiplier default to 0 + void encode_emptyArray_throws() { + // Given — Rust's sequence_encode: "we cannot encode an empty array" long[] data = {}; + // When / Then + assertThatThrownBy(() -> ENCODER.encode(DTypes.I64, data, EncodeTestHelper.testCtx())) + .isInstanceOf(VortexException.class) + .hasMessageContaining("empty array is not a sequence"); + } + + @org.junit.jupiter.params.ParameterizedTest(name = "{0} {1}") + @org.junit.jupiter.params.provider.MethodSource("notSequencesInRust") + void encode_stepRustRejects_throws(PType ptype, Object data, String reason) { + // Given — data whose step Rust's checked arithmetic in the column type refuses: these + // used to be written, and a sign-extended or out-of-range scalar made the file + // unreadable for Rust (found by the differential fuzz target) + + // When / Then + assertThatThrownBy(() -> ENCODER.encode(new DType.Primitive(ptype, false), data, EncodeTestHelper.testCtx())) + .isInstanceOf(VortexException.class) + .hasMessageContaining(reason); + } + + static Stream notSequencesInRust() { + return Stream.of( + Arguments.of(PType.I64, new long[]{5, 5, 5}, "zero step"), + Arguments.of(PType.U8, new byte[]{3, 2, 1}, "overflows U8"), + Arguments.of(PType.I8, new byte[]{-128, 127}, "overflows I8"), + Arguments.of(PType.I64, new long[]{Long.MIN_VALUE, Long.MAX_VALUE}, "overflows I64"), + Arguments.of(PType.U64, new long[]{-1L, 0L}, "overflows U64")); + } + + @Test + void encode_unsignedAboveSignedRange_writesZeroExtendedScalars() { + // Given — a sorted u16 sequence past 0x7FFF: its base was sign-extended into a u64 scalar + // of 2^64 - 2067, which vortex-jni refuses ("out of range for dtype u16") + short[] data = {(short) 63_469, (short) 63_470, (short) 63_471}; + // When - EncodeResult resultEncoded = ENCODER.encode(DTypes.I64, data, EncodeTestHelper.testCtx()); - DecodeContext ctx = encodeResultToCtx(resultEncoded, DTypes.I64, 0); - Array result = DECODER.decode(ctx); + EncodeResult result = ENCODER.encode(DTypes.U16, data, EncodeTestHelper.testCtx()); // Then - assertThat(result.length()).isZero(); + assertThat(result.statsMin().uint64_value()).isEqualTo(63_469L); + assertThat(result.statsMax().uint64_value()).isEqualTo(63_471L); } @Test @@ -427,19 +460,6 @@ void encode_i64_decreasing_reportsLastAsMinFirstAsMax() throws java.io.IOExcepti assertThat(result.statsMax().int64_value()).isEqualTo(40L); } - @Test - void encode_empty_statsAreNull() { - // Given - long[] data = {}; - - // When - EncodeResult result = ENCODER.encode(DTypes.I64, data, EncodeTestHelper.testCtx()); - - // Then - assertThat(result.statsMin()).isNull(); - assertThat(result.statsMax()).isNull(); - } - private static ProtoScalarValue scalar(byte[] bytes) throws java.io.IOException { MemorySegment seg = MemorySegment.ofArray(bytes); return ProtoScalarValue.decode(seg, 0, seg.byteSize());