diff --git a/documentation/components/libs/parquet.md b/documentation/components/libs/parquet.md index 49b89484c2..55dbd7e508 100644 --- a/documentation/components/libs/parquet.md +++ b/documentation/components/libs/parquet.md @@ -391,8 +391,11 @@ $writer->close(); ### Writer Options - `COLUMN_INDEX_TRUNCATE_LENGTH` - default: `64` - bytes a string/binary page's min/max in the column index are cut to. +- `DICTIONARY_ENABLED` - default: `true` - columns without a `COLUMNS_ENCODINGS` entry are written `RLE_DICTIONARY`, + except BOOLEAN and, under `WRITER_VERSION` 1, FIXED_LEN_BYTE_ARRAY (as arrow-rs and parquet-java); `false` writes the + fallback encoding instead. - `DICTIONARY_PAGE_SIZE` - default: `1Mb` - maximum size of dictionary page; the pages of a column chunk after it is - reached are written `PLAIN`. + reached use the fallback encoding. - `GZIP_COMPRESSION_LEVEL` - default: `9` - compression level for GZIP compression (applied only when GZIP compression is enabled). - `COLUMNS_BLOOM_FILTERS` - default: `null` - `BloomFilterOptions` by column flat path: those columns get a bloom filter @@ -409,7 +412,9 @@ $writer->close(); column index); the offset index is written at every level. - `STATISTICS_TRUNCATE_LENGTH` - default: `64` - bytes a string/binary column chunk's min/max statistics are cut to. - `VALIDATE_DATA` - default: `true` - if set to `true` then writer will validate data against schema. -- `WRITER_VERSION` - default `1` - tells writer which version of parquet format should be used. +- `WRITER_VERSION` - default `1` - `2` writes data page v2. It also picks the fallback encoding used without a + dictionary: `PLAIN` under `1`; under `2` `RLE` (BOOLEAN), `DELTA_BINARY_PACKED` (INT32/INT64), `DELTA_BYTE_ARRAY` + (BYTE_ARRAY/FIXED_LEN_BYTE_ARRAY), `PLAIN` otherwise, as arrow-rs and parquet-java fall back. Two most important options that can heavily affect memory usage are: @@ -715,9 +720,12 @@ You can specify custom encodings for individual columns using flat path notation ### Available Encodings +Without a `COLUMNS_ENCODINGS` entry both engines write `RLE_DICTIONARY` (see `DICTIONARY_ENABLED`), and the +`WRITER_VERSION` fallback encoding past `DICTIONARY_PAGE_SIZE`. + #### PLAIN -The default encoding that stores values as-is without any compression scheme. +Stores values as-is without any compression scheme. **When to use:** @@ -748,7 +756,7 @@ Run Length Encoding with Dictionary compression. Values are stored in a dictiona - Enumeration-like data - Significant file size reduction (often 50-90% smaller) -**Supported types:** All types except `FIXED_LEN_BYTE_ARRAY` +**Supported types:** All types except `BOOLEAN`; `FIXED_LEN_BYTE_ARRAY` only under `WRITER_VERSION` 2 ```php $options = Options::default()->set(Option::COLUMNS_ENCODINGS, [ @@ -758,6 +766,8 @@ $options = Options::default()->set(Option::COLUMNS_ENCODINGS, [ ]); ``` +Past `DICTIONARY_PAGE_SIZE`, the rest of the chunk uses the `WRITER_VERSION` fallback encoding (PLAIN under v1). + #### DELTA_BINARY_PACKED Delta encoding with binary packing for integer columns. Stores differences between consecutive values. @@ -778,8 +788,35 @@ $options = Options::default()->set(Option::COLUMNS_ENCODINGS, [ ]); ``` +#### DELTA_LENGTH_BYTE_ARRAY + +**Supported types:** Only `BYTE_ARRAY` + +```php +$options = Options::default()->set(Option::COLUMNS_ENCODINGS, [ + 'comment' => Encodings::DELTA_LENGTH_BYTE_ARRAY, +]); +``` + +**When to use:** many distinct strings. + +#### DELTA_BYTE_ARRAY + +**Supported types:** `BYTE_ARRAY` and `FIXED_LEN_BYTE_ARRAY` + +```php +$options = Options::default()->set(Option::COLUMNS_ENCODINGS, [ + 'url' => Encodings::DELTA_BYTE_ARRAY, +]); +``` + +**When to use:** sorted strings sharing prefixes. + ### Using Custom Encodings +Both engines write exactly the encoding given and refuse one the column type cannot take; only `RLE_DICTIONARY` adds a +dictionary. + #### Basic Column Encoding ```php @@ -914,11 +951,14 @@ $options = Options::default()->set(Option::COLUMNS_ENCODINGS, [ ### Encoding Compatibility -| Encoding | INT32/INT64 | BYTE_ARRAY | BOOLEAN | FLOAT/DOUBLE | FIXED_LEN_BYTE_ARRAY | -|---------------------|-------------|------------|---------|--------------|----------------------| -| PLAIN | yes | yes | yes | yes | yes | -| RLE_DICTIONARY | yes | yes | yes | yes | no | -| DELTA_BINARY_PACKED | yes | no | no | no | no | +| Encoding | INT32/INT64 | BYTE_ARRAY | BOOLEAN | FLOAT/DOUBLE | FIXED_LEN_BYTE_ARRAY | +|-------------------------|-------------|------------|---------|--------------|----------------------| +| PLAIN | yes | yes | yes | yes | yes | +| RLE | no | no | yes | no | no | +| RLE_DICTIONARY | yes | yes | no | yes | v2 only | +| DELTA_BINARY_PACKED | yes | no | no | no | no | +| DELTA_LENGTH_BYTE_ARRAY | no | yes | no | no | no | +| DELTA_BYTE_ARRAY | no | yes | no | no | yes | ### Performance Guidelines diff --git a/documentation/upgrading.md b/documentation/upgrading.md index 760acee843..4d1ae933fe 100644 --- a/documentation/upgrading.md +++ b/documentation/upgrading.md @@ -147,6 +147,31 @@ public function scan(FlowContext $context, FileScan $scan): Generator | `from_parquet(...)->withSchema()`, `from_floe(...)->withSchema()` threw `InvalidArgumentException` | read every file by name against the declared schema: undeclared columns are not read, a missing nullable column is null | | `from_cache($id)` without a cache instance described an empty schema | `SchemaNotDerivableException`; the plan runs unbound | +### 10) `flow-php/parquet` - the PHP engine reads and writes DELTA_LENGTH_BYTE_ARRAY / DELTA_BYTE_ARRAY; both engines write the same default encodings (dictionary on, arrow-rs fallbacks); COLUMNS_ENCODINGS is literal on both engines + +| Before | After | +|-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| +| `WRITER_VERSION 2` fallback encoding of BYTE_ARRAY / FIXED_LEN_BYTE_ARRAY: PLAIN | `DELTA_BYTE_ARRAY`, as arrow-rs and parquet-java | +| dictionary overflow past `DICTIONARY_PAGE_SIZE`: the rest of the chunk PLAIN | the `WRITER_VERSION` fallback: under v2 `DELTA_BINARY_PACKED` (INT32/INT64), `DELTA_BYTE_ARRAY` (BYTE_ARRAY/FLBA); PLAIN under v1 | +| arrow writer: a non-dictionary `COLUMNS_ENCODINGS` entry kept the column's dictionary, used only once it fell back | turns the column's dictionary off: the entry is the column's encoding, as the PHP writer writes it | +| `DeltaBinaryPackedDecoder::decode(string $data, int $valueCount)` | `decode(BinaryBufferReader $reader, int $valueCount)`, leaving the reader at the end of its stream | +| `new PlainFlatColumnChunkBuilder($column, $options, $compression, $statistics)`; `DeltaBinaryPackedColumnChunkBuilder` | `new FlatColumnChunkBuilder($column, $options, $compression, $statistics, ValueStorage $valueStorage)`; `DeltaBinaryPackedColumnChunkBuilder` removed | +| `ValueStorage` without an encoding | `ValueStorage::encoding(): Encodings`; `ValueStorage::size()` renamed to `plainSize()`, the PLAIN-encoded byte size | +| `new RLEDictionaryChunkBuilder($column, $options, $compression, $statistics)` | `new RLEDictionaryChunkBuilder($column, $options, $compression, $statistics, ValueStorage $fallback)` | +| a failed `Writer::write()` / `Writer::open()` left a 4-byte `PAR1` file behind | the file is removed; the exception is rethrown | +| PHP writer: columns without `COLUMNS_ENCODINGS` written without a dictionary | `RLE_DICTIONARY` (except BOOLEAN, and FIXED_LEN_BYTE_ARRAY under v1), as the arrow writer, arrow-rs and parquet-java; `Option::DICTIONARY_ENABLED` `false` restores the fallback encodings | +| - | `Option::DICTIONARY_ENABLED` (default `true`), on both engines | +| PHP writer `WRITER_VERSION 2`: BOOLEAN written PLAIN | `RLE`, as the arrow writer | +| `RLE_DICTIONARY` on a FIXED_LEN_BYTE_ARRAY column threw | allowed under `WRITER_VERSION 2`; still refused under 1 | +| PHP writer fell back once the dictionary exceeded `DICTIONARY_PAGE_SIZE` | once it reaches it (`>=`), as arrow-rs | +| PHP writer v2 chunks listed only the data page encoding | also `RLE` for the levels, as arrow-rs | +| PHP writer `DELTA_BINARY_PACKED`: a miniblock whose largest delta sits at or just above a power of two from about 2^47, or needs more than 55 bits, was packed wrong and read back wrong values | packed exactly; rewrite `WRITER_VERSION 2` files written by earlier versions whose INT32/INT64 columns hold such deltas | +| `COLUMNS_ENCODINGS` `RLE_DICTIONARY` on a BOOLEAN column: the PHP writer wrote a dictionary, the arrow writer ignored it | refused by both engines (`InvalidArgumentException`), as arrow-rs and parquet-java never dictionary-encode booleans | +| arrow writer: a `COLUMNS_ENCODINGS` entry the column type cannot take panicked (`\Error: Rust panic`) or was ignored (FIXED_LEN_BYTE_ARRAY dictionary under v1) | `InvalidArgumentException` with the PHP writer's message | +| INT32 `DELTA_BINARY_PACKED` with consecutive values more than 2^31 apart: written with 64-bit deltas arrow-rs refuses, read back as wrong values | 32-bit wrapping deltas, as arrow-rs and parquet-java; such INT32 files from earlier versions must be rewritten | + +The `arrow` extension must be rebuilt. + --- ## Upgrading from 0.45.x to 0.46.x diff --git a/src/extension/arrow-ext/src/parquet/library.rs b/src/extension/arrow-ext/src/parquet/library.rs index facb8caa35..7dd5d33716 100644 --- a/src/extension/arrow-ext/src/parquet/library.rs +++ b/src/extension/arrow-ext/src/parquet/library.rs @@ -13,6 +13,7 @@ use ext_php_rs::flags::{ClassFlags, DataType}; use ext_php_rs::prelude::*; use ext_php_rs::types::{ZendClassObject, ZendHashTable, Zval}; use ext_php_rs::zend::ce; +use parquet::arrow::ArrowSchemaConverter; use parquet::basic::Type as PhysicalType; use parquet::file::metadata::ParquetMetaData; @@ -911,10 +912,14 @@ impl RustParquetFileWriter { .collect::, Error>>() .map_err(refused)?; let scalar = columns.iter().all(|column| column.cell.is_scalar()); + let leaves = ArrowSchemaConverter::new() + .convert(&schema) + .map_err(Error::Parquet) + .map_err(refused)?; let writer = ParquetWriter::open( PhpSink::new(Arc::clone(&stream)), schema, - options::properties(compression, options).map_err(refused)?, + options::properties(compression, options, &leaves).map_err(refused)?, ) .map_err(refused)?; diff --git a/src/extension/arrow-ext/src/parquet/options.rs b/src/extension/arrow-ext/src/parquet/options.rs index 379d41ac8c..7c739e1aba 100644 --- a/src/extension/arrow-ext/src/parquet/options.rs +++ b/src/extension/arrow-ext/src/parquet/options.rs @@ -10,9 +10,9 @@ use arrow_schema::{ }; use ext_php_rs::ffi::zend_ulong; use ext_php_rs::types::{ZendHashTable, ZendStr}; -use parquet::basic::{BrotliLevel, Compression, Encoding, GzipLevel, ZstdLevel}; +use parquet::basic::{BrotliLevel, Compression, Encoding, GzipLevel, Type as PhysicalType, ZstdLevel}; use parquet::file::properties::{EnabledStatistics, WriterProperties, WriterVersion, DEFAULT_WRITE_BATCH_SIZE}; -use parquet::schema::types::ColumnPath; +use parquet::schema::types::{ColumnPath, SchemaDescriptor}; use crate::parquet::error::Error; use crate::php::ht_for_each; @@ -24,14 +24,19 @@ pub fn schema(ht: &ZendHashTable) -> Result { php_schema_to_arrow(ht).map_err(Error::Options) } -pub fn properties(compression: &str, options: &ZendHashTable) -> Result { +/// `leaves`: the parquet leaves the written schema converts to, which `COLUMNS_ENCODINGS` entries are checked against. +pub fn properties( + compression: &str, + options: &ZendHashTable, + leaves: &SchemaDescriptor, +) -> Result { if options.is_empty() { let codec = parse_compression(compression, None, None, None).map_err(Error::Options)?; return Ok(WriterProperties::builder().set_compression(codec).build()); } - Ok(apply_writer_options(WriterProperties::builder(), options, compression) + Ok(apply_writer_options(WriterProperties::builder(), options, compression, leaves) .map_err(Error::Options)? .build()) } @@ -253,6 +258,7 @@ fn apply_writer_options( mut builder: parquet::file::properties::WriterPropertiesBuilder, options: &ZendHashTable, compression_str: &str, + leaves: &SchemaDescriptor, ) -> Result { let gzip_level = options .get("GZIP_COMPRESSION_LEVEL") @@ -332,6 +338,10 @@ fn apply_writer_options( builder = builder.set_write_page_header_statistics(enabled); } + if let Some(enabled) = options.get("DICTIONARY_ENABLED").and_then(|z| z.bool()) { + builder = builder.set_dictionary_enabled(enabled); + } + if let Some(val) = options.get("DICTIONARY_PAGE_SIZE").and_then(|z| z.long()) { builder = builder.set_dictionary_page_size_limit(val as usize); } @@ -374,18 +384,33 @@ fn apply_writer_options( let encoding = value .str() .ok_or_else(|| format!("COLUMNS_ENCODINGS['{path}'] must be a string"))?; - encodings.push((column_path(&path), parse_encoding(encoding)?)); + let encoding = parse_encoding(encoding)?; + let writer_version = options.get("WRITER_VERSION").and_then(|z| z.long()).unwrap_or(1); + + if let Some(column) = leaves.columns().iter().find(|column| column.path().string() == path) { + if let Some(requirement) = encoding_requirement(encoding, column.physical_type(), writer_version) { + return Err(format!( + "{encoding:?} encoding {requirement}. Column '{path}' has type: {:?}", + column.physical_type() + )); + } + } + + encodings.push((column_path(&path), encoding)); Ok::<(), String>(()) })?; for (path, encoding) in encodings { - // arrow-rs panics on a dictionary encoding set as the column encoding: the dictionary is its own flag + // arrow-rs panics on a dictionary encoding set as the column encoding, and uses any other one only once the + // dictionary falls back: turning the dictionary off makes it the column's encoding, as the PHP writer writes it builder = match encoding { Encoding::PLAIN_DICTIONARY | Encoding::RLE_DICTIONARY => { builder.set_column_dictionary_enabled(path, true) } - encoding => builder.set_column_encoding(path, encoding), + encoding => builder + .set_column_encoding(path.clone(), encoding) + .set_column_dictionary_enabled(path, false), }; } } @@ -447,6 +472,45 @@ fn column_key(option: &str, key: Option<&ZendStr>, index: zend_ulong) -> Result< } } +/// The PHP writer's `ColumnEncodingPolicy` rules: what an encoding requires of the column, when the column lacks it. +fn encoding_requirement(encoding: Encoding, physical: PhysicalType, writer_version: i64) -> Option<&'static str> { + match encoding { + Encoding::RLE if physical != PhysicalType::BOOLEAN => Some("is only supported for BOOLEAN columns"), + Encoding::RLE_DICTIONARY | Encoding::PLAIN_DICTIONARY if physical == PhysicalType::BOOLEAN => { + Some("is not supported for BOOLEAN columns") + } + Encoding::RLE_DICTIONARY | Encoding::PLAIN_DICTIONARY + if physical == PhysicalType::FIXED_LEN_BYTE_ARRAY && writer_version != 2 => + { + Some("is not supported for FIXED_LEN_BYTE_ARRAY columns under WRITER_VERSION 1") + } + Encoding::DELTA_BINARY_PACKED if !matches!(physical, PhysicalType::INT32 | PhysicalType::INT64) => { + Some("is only supported for INT32 and INT64 columns") + } + Encoding::DELTA_LENGTH_BYTE_ARRAY if physical != PhysicalType::BYTE_ARRAY => { + Some("is only supported for BYTE_ARRAY columns") + } + Encoding::DELTA_BYTE_ARRAY + if !matches!(physical, PhysicalType::BYTE_ARRAY | PhysicalType::FIXED_LEN_BYTE_ARRAY) => + { + Some("is only supported for BYTE_ARRAY and FIXED_LEN_BYTE_ARRAY columns") + } + Encoding::BYTE_STREAM_SPLIT + if !matches!( + physical, + PhysicalType::FLOAT + | PhysicalType::DOUBLE + | PhysicalType::INT32 + | PhysicalType::INT64 + | PhysicalType::FIXED_LEN_BYTE_ARRAY + ) => + { + Some("is only supported for FLOAT, DOUBLE, INT32, INT64 and FIXED_LEN_BYTE_ARRAY columns") + } + _ => None, + } +} + fn column_path(path: &str) -> ColumnPath { ColumnPath::new(path.split('.').map(str::to_string).collect()) } diff --git a/src/extension/arrow-ext/tests/phpt/045_parquet_lib_read_parity.phpt b/src/extension/arrow-ext/tests/phpt/045_parquet_lib_read_parity.phpt index 9e602e655f..b3711a1ba6 100644 --- a/src/extension/arrow-ext/tests/phpt/045_parquet_lib_read_parity.phpt +++ b/src/extension/arrow-ext/tests/phpt/045_parquet_lib_read_parity.phpt @@ -60,6 +60,10 @@ decimals_fixed_len.parquet 100: identical decimals_fixed_len.parquet 10000: identical delta_binary_acked_encoded_integers.parquet 100: identical delta_binary_acked_encoded_integers.parquet 10000: identical +delta_byte_array_arrow_v2.parquet 100: identical +delta_byte_array_arrow_v2.parquet 10000: identical +delta_length_byte_array_duckdb_v2.parquet 100: identical +delta_length_byte_array_duckdb_v2.parquet 10000: identical lists.parquet 100: identical lists.parquet 10000: identical logical_types.parquet 100: identical diff --git a/src/extension/arrow-ext/tests/phpt/047_parquet_footer_thrift_parity.phpt b/src/extension/arrow-ext/tests/phpt/047_parquet_footer_thrift_parity.phpt index 2eeaf011ca..5f62896c84 100644 --- a/src/extension/arrow-ext/tests/phpt/047_parquet_footer_thrift_parity.phpt +++ b/src/extension/arrow-ext/tests/phpt/047_parquet_footer_thrift_parity.phpt @@ -66,6 +66,8 @@ $file->close(); columns.required.parquet: thrift identical, schema identical, footer reads 1 decimals_fixed_len.parquet: thrift identical, schema identical, footer reads 1 delta_binary_acked_encoded_integers.parquet: thrift identical, schema identical, footer reads 1 +delta_byte_array_arrow_v2.parquet: thrift identical, schema identical, footer reads 1 +delta_length_byte_array_duckdb_v2.parquet: thrift identical, schema identical, footer reads 1 lists.parquet: thrift identical, schema identical, footer reads 1 logical_types.parquet: thrift identical, schema identical, footer reads 1 maps.parquet: thrift identical, schema identical, footer reads 1 diff --git a/src/extension/arrow-ext/tests/phpt/065_parquet_column_option_keys.phpt b/src/extension/arrow-ext/tests/phpt/065_parquet_column_option_keys.phpt index 31324259bb..0cac2fa9cf 100644 --- a/src/extension/arrow-ext/tests/phpt/065_parquet_column_option_keys.phpt +++ b/src/extension/arrow-ext/tests/phpt/065_parquet_column_option_keys.phpt @@ -1,5 +1,5 @@ --TEST-- -COLUMNS_COMPRESSIONS, COLUMNS_ENCODINGS and COLUMNS_BLOOM_FILTERS reach the arrow writer by column path: a digit-only path (an int array key) applies to its column, a dictionary encoding enables the column's dictionary as the PHP writer writes it, a key that is not UTF-8 or a value that is not a string is refused instead of crashing +COLUMNS_COMPRESSIONS, COLUMNS_ENCODINGS and COLUMNS_BLOOM_FILTERS reach the arrow writer by column path: a digit-only path (an int array key) applies to its column, a dictionary encoding enables the column's dictionary, any other encoding is the column's only encoding, as the PHP writer writes them, an encoding the column type cannot take is refused with the PHP writer's message, a key that is not UTF-8 or a value that is not a string is refused instead of crashing --SKIPIF-- --FILE-- @@ -25,8 +25,7 @@ use function Flow\Filesystem\DSL\path; $write = static fn(Option $option, array $columns): string => MemoryParquetFile::written( new Writer( - // a dictionary page of one byte falls back to the column encoding at once - options: (new Options())->set($option, $columns)->set(Option::DICTIONARY_PAGE_SIZE, 1), + options: (new Options())->set($option, $columns), engine: new RustParquetEngine(), ), Schema::with(FlatColumn::int64('2024'), FlatColumn::int64('other')), @@ -53,6 +52,18 @@ foreach (['php' => static fn(Options $options): Writer => Writer::php(options: $ } } +foreach (['php' => static fn(Options $options): Writer => Writer::php(options: $options), 'arrow' => static fn(Options $options): Writer => new Writer(options: $options, engine: new RustParquetEngine())] as $engine => $writer) { + $file = MemoryParquetFile::read(Reader::php(), MemoryParquetFile::written( + $writer((new Options())->set(Option::COLUMNS_ENCODINGS, ['word' => Encodings::DELTA_BYTE_ARRAY])), + Schema::with(FlatColumn::string('word')), + [['word' => 'a'], ['word' => 'b'], ['word' => 'a']], + )); + + foreach ($file->pageHeaders() as $page) { + echo $engine, ': ', $page->pageHeader->type()->name, ' ', $page->pageHeader->encoding()->name, "\n"; + } +} + foreach (MemoryParquetFile::read(Reader::php(), MemoryParquetFile::written( new Writer(options: (new Options())->set(Option::COLUMNS_ENCODINGS, ['word' => Encodings::PLAIN_DICTIONARY]), engine: new RustParquetEngine()), Schema::with(FlatColumn::string('word')), @@ -61,6 +72,20 @@ foreach (MemoryParquetFile::read(Reader::php(), MemoryParquetFile::written( echo 'arrow PLAIN_DICTIONARY: ', $page->pageHeader->type()->name, ' ', $page->pageHeader->encoding()->name, "\n"; } +foreach ([ + 'flag' => [FlatColumn::boolean('flag'), Encodings::RLE_DICTIONARY, true], + 'fixed' => [FlatColumn::fixedSizeByteArray('fixed', 4), Encodings::RLE_DICTIONARY, 'abcd'], + 'amount' => [FlatColumn::double('amount'), Encodings::DELTA_BINARY_PACKED, 1.5], +] as $name => [$column, $encoding, $value]) { + foreach (['php' => static fn(Options $options): Writer => Writer::php(options: $options), 'arrow' => static fn(Options $options): Writer => new Writer(options: $options, engine: new RustParquetEngine())] as $engine => $writer) { + echo $engine, ' ', $name, ': ', arrow_outcome(static fn() => MemoryParquetFile::written( + $writer((new Options())->set(Option::COLUMNS_ENCODINGS, [$name => $encoding])), + Schema::with($column), + [[$name => $value]], + )), "\n"; + } +} + echo arrow_outcome(static fn() => new RustParquetFileWriter(new StringDestinationStream(path('memory://file.parquet')), SchemaConverter::toExtension(Schema::with(FlatColumn::int64('id'))), Compressions::SNAPPY, ['COLUMNS_COMPRESSIONS' => ['id' => 1]], 100)), "\n"; echo arrow_outcome(static fn() => new RustParquetFileWriter(new StringDestinationStream(path('memory://file.parquet')), SchemaConverter::toExtension(Schema::with(FlatColumn::int64('id'))), Compressions::SNAPPY, ['COLUMNS_ENCODINGS' => ['id' => null]], 100)), "\n"; echo arrow_outcome(static fn() => $write(Option::COLUMNS_COMPRESSIONS, ["\xFF" => Compressions::GZIP])), "\n"; @@ -76,8 +101,16 @@ php: DICTIONARY_PAGE PLAIN php: DATA_PAGE RLE_DICTIONARY arrow: DICTIONARY_PAGE PLAIN arrow: DATA_PAGE RLE_DICTIONARY +php: DATA_PAGE DELTA_BYTE_ARRAY +arrow: DATA_PAGE DELTA_BYTE_ARRAY arrow PLAIN_DICTIONARY: DICTIONARY_PAGE PLAIN arrow PLAIN_DICTIONARY: DATA_PAGE RLE_DICTIONARY +php flag: Flow\Parquet\Exception\InvalidArgumentException: RLE_DICTIONARY encoding is not supported for BOOLEAN columns. Column 'flag' has type: BOOLEAN +arrow flag: Flow\Parquet\Exception\InvalidArgumentException: RLE_DICTIONARY encoding is not supported for BOOLEAN columns. Column 'flag' has type: BOOLEAN +php fixed: Flow\Parquet\Exception\InvalidArgumentException: RLE_DICTIONARY encoding is not supported for FIXED_LEN_BYTE_ARRAY columns under WRITER_VERSION 1. Column 'fixed' has type: FIXED_LEN_BYTE_ARRAY +arrow fixed: Flow\Parquet\Exception\InvalidArgumentException: RLE_DICTIONARY encoding is not supported for FIXED_LEN_BYTE_ARRAY columns under WRITER_VERSION 1. Column 'fixed' has type: FIXED_LEN_BYTE_ARRAY +php amount: Flow\Parquet\Exception\InvalidArgumentException: DELTA_BINARY_PACKED encoding is only supported for INT32 and INT64 columns. Column 'amount' has type: DOUBLE +arrow amount: Flow\Parquet\Exception\InvalidArgumentException: DELTA_BINARY_PACKED encoding is only supported for INT32 and INT64 columns. Column 'amount' has type: DOUBLE Flow\Parquet\Exception\InvalidArgumentException: COLUMNS_COMPRESSIONS['id'] must be a string Flow\Parquet\Exception\InvalidArgumentException: COLUMNS_ENCODINGS['id'] must be a string Flow\Parquet\Exception\InvalidArgumentException: COLUMNS_COMPRESSIONS has a key that is not a column path diff --git a/src/extension/flow-php-ext/tests/phpt/068_parquet_read_parity.phpt b/src/extension/flow-php-ext/tests/phpt/068_parquet_read_parity.phpt index a3230682ba..f7a246da0d 100644 --- a/src/extension/flow-php-ext/tests/phpt/068_parquet_read_parity.phpt +++ b/src/extension/flow-php-ext/tests/phpt/068_parquet_read_parity.phpt @@ -83,6 +83,10 @@ decimals_fixed_len.parquet 100: identical decimals_fixed_len.parquet 10000: identical delta_binary_acked_encoded_integers.parquet 100: identical delta_binary_acked_encoded_integers.parquet 10000: identical +delta_byte_array_arrow_v2.parquet 100: both refuse +delta_byte_array_arrow_v2.parquet 10000: both refuse +delta_length_byte_array_duckdb_v2.parquet 100: identical +delta_length_byte_array_duckdb_v2.parquet 10000: identical lists.parquet 100: identical lists.parquet 10000: identical logical_types.parquet 100: identical diff --git a/src/lib/parquet/src/Flow/Parquet/Data/BitWidth.php b/src/lib/parquet/src/Flow/Parquet/Data/BitWidth.php index 64d4d5e506..4df19942eb 100644 --- a/src/lib/parquet/src/Flow/Parquet/Data/BitWidth.php +++ b/src/lib/parquet/src/Flow/Parquet/Data/BitWidth.php @@ -4,17 +4,23 @@ namespace Flow\Parquet\Data; -use function ceil; use function count; -use function log; +use function decbin; use function max; use function min; +use function strlen; final class BitWidth { + /** + * Packing through one int accumulator holds the value's bits plus up to 7 pending bits, which must stay below the + * sign bit: 56 + 7 = 63. + */ + public const int MAX_PACKED_IN_ACCUMULATOR = 56; + public static function calculate(int $value): int { - return (int) ceil(log($value + 1, 2)); + return $value === 0 ? 0 : strlen(decbin($value)); } /** diff --git a/src/lib/parquet/src/Flow/Parquet/Data/DeltaBinaryPackedDecoder.php b/src/lib/parquet/src/Flow/Parquet/Data/DeltaBinaryPackedDecoder.php index 54bfd95b81..e17b9f7994 100644 --- a/src/lib/parquet/src/Flow/Parquet/Data/DeltaBinaryPackedDecoder.php +++ b/src/lib/parquet/src/Flow/Parquet/Data/DeltaBinaryPackedDecoder.php @@ -7,11 +7,10 @@ use Flow\Parquet\BinaryReader\BinaryBufferReader; use Flow\Parquet\Exception\InvalidArgumentException; use Flow\Parquet\Exception\RuntimeException; +use Flow\Parquet\ParquetFile\Schema\PhysicalType; -use function bcadd; -use function bccomp; -use function bcsub; -use function is_float; +use function array_map; +use function array_merge; use function ord; use function strlen; @@ -26,6 +25,7 @@ public function __construct( private int $miniblockSize = self::DEFAULT_MINIBLOCK_SIZE, private DeltaCalculator $deltaCalculator = new DeltaCalculator(), private ZigZag $zigzag = new ZigZag(), + private PhysicalType $physicalType = PhysicalType::INT64, ) { if (($this->blockSize % 128) !== 0) { throw new InvalidArgumentException('Block size must be a multiple of 128'); @@ -43,27 +43,31 @@ public function __construct( /** * @return array */ - public function decode(string $data, int $valueCount): array + public function decode(BinaryBufferReader $reader, int $valueCount): array { if ($valueCount === 0) { return []; } - if ($data === '') { + if ($reader->remainingLength()->bytes() === 0) { throw new RuntimeException('Cannot decode empty data when value count is greater than 0'); } - $header = $this->readHeader($reader = new BinaryBufferReader($data)); + $header = $this->readHeader($reader); if ($header->totalValues !== $valueCount) { throw new RuntimeException("Value count mismatch: expected {$valueCount}, got {$header->totalValues}"); } - if ($valueCount === 1) { - return [$header->firstValue]; - } + $values = $valueCount === 1 + ? [$header->firstValue] + : $this->reconstructValues($header->firstValue, $this->readBlocks($reader, $valueCount - 1, $header)); - return $this->reconstructValues($header->firstValue, $this->readBlocks($reader, $valueCount - 1, $header)); + return ( + $this->physicalType === PhysicalType::INT32 + ? array_map($this->deltaCalculator->toInt32(...), $values) + : $values + ); } /** @@ -82,7 +86,7 @@ private function readBlock(BinaryBufferReader $reader, int $blockDeltaCount, Del $bitWidths[] = ord($reader->readBytes(1)); } - $deltas = []; + $miniblocks = []; $deltasRead = 0; for ( @@ -108,41 +112,12 @@ private function readBlock(BinaryBufferReader $reader, int $blockDeltaCount, Del $miniblockDeltas = $this->unpackMiniblockFromString($packedRaw, $bitWidth, $valuesToRead); } - $actualDeltas = array_map( - static function ($delta) use ($minDelta) { - // PHP converts int overflow to float at runtime, so $result may be float even though both operands are int. - // PHPStan models int arithmetic as never overflowing, which is why the is_float check below is suppressed. - $result = $delta + $minDelta; - - // Handle float overflow precisely using BCMath - // @mago-ignore analysis:impossible-condition - if (is_float($result)) { - // Use BCMath for precise integer arithmetic - $preciseResult = bcadd((string) $delta, (string) $minDelta, 0); - - // Apply 2's complement wrapping for 64-bit integers - // @mago-ignore analysis:redundant-condition - // @mago-ignore analysis:redundant-comparison - if (PHP_INT_SIZE === 8) { - if (bccomp($preciseResult, (string) PHP_INT_MAX, 0) > 0) { - $preciseResult = bcsub($preciseResult, '18446744073709551616', 0); - } elseif (bccomp($preciseResult, (string) PHP_INT_MIN, 0) < 0) { - $preciseResult = bcadd($preciseResult, '18446744073709551616', 0); - } - } - - return (int) $preciseResult; - } - - return $result; - }, - $miniblockDeltas, - ); - $deltas = array_merge($deltas, $actualDeltas); + $actualDeltas = $this->deltaCalculator->calculateAbsoluteDeltas($miniblockDeltas, $minDelta); + $miniblocks[] = $actualDeltas; $deltasRead += count($actualDeltas); } - return array_slice($deltas, 0, $blockDeltaCount); + return array_slice(array_merge(...$miniblocks), 0, $blockDeltaCount); } /** @@ -150,7 +125,7 @@ static function ($delta) use ($minDelta) { */ private function readBlocks(BinaryBufferReader $reader, int $deltaCount, DeltaHeader $header): array { - $deltas = []; + $blocks = []; $deltasRead = 0; while ($deltasRead < $deltaCount) { @@ -158,11 +133,11 @@ private function readBlocks(BinaryBufferReader $reader, int $deltaCount, DeltaHe $blockDeltaCount = min($header->blockSize, $remainingDeltas); $blockDeltas = $this->readBlock($reader, $blockDeltaCount, $header); - $deltas = array_merge($deltas, $blockDeltas); + $blocks[] = $blockDeltas; $deltasRead += count($blockDeltas); } - return $deltas; + return array_merge(...$blocks); } private function readHeader(BinaryBufferReader $reader): DeltaHeader @@ -205,32 +180,27 @@ private function reconstructValues(int $firstValue, array $deltas): array */ private function unpackMiniblockFromString(string $packedData, int $bitWidth, int $valuesToRead): array { - if ($bitWidth >= 62) { + if ($bitWidth > BitWidth::MAX_PACKED_IN_ACCUMULATOR) { return $this->unpackMiniblockFromStringSafe($packedData, $bitWidth, $valuesToRead); } $values = []; - $bitOffset = 0; + $mask = (1 << $bitWidth) - 1; + $accumulator = 0; + $accumulatedBits = 0; + $byteIndex = 0; $dataLen = strlen($packedData); for ($valueIndex = 0; $valueIndex < $valuesToRead; $valueIndex++) { - $value = 0; - - for ($bit = 0; $bit < $bitWidth; $bit++) { - $byteIndex = intdiv($bitOffset, 8); - $bitIndex = $bitOffset % 8; - - if ($byteIndex >= $dataLen) { - break; - } - - $byte = ord($packedData[$byteIndex]); - $bitValue = ($byte >> $bitIndex) & 1; - $value |= $bitValue << $bit; - $bitOffset++; + while ($accumulatedBits < $bitWidth) { + $accumulator |= ($byteIndex < $dataLen ? ord($packedData[$byteIndex]) : 0) << $accumulatedBits; + $byteIndex++; + $accumulatedBits += 8; } - $values[] = $value; + $values[] = $accumulator & $mask; + $accumulator >>= $bitWidth; + $accumulatedBits -= $bitWidth; } return $values; diff --git a/src/lib/parquet/src/Flow/Parquet/Data/DeltaBinaryPackedEncoder.php b/src/lib/parquet/src/Flow/Parquet/Data/DeltaBinaryPackedEncoder.php index 2301841bba..9000257b55 100644 --- a/src/lib/parquet/src/Flow/Parquet/Data/DeltaBinaryPackedEncoder.php +++ b/src/lib/parquet/src/Flow/Parquet/Data/DeltaBinaryPackedEncoder.php @@ -6,9 +6,20 @@ use Flow\Parquet\BinaryWriter\BinaryBufferWriter; use Flow\Parquet\Exception\InvalidArgumentException; +use Flow\Parquet\ParquetFile\Schema\PhysicalType; +use function array_fill; +use function array_map; use function array_pad; +use function array_slice; +use function ceil; +use function chr; use function count; +use function intdiv; +use function max; +use function min; +use function pack; +use function str_pad; final readonly class DeltaBinaryPackedEncoder { @@ -21,6 +32,7 @@ public function __construct( private int $miniblockSize = self::DEFAULT_MINIBLOCK_SIZE, private DeltaCalculator $deltaCalculator = new DeltaCalculator(), private ZigZag $zigzag = new ZigZag(), + private PhysicalType $physicalType = PhysicalType::INT64, ) { if (($this->blockSize % 128) !== 0) { throw new InvalidArgumentException('Block size must be a multiple of 128'); @@ -40,10 +52,6 @@ public function __construct( */ public function encode(array $values): string { - if (!count($values)) { - return ''; - } - $buffer = ''; $writer = new BinaryBufferWriter($buffer); @@ -63,15 +71,11 @@ private function packMiniblock(array $values, int $bitWidth): string return ''; } - $buffer = ''; - $writer = new BinaryBufferWriter($buffer); - - // For large bit widths (>= 62), use bit-by-bit packing to avoid overflow - if ($bitWidth >= 62) { - return $this->packMiniblockSafe($values, $bitWidth, $writer); + if ($bitWidth > BitWidth::MAX_PACKED_IN_ACCUMULATOR) { + return $this->packMiniblockSafe($values, $bitWidth); } - // Use the original fast method for smaller bit widths + $buffer = ''; $currentByte = 0; $bitsInByte = 0; @@ -80,23 +84,17 @@ private function packMiniblock(array $values, int $bitWidth): string $bitsInByte += $bitWidth; while ($bitsInByte >= 8) { - $writer->writeBytes([$currentByte & 0xFF]); + $buffer .= chr($currentByte & 0xFF); $currentByte >>= 8; $bitsInByte -= 8; } } if ($bitsInByte > 0) { - $writer->writeBytes([$currentByte & 0xFF]); + $buffer .= chr($currentByte & 0xFF); } - $expectedByteCount = (int) ceil(($this->miniblockSize * $bitWidth) / 8); - - while (strlen($buffer) < $expectedByteCount) { - $writer->writeBytes([0]); - } - - return $buffer; + return str_pad($buffer, (int) ceil(($this->miniblockSize * $bitWidth) / 8), "\0"); } /** @@ -105,7 +103,7 @@ private function packMiniblock(array $values, int $bitWidth): string * * @param array $values */ - private function packMiniblockSafe(array $values, int $bitWidth, BinaryBufferWriter $writer): string + private function packMiniblockSafe(array $values, int $bitWidth): string { $expectedByteCount = (int) ceil(($this->miniblockSize * $bitWidth) / 8); $bytes = array_fill(0, max(0, $expectedByteCount), 0); @@ -130,7 +128,6 @@ private function packMiniblockSafe(array $values, int $bitWidth, BinaryBufferWri } } - // Return the packed data as a string (don't write to the passed writer) return pack('C*', ...$bytes); } @@ -141,13 +138,7 @@ private function packMiniblockSafe(array $values, int $bitWidth, BinaryBufferWri */ private function padMiniblock(array $miniblockDeltas): array { - $padded = $miniblockDeltas; - - while (count($padded) < $this->miniblockSize) { - $padded[] = 0; - } - - return $padded; + return array_pad($miniblockDeltas, $this->miniblockSize, 0); } /** @@ -162,10 +153,7 @@ private function writeBlock(BinaryBufferWriter $writer, array $blockDeltas): voi $minDelta = min($blockDeltas); $this->writeSignedLEB128($writer, $minDelta); - $relativeDeltas = array_map(fn($delta) => $this->deltaCalculator->calculateRelativeDelta( - $delta, - $minDelta, - ), $blockDeltas); + $relativeDeltas = $this->deltaCalculator->calculateRelativeDeltas($blockDeltas, $minDelta); $miniblockCount = (int) ceil(count($relativeDeltas) / $this->miniblockSize); $bitWidths = []; @@ -201,6 +189,11 @@ private function writeBlocks(BinaryBufferWriter $writer, array $values): void } $deltas = $this->deltaCalculator->calculateDeltas($values); + + // INT32 deltas wrap at 32 bits, as arrow-rs and parquet-java write them, so relative deltas need at most 32 bits + if ($this->physicalType === PhysicalType::INT32) { + $deltas = array_map($this->deltaCalculator->toInt32(...), $deltas); + } $blockCount = (int) ceil(count($deltas) / $this->blockSize); for ($blockIndex = 0; $blockIndex < $blockCount; $blockIndex++) { @@ -222,7 +215,7 @@ private function writeHeader(BinaryBufferWriter $writer, array $values): void $this->writeULEB128($writer, $this->blockSize); $this->writeULEB128($writer, $miniblockCount); $this->writeULEB128($writer, count($values)); - $this->writeSignedLEB128($writer, $values[0]); + $this->writeSignedLEB128($writer, $values[0] ?? 0); } private function writeSignedLEB128(BinaryBufferWriter $writer, int $value): void diff --git a/src/lib/parquet/src/Flow/Parquet/Data/DeltaByteArrayDecoder.php b/src/lib/parquet/src/Flow/Parquet/Data/DeltaByteArrayDecoder.php new file mode 100644 index 0000000000..54790f6f5a --- /dev/null +++ b/src/lib/parquet/src/Flow/Parquet/Data/DeltaByteArrayDecoder.php @@ -0,0 +1,46 @@ + + */ + public function decode(BinaryBufferReader $reader, int $valueCount): array + { + $prefixLengths = $this->prefixLengths->decode($reader, $valueCount); + $values = []; + $previous = ''; + + foreach ($this->suffixes->decode($reader, $valueCount) as $index => $suffix) { + if ($prefixLengths[$index] < 0 || $prefixLengths[$index] > strlen($previous)) { + throw new RuntimeException(sprintf( + 'Invalid DELTA_BYTE_ARRAY prefix length %d of value %d, the previous value has %d bytes', + $prefixLengths[$index], + $index, + strlen($previous), + )); + } + + $previous = substr($previous, 0, $prefixLengths[$index]) . $suffix; + $values[] = $previous; + } + + return $values; + } +} diff --git a/src/lib/parquet/src/Flow/Parquet/Data/DeltaByteArrayEncoder.php b/src/lib/parquet/src/Flow/Parquet/Data/DeltaByteArrayEncoder.php new file mode 100644 index 0000000000..d741bd5c99 --- /dev/null +++ b/src/lib/parquet/src/Flow/Parquet/Data/DeltaByteArrayEncoder.php @@ -0,0 +1,36 @@ + $values + */ + public function encode(array $values): string + { + $prefixLengths = []; + $suffixes = []; + $previous = ''; + + foreach ($values as $value) { + // string XOR stops at the shorter operand: leading "\0" bytes are the common prefix + $prefixLength = strspn($previous ^ $value, "\0"); + $prefixLengths[] = $prefixLength; + $suffixes[] = substr($value, $prefixLength); + $previous = $value; + } + + return $this->prefixLengths->encode($prefixLengths) . $this->suffixes->encode($suffixes); + } +} diff --git a/src/lib/parquet/src/Flow/Parquet/Data/DeltaCalculator.php b/src/lib/parquet/src/Flow/Parquet/Data/DeltaCalculator.php index baf97ba170..d581ed4c4c 100644 --- a/src/lib/parquet/src/Flow/Parquet/Data/DeltaCalculator.php +++ b/src/lib/parquet/src/Flow/Parquet/Data/DeltaCalculator.php @@ -4,13 +4,21 @@ namespace Flow\Parquet\Data; +use function abs; use function bcadd; use function bccomp; use function bcsub; use function is_float; +use function max; +use function min; final readonly class DeltaCalculator { + /** + * Values within ±2^61 add and subtract without integer overflow, so they skip the BCMath fallback. + */ + private const int OVERFLOW_FREE_MAGNITUDE = 1 << 61; + public function calculateDelta(int $previous, int $current): int { $result = $current - $previous; @@ -59,6 +67,18 @@ public function calculateDeltas(array $values): array $deltas = []; $valuesCount = count($values); + if ($valuesCount < 2) { + return $deltas; + } + + if (min($values) >= -self::OVERFLOW_FREE_MAGNITUDE && max($values) <= self::OVERFLOW_FREE_MAGNITUDE) { + for ($i = 1; $i < $valuesCount; $i++) { + $deltas[] = $values[$i] - $values[$i - 1]; + } + + return $deltas; + } + for ($i = 1; $i < $valuesCount; $i++) { $deltas[] = $this->calculateDelta($values[$i - 1], $values[$i]); } @@ -66,6 +86,51 @@ public function calculateDeltas(array $values): array return $deltas; } + /** + * @param array $relativeDeltas + * + * @return array + */ + public function calculateAbsoluteDeltas(array $relativeDeltas, int $minDelta): array + { + $deltas = []; + + if ( + $relativeDeltas === [] + || abs($minDelta) <= self::OVERFLOW_FREE_MAGNITUDE + && max($relativeDeltas) <= self::OVERFLOW_FREE_MAGNITUDE + && min($relativeDeltas) >= -self::OVERFLOW_FREE_MAGNITUDE + ) { + foreach ($relativeDeltas as $relativeDelta) { + $deltas[] = $relativeDelta + $minDelta; + } + + return $deltas; + } + + foreach ($relativeDeltas as $relativeDelta) { + // PHP converts int overflow to float at runtime, so $result may be float even though both operands are int. + $result = $relativeDelta + $minDelta; + + // @mago-ignore analysis:impossible-condition + if (is_float($result)) { + $preciseResult = bcadd((string) $relativeDelta, (string) $minDelta, 0); + + if (bccomp($preciseResult, (string) PHP_INT_MAX, 0) > 0) { + $preciseResult = bcsub($preciseResult, '18446744073709551616', 0); + } elseif (bccomp($preciseResult, (string) PHP_INT_MIN, 0) < 0) { + $preciseResult = bcadd($preciseResult, '18446744073709551616', 0); + } + + $result = (int) $preciseResult; + } + + $deltas[] = $result; + } + + return $deltas; + } + public function calculateRelativeDelta(int $delta, int $minDelta): int { $result = $delta - $minDelta; @@ -104,6 +169,33 @@ public function calculateRelativeDelta(int $delta, int $minDelta): int return (int) $result; } + /** + * @param array $deltas + * + * @return array + */ + public function calculateRelativeDeltas(array $deltas, int $minDelta): array + { + $relativeDeltas = []; + + if ( + $deltas === [] + || $minDelta >= -self::OVERFLOW_FREE_MAGNITUDE && max($deltas) <= self::OVERFLOW_FREE_MAGNITUDE + ) { + foreach ($deltas as $delta) { + $relativeDeltas[] = $delta - $minDelta; + } + + return $relativeDeltas; + } + + foreach ($deltas as $delta) { + $relativeDeltas[] = $this->calculateRelativeDelta($delta, $minDelta); + } + + return $relativeDeltas; + } + /** * @param array $deltas * @@ -156,4 +248,11 @@ public function reconstructValues(int $firstValue, array $deltas): array return $values; } + + public function toInt32(int $value): int + { + $unsigned = $value & 0xFFFFFFFF; + + return $unsigned >= 0x80000000 ? $unsigned - 0x100000000 : $unsigned; + } } diff --git a/src/lib/parquet/src/Flow/Parquet/Data/DeltaLengthByteArrayDecoder.php b/src/lib/parquet/src/Flow/Parquet/Data/DeltaLengthByteArrayDecoder.php new file mode 100644 index 0000000000..5bfa89f640 --- /dev/null +++ b/src/lib/parquet/src/Flow/Parquet/Data/DeltaLengthByteArrayDecoder.php @@ -0,0 +1,41 @@ + + */ + public function decode(BinaryBufferReader $reader, int $valueCount): array + { + $values = []; + + foreach ($this->lengths->decode($reader, $valueCount) as $index => $length) { + if ($length < 0) { + throw new RuntimeException("Invalid DELTA_LENGTH_BYTE_ARRAY length {$length} of value {$index}"); + } + + $value = $reader->readBytes($length); + + if (strlen($value) !== $length) { + throw new RuntimeException("DELTA_LENGTH_BYTE_ARRAY data ends before value {$index}"); + } + + $values[] = $value; + } + + return $values; + } +} diff --git a/src/lib/parquet/src/Flow/Parquet/Data/DeltaLengthByteArrayEncoder.php b/src/lib/parquet/src/Flow/Parquet/Data/DeltaLengthByteArrayEncoder.php new file mode 100644 index 0000000000..817d2c2414 --- /dev/null +++ b/src/lib/parquet/src/Flow/Parquet/Data/DeltaLengthByteArrayEncoder.php @@ -0,0 +1,24 @@ + $values + */ + public function encode(array $values): string + { + return $this->lengths->encode(array_map(strlen(...), $values)) . implode('', $values); + } +} diff --git a/src/lib/parquet/src/Flow/Parquet/Engine/Arrow/OptionsConverter.php b/src/lib/parquet/src/Flow/Parquet/Engine/Arrow/OptionsConverter.php index d6fc62f42f..230eab0bac 100644 --- a/src/lib/parquet/src/Flow/Parquet/Engine/Arrow/OptionsConverter.php +++ b/src/lib/parquet/src/Flow/Parquet/Engine/Arrow/OptionsConverter.php @@ -46,6 +46,7 @@ public static function toExtension(Options $options): array [Option::ROW_GROUP_SIZE_BYTES, 'ROW_GROUP_SIZE_BYTES'], [Option::PAGE_SIZE_BYTES, 'PAGE_SIZE_BYTES'], [Option::PAGE_MAXIMUM_ROWS_COUNT, 'PAGE_MAXIMUM_ROWS_COUNT'], + [Option::DICTIONARY_ENABLED, 'DICTIONARY_ENABLED'], [Option::DICTIONARY_PAGE_SIZE, 'DICTIONARY_PAGE_SIZE'], [Option::WRITER_VERSION, 'WRITER_VERSION'], [Option::STATISTICS_TRUNCATE_LENGTH, 'STATISTICS_TRUNCATE_LENGTH'], diff --git a/src/lib/parquet/src/Flow/Parquet/Engine/PhpParquetFileWriter.php b/src/lib/parquet/src/Flow/Parquet/Engine/PhpParquetFileWriter.php index a63638b4f9..8c9702b0c0 100644 --- a/src/lib/parquet/src/Flow/Parquet/Engine/PhpParquetFileWriter.php +++ b/src/lib/parquet/src/Flow/Parquet/Engine/PhpParquetFileWriter.php @@ -79,9 +79,6 @@ public function __construct( Options $options, private readonly int $rowGroupSizeCheckInterval, ) { - $stream->append(ParquetFile::PARQUET_MAGIC_NUMBER); - $this->stream = $stream; - $this->fileOffset = strlen(ParquetFile::PARQUET_MAGIC_NUMBER); $this->metadata = new Metadata( $schema, new RowGroups([]), @@ -99,6 +96,9 @@ public function __construct( DataConverter::initialize($options), ), ); + $stream->append(ParquetFile::PARQUET_MAGIC_NUMBER); + $this->stream = $stream; + $this->fileOffset = strlen(ParquetFile::PARQUET_MAGIC_NUMBER); } public function close(): void diff --git a/src/lib/parquet/src/Flow/Parquet/Option.php b/src/lib/parquet/src/Flow/Parquet/Option.php index 5332d048a6..2f78c9bc26 100644 --- a/src/lib/parquet/src/Flow/Parquet/Option.php +++ b/src/lib/parquet/src/Flow/Parquet/Option.php @@ -82,7 +82,17 @@ enum Option case COLUMN_INDEX_TRUNCATE_LENGTH; /** - * Whenever size of the dictionary goes above this value, PagesBuilders is going to fallback to PLAIN encoding. + * Columns without a COLUMNS_ENCODINGS entry are dictionary-encoded (RLE_DICTIONARY), except BOOLEAN and, under + * writer version 1, FIXED_LEN_BYTE_ARRAY, as arrow-rs and parquet-java write them. False writes the writer + * version's fallback encoding instead. A COLUMNS_ENCODINGS entry always wins. + * + * Default value is true + */ + case DICTIONARY_ENABLED; + + /** + * Once the column chunk's dictionary reaches this size, the rest of the chunk falls back to the writer version's + * fallback encoding (PLAIN under v1). * * Default value is 1Mb */ diff --git a/src/lib/parquet/src/Flow/Parquet/Options.php b/src/lib/parquet/src/Flow/Parquet/Options.php index edfd30ddde..d46457d168 100644 --- a/src/lib/parquet/src/Flow/Parquet/Options.php +++ b/src/lib/parquet/src/Flow/Parquet/Options.php @@ -37,6 +37,7 @@ public function __construct() Option::STATISTICS_TRUNCATE_LENGTH->name => 64, Option::COLUMN_INDEX_TRUNCATE_LENGTH->name => 64, Option::PAGE_HEADER_STATISTICS->name => false, + Option::DICTIONARY_ENABLED->name => true, Option::DICTIONARY_PAGE_SIZE->name => SizeUnits::MiB_SIZE, Option::BROTLI_COMPRESSION_LEVEL->name => 11, Option::GZIP_COMPRESSION_LEVEL->name => 9, diff --git a/src/lib/parquet/src/Flow/Parquet/ParquetFile/Page/Header/DataPageHeaderV2.php b/src/lib/parquet/src/Flow/Parquet/ParquetFile/Page/Header/DataPageHeaderV2.php index b4d8dcbdf6..3363efb4d4 100644 --- a/src/lib/parquet/src/Flow/Parquet/ParquetFile/Page/Header/DataPageHeaderV2.php +++ b/src/lib/parquet/src/Flow/Parquet/ParquetFile/Page/Header/DataPageHeaderV2.php @@ -47,6 +47,11 @@ public function encoding(): Encodings return $this->encoding; } + public function isCompressed(): bool + { + return $this->isCompressed ?? true; + } + public function repetitionsByteLength(): int { return $this->repetitionsByteLength; diff --git a/src/lib/parquet/src/Flow/Parquet/Reader/ColumnDataDecoder.php b/src/lib/parquet/src/Flow/Parquet/Reader/ColumnDataDecoder.php index 3d83bc730e..4bc3f063b4 100644 --- a/src/lib/parquet/src/Flow/Parquet/Reader/ColumnDataDecoder.php +++ b/src/lib/parquet/src/Flow/Parquet/Reader/ColumnDataDecoder.php @@ -7,32 +7,27 @@ use Flow\Parquet\Binary\ByteOrder; use Flow\Parquet\BinaryReader\BinaryBufferReader; use Flow\Parquet\Data\BitWidth; -use Flow\Parquet\Data\DeltaBinaryPackedDecoder; use Flow\Parquet\Data\PlainValueUnpacker; use Flow\Parquet\Data\RLEBitPackedHybrid; use Flow\Parquet\Dremel\ColumnData\ReadFlatColumnValues; -use Flow\Parquet\Exception\RuntimeException; -use Flow\Parquet\ParquetFile\Encodings; use Flow\Parquet\ParquetFile\Page\Dictionary; use Flow\Parquet\ParquetFile\Page\Header\DataPageHeader; use Flow\Parquet\ParquetFile\Page\Header\DataPageHeaderV2; use Flow\Parquet\ParquetFile\Page\Header\DictionaryPageHeader; use Flow\Parquet\ParquetFile\Schema\FlatColumn; -use Flow\Parquet\ParquetFile\Schema\PhysicalType; -use Generator; use function array_fill; -use function array_key_exists; -use function count; -use function Flow\Parquet\empty_generator; use function iterator_to_array; -use function ord; final readonly class ColumnDataDecoder { + private PageValuesDecoder $pageValues; + public function __construct( private ByteOrder $byteOrder = ByteOrder::LITTLE_ENDIAN, - ) {} + ) { + $this->pageValues = new PageValuesDecoder($this->byteOrder); + } public function decodeData( string $buffer, @@ -68,64 +63,18 @@ public function decodeData( $definitionLevels = array_fill(0, max(0, $pageHeader->valuesCount()), $column->maxDefinitionsLevel()); } - $nonEmptyValuesCount = $this->countValues($definitionLevels, $column); - - if ($pageHeader->encoding() === Encodings::PLAIN) { - return new ReadFlatColumnValues( + return new ReadFlatColumnValues( + $column, + $this->pageValues->decode( + $reader, $column, - (new PlainValueUnpacker($reader, $this->byteOrder))->unpack($column, $nonEmptyValuesCount), - $repetitionLevels, - $definitionLevels, - ); - } - - if ($pageHeader->encoding() === Encodings::DELTA_BINARY_PACKED) { - if (!in_array($column->type(), [PhysicalType::INT32, PhysicalType::INT64], true)) { - throw new RuntimeException('Delta encoding only supports INT32 and INT64 physical types'); - } - - if ($nonEmptyValuesCount === 0) { - return new ReadFlatColumnValues($column, empty_generator(), $repetitionLevels, $definitionLevels); - } - - $remainingData = $reader->readBytes($reader->remainingLength()->bytes()); - - $decoder = new DeltaBinaryPackedDecoder(); - $values = $decoder->decode($remainingData, $nonEmptyValuesCount); - - $valuesGenerator = static function () use ($values) { - foreach ($values as $value) { - yield $value; - } - }; - - return new ReadFlatColumnValues($column, $valuesGenerator(), $repetitionLevels, $definitionLevels); - } - - if ( - $pageHeader->encoding() === Encodings::RLE_DICTIONARY - || $pageHeader->encoding() === Encodings::PLAIN_DICTIONARY - ) { - if ($nonEmptyValuesCount) { - $bitWidth = ord($reader->readBytes(1)); - - $indices = $this->readRLEBitPackedHybrid($reader, $RLEBitPackedHybrid, $bitWidth, $nonEmptyValuesCount); - - $valuesGenerator = static function () use ($indices, $dictionary) { - foreach ($indices as $index) { - yield $dictionary && array_key_exists($index, $dictionary->values) - ? $dictionary->values[$index] - : null; - } - }; - - return new ReadFlatColumnValues($column, $valuesGenerator(), $repetitionLevels, $definitionLevels); - } - - return new ReadFlatColumnValues($column, empty_generator(), $repetitionLevels, $definitionLevels); - } - - throw new RuntimeException('Encoding ' . $pageHeader->encoding()->name . ' not supported'); + $pageHeader->encoding(), + $this->countValues($definitionLevels, $column), + $dictionary, + ), + $repetitionLevels, + $definitionLevels, + ); } public function decodeDataV2( @@ -160,79 +109,18 @@ public function decodeDataV2( $definitionLevels = array_fill(0, max(0, $pageHeader->valuesCount()), $column->maxDefinitionsLevel()); } - $nonEmptyValuesCount = $this->countValues($definitionLevels, $column); - - if ($pageHeader->encoding() === Encodings::PLAIN) { - return new ReadFlatColumnValues( - $column, - (new PlainValueUnpacker($reader, $this->byteOrder))->unpack($column, $nonEmptyValuesCount), - $repetitionLevels, - $definitionLevels, - ); - } - - if ($pageHeader->encoding() === Encodings::RLE && $column->type() === PhysicalType::BOOLEAN) { - // RLE data values, unlike levels in a v2 page, carry their 4-byte length first - $reader->seekBytes(4); - $values = $this->readRLEBitPackedHybrid($reader, $RLEBitPackedHybrid, 1, $nonEmptyValuesCount); - - return new ReadFlatColumnValues( + return new ReadFlatColumnValues( + $column, + $this->pageValues->decode( + $reader, $column, - (static function () use ($values): Generator { - foreach ($values as $value) { - yield $value === 1; - } - })(), - $repetitionLevels, - $definitionLevels, - ); - } - - if ($pageHeader->encoding() === Encodings::DELTA_BINARY_PACKED) { - if (!in_array($column->type(), [PhysicalType::INT32, PhysicalType::INT64], true)) { - throw new RuntimeException('Delta encoding only supports INT32 and INT64 physical types'); - } - - if ($nonEmptyValuesCount === 0) { - return new ReadFlatColumnValues($column, empty_generator(), $repetitionLevels, $definitionLevels); - } - - $remainingData = $reader->readBytes($reader->remainingLength()->bytes()); - - $decoder = new DeltaBinaryPackedDecoder(); - $values = $decoder->decode($remainingData, $nonEmptyValuesCount); - - $valuesGenerator = static function () use ($values) { - foreach ($values as $value) { - yield $value; - } - }; - - return new ReadFlatColumnValues($column, $valuesGenerator(), $repetitionLevels, $definitionLevels); - } - - if ( - $pageHeader->encoding() === Encodings::RLE_DICTIONARY - || $pageHeader->encoding() === Encodings::PLAIN_DICTIONARY - ) { - if (count($definitionLevels)) { - $bitWidth = ord($reader->readBytes(1)); - - $indices = $this->readRLEBitPackedHybrid($reader, $RLEBitPackedHybrid, $bitWidth, $nonEmptyValuesCount); - - $valuesGenerator = static function () use ($indices, $dictionary) { - foreach ($indices as $index) { - yield $dictionary?->values[$index]; - } - }; - - return new ReadFlatColumnValues($column, $valuesGenerator(), $repetitionLevels, $definitionLevels); - } - - return new ReadFlatColumnValues($column, empty_generator(), $repetitionLevels, $definitionLevels); - } - - throw new RuntimeException('Encoding ' . $pageHeader->encoding()->name . ' not supported'); + $pageHeader->encoding(), + $this->countValues($definitionLevels, $column), + $dictionary, + ), + $repetitionLevels, + $definitionLevels, + ); } public function decodeDictionary(string $buffer, FlatColumn $column, DictionaryPageHeader $pageHeader): Dictionary diff --git a/src/lib/parquet/src/Flow/Parquet/Reader/PageReader.php b/src/lib/parquet/src/Flow/Parquet/Reader/PageReader.php index 973cb7882e..1907fdf411 100644 --- a/src/lib/parquet/src/Flow/Parquet/Reader/PageReader.php +++ b/src/lib/parquet/src/Flow/Parquet/Reader/PageReader.php @@ -19,10 +19,14 @@ final readonly class PageReader { + private ColumnDataDecoder $decoder; + public function __construct( private ByteOrder $byteOrder, private Options $options, - ) {} + ) { + $this->decoder = new ColumnDataDecoder($this->byteOrder); + } /** * @param resource $stream @@ -48,12 +52,7 @@ public function readData( $pageHeader->uncompressedPageSize(), ); - return (new ColumnDataDecoder($this->byteOrder))->decodeData( - $data, - $column, - $dataPageHeader, - $dictionary, - ); + return $this->decoder->decodeData($data, $column, $dataPageHeader, $dictionary); case Type::DATA_PAGE_V2: $dataPageHeaderV2 = $pageHeader->dataPageHeaderV2(); @@ -65,18 +64,16 @@ public function readData( $levels = $levelsLength > 0 ? $this->readBytes($stream, $levelsLength) : ''; - $data = (new Codec($this->options))->decompress( - $this->readBytes($stream, $pageHeader->compressedPageSize() - $levelsLength), - $codec, - $pageHeader->uncompressedPageSize() - $levelsLength, - ); + $values = $this->readBytes($stream, $pageHeader->compressedPageSize() - $levelsLength); + $data = $dataPageHeaderV2->isCompressed() + ? (new Codec($this->options))->decompress( + $values, + $codec, + $pageHeader->uncompressedPageSize() - $levelsLength, + ) + : $values; - return (new ColumnDataDecoder($this->byteOrder))->decodeDataV2( - $levels . $data, - $column, - $dataPageHeaderV2, - $dictionary, - ); + return $this->decoder->decodeDataV2($levels . $data, $column, $dataPageHeaderV2, $dictionary); default: throw new RuntimeException("Unknown page header type '{$pageHeader->type()->name}'"); @@ -97,7 +94,7 @@ public function readDictionary(FlatColumn $column, PageHeader $pageHeader, Compr $compressedSize = $pageHeader->compressedPageSize(); $compressed = $compressedSize === 0 ? '' : $this->readBytes($stream, $compressedSize); - return (new ColumnDataDecoder($this->byteOrder))->decodeDictionary( + return $this->decoder->decodeDictionary( (new Codec($this->options))->decompress($compressed, $codec, $pageHeader->uncompressedPageSize()), $column, $dictionaryPageHeader, diff --git a/src/lib/parquet/src/Flow/Parquet/Reader/PageValuesDecoder.php b/src/lib/parquet/src/Flow/Parquet/Reader/PageValuesDecoder.php new file mode 100644 index 0000000000..55d41bcc50 --- /dev/null +++ b/src/lib/parquet/src/Flow/Parquet/Reader/PageValuesDecoder.php @@ -0,0 +1,153 @@ + (new PlainValueUnpacker($reader, $this->byteOrder))->unpack($column, $valuesCount), + Encodings::RLE => $this->rleBooleans($reader, $column, $valuesCount), + Encodings::DELTA_BINARY_PACKED => $this->deltaBinaryPacked($reader, $column, $valuesCount), + Encodings::DELTA_LENGTH_BYTE_ARRAY => $this->deltaLengthByteArray($reader, $column, $valuesCount), + Encodings::DELTA_BYTE_ARRAY => $this->deltaByteArray($reader, $column, $valuesCount), + Encodings::RLE_DICTIONARY, Encodings::PLAIN_DICTIONARY => $this->dictionary( + $reader, + $valuesCount, + $dictionary, + ), + default => throw new RuntimeException('Encoding ' . $encoding->name . ' not supported'), + }; + } + + /** + * @return array + */ + private function deltaBinaryPacked(BinaryBufferReader $reader, FlatColumn $column, int $valuesCount): array + { + if (!in_array($column->type(), [PhysicalType::INT32, PhysicalType::INT64], true)) { + throw new RuntimeException('Delta encoding only supports INT32 and INT64 physical types'); + } + + return $column->type() === PhysicalType::INT32 + ? $this->int32Deltas->decode($reader, $valuesCount) + : $this->int64Deltas->decode($reader, $valuesCount); + } + + /** + * @return list + */ + private function deltaByteArray(BinaryBufferReader $reader, FlatColumn $column, int $valuesCount): array + { + if (!in_array($column->type(), [PhysicalType::BYTE_ARRAY, PhysicalType::FIXED_LEN_BYTE_ARRAY], true)) { + throw new RuntimeException( + 'DELTA_BYTE_ARRAY encoding only supports BYTE_ARRAY and FIXED_LEN_BYTE_ARRAY physical types', + ); + } + + $values = $this->deltaByteArray->decode($reader, $valuesCount); + + if ($column->type() === PhysicalType::FIXED_LEN_BYTE_ARRAY) { + foreach ($values as $index => $value) { + if (strlen($value) !== $column->typeLength()) { + throw new RuntimeException(sprintf( + "DELTA_BYTE_ARRAY value %d of FIXED_LEN_BYTE_ARRAY column '%s' has %d bytes, expected %d", + $index, + $column->flatPath(), + strlen($value), + (int) $column->typeLength(), + )); + } + } + } + + return $values; + } + + /** + * @return list + */ + private function deltaLengthByteArray(BinaryBufferReader $reader, FlatColumn $column, int $valuesCount): array + { + if ($column->type() !== PhysicalType::BYTE_ARRAY) { + throw new RuntimeException('DELTA_LENGTH_BYTE_ARRAY encoding only supports BYTE_ARRAY physical type'); + } + + return $this->deltaLengthByteArray->decode($reader, $valuesCount); + } + + /** + * @return array + */ + private function dictionary(BinaryBufferReader $reader, int $valuesCount, ?Dictionary $dictionary): array + { + if ($valuesCount === 0) { + return []; + } + + $bitWidth = ord($reader->readBytes(1)); + + return array_map( + static fn(int $index): mixed => $dictionary !== null && array_key_exists($index, $dictionary->values) + ? $dictionary->values[$index] + : null, + $this->rleBitPackedHybrid->decodeHybrid($reader, $bitWidth, $valuesCount), + ); + } + + /** + * @return array + */ + private function rleBooleans(BinaryBufferReader $reader, FlatColumn $column, int $valuesCount): array + { + if ($column->type() !== PhysicalType::BOOLEAN) { + throw new RuntimeException('Encoding RLE not supported'); + } + + // RLE data values carry their 4-byte length first + $reader->seekBytes(4); + + return array_map( + static fn(int $value): bool => $value === 1, + $this->rleBitPackedHybrid->decodeHybrid($reader, 1, $valuesCount), + ); + } +} diff --git a/src/lib/parquet/src/Flow/Parquet/Writer.php b/src/lib/parquet/src/Flow/Parquet/Writer.php index 3dcfae9114..87fd34fa14 100644 --- a/src/lib/parquet/src/Flow/Parquet/Writer.php +++ b/src/lib/parquet/src/Flow/Parquet/Writer.php @@ -12,9 +12,11 @@ use Flow\Parquet\Exception\RuntimeException; use Flow\Parquet\ParquetFile\Compressions; use Flow\Parquet\ParquetFile\Schema; +use Throwable; use function file_exists; use function Flow\Filesystem\DSL\path; +use function unlink; final class Writer { @@ -78,12 +80,16 @@ public function open(string $path, Schema $schema): void throw new InvalidArgumentException("File {$path} already exists"); } - $this->file = $this->engine->openForWrite( - NativeLocalDestinationStream::openBlank(path($path)), - $schema, - $this->compression, - $this->options, - ); + $stream = NativeLocalDestinationStream::openBlank(path($path)); + + try { + $this->file = $this->engine->openForWrite($stream, $schema, $this->compression, $this->options); + } catch (Throwable $exception) { + $stream->close(); + unlink($path); + + throw $exception; + } } public function openForStream(DestinationStream $stream, Schema $schema): void @@ -100,13 +106,16 @@ public function write(string $path, Schema $schema, iterable $rows): void throw new InvalidArgumentException("File {$path} already exists"); } - $this->engine->writeRows( - NativeLocalDestinationStream::openBlank(path($path)), - $schema, - $this->compression, - $this->options, - $rows, - ); + $stream = NativeLocalDestinationStream::openBlank(path($path)); + + try { + $this->engine->writeRows($stream, $schema, $this->compression, $this->options, $rows); + } catch (Throwable $exception) { + $stream->close(); + unlink($path); + + throw $exception; + } } /** diff --git a/src/lib/parquet/src/Flow/Parquet/Writer/ColumnChunkBuilder/DeltaBinaryPackedColumnChunkBuilder.php b/src/lib/parquet/src/Flow/Parquet/Writer/ColumnChunkBuilder/DeltaBinaryPackedColumnChunkBuilder.php deleted file mode 100644 index 2fb7ff7cce..0000000000 --- a/src/lib/parquet/src/Flow/Parquet/Writer/ColumnChunkBuilder/DeltaBinaryPackedColumnChunkBuilder.php +++ /dev/null @@ -1,319 +0,0 @@ - - */ - private array $definitionLevels = []; - - private int $nonNullValuesCount = 0; - - private int $nullCount = 0; - - private PageContainers $pages; - - private StatisticsCounter $pageStatistics; - - private PageIndexBuilder $pageIndex; - - /** - * @var array - */ - private array $repetitionLevels = []; - - private int $rowsCount = 0; - - private readonly ValueStorage $valueStorage; - - public function __construct( - private readonly FlatColumn $column, - private readonly Options $options, - private readonly Compressions $compression, - private readonly StatisticsOptions $statistics, - ) { - $this->pageIndex = $this->statistics->pageIndex($this->column); - $this->pages = new PageContainers(); - $this->chunkStatistics = $this->statistics->counter($this->column); - $this->pageStatistics = $this->statistics->counter($this->column); - $this->valueStorage = new DeltaBinaryPackedValueStorage(); - $this->byteOrder = ByteOrder::LITTLE_ENDIAN; - - if (!in_array($this->column->type(), [PhysicalType::INT32, PhysicalType::INT64], true)) { - throw new InvalidArgumentException('Delta encoding only supports INT32 and INT64 physical types'); - } - } - - public function addColumn(WriteFlatColumnValues $columnValues): void - { - array_push($this->repetitionLevels, ...$columnValues->repetitionLevels()); - - $defLevels = $columnValues->definitionLevels(); - array_push($this->definitionLevels, ...$defLevels); - - $maxDefinitionLevel = $this->column->maxDefinitionsLevel(); - - $nullsInBatch = 0; - - foreach ($defLevels as $definitionLevel) { - if ($definitionLevel < $maxDefinitionLevel) { - $this->nullCount++; - $nullsInBatch++; - } else { - $this->nonNullValuesCount++; - } - } - - $this->valueStorage->addValues($this->column, $columnValues->values()); - $this->pageStatistics->addBatch($columnValues->values()); - - if ($nullsInBatch > 0) { - $this->pageStatistics->addNulls($nullsInBatch); - } - - $this->rowsCount += $columnValues->rowsCount(); - } - - public function closePage(): void - { - if ($this->isEmpty()) { - return; - } - - $codec = new Codec($this->options); - - $pageContainer = match ($writerVersion = $this->options->getInt(Option::WRITER_VERSION)) { - 1 => $this->buildDataPage($codec, $this->compression), - 2 => $this->buildDataPageV2($codec, $this->compression), - default => throw new RuntimeException( - 'Flow Parquet Writer does not support given version of Parquet format, supported versions are [1,2], given: ' - . $writerVersion, - ), - }; - - $this->pages->add($pageContainer); - $this->pageIndex->addPage($this->pageStatistics, $this->rowsCount); - $this->chunkStatistics = $this->chunkStatistics->merge($this->pageStatistics); - - $this->repetitionLevels = []; - $this->definitionLevels = []; - $this->valueStorage->reset(); - $this->rowsCount = 0; - $this->nullCount = 0; - $this->nonNullValuesCount = 0; - $this->pageStatistics = $this->statistics->counter($this->column); - } - - public function column(): Column - { - return $this->column; - } - - public function flush(int $fileOffset): array - { - $this->closePage(); - - $dataPageOffset = $fileOffset; - $containers = [new ColumnChunkContainer( - $this->pages->buffer(), - new ColumnChunk( - type: $this->column->type(), - codec: $this->compression, - valuesCount: $this->pages->valuesCount(), - fileOffset: $fileOffset, - path: $this->column->path(), - encodings: $this->pages->encodings(), - totalCompressedSize: $this->pages->compressedSize(), - totalUncompressedSize: $this->pages->uncompressedSize(), - dictionaryPageOffset: null, - dataPageOffset: $dataPageOffset, - indexPageOffset: null, - statistics: $this->statistics->chunk($this->chunkStatistics), - ), - $this->pageIndex->columnIndex(), - $this->pageIndex->offsetIndex($dataPageOffset, $this->pages), - )]; - - $this->pages = new PageContainers(); - $this->pageIndex = $this->statistics->pageIndex($this->column); - $this->chunkStatistics = $this->statistics->counter($this->column); - $this->pageStatistics = $this->statistics->counter($this->column); - $this->repetitionLevels = []; - $this->definitionLevels = []; - $this->valueStorage->reset(); - $this->rowsCount = 0; - $this->nullCount = 0; - $this->nonNullValuesCount = 0; - - return $containers; - } - - public function isEmpty(): bool - { - return ( - $this->valueStorage->size() === 0 - && count($this->definitionLevels) === 0 - && count($this->repetitionLevels) === 0 - ); - } - - public function isFull(): bool - { - return ( - ( - $this->valueStorage->size() * ($this->column->type() === PhysicalType::INT32 ? 4 : 8) - ) >= $this->options->getInt(Option::PAGE_SIZE_BYTES) - ); - } - - public function uncompressedSize(): int - { - return $this->pages->uncompressedSize() + $this->currentPageUncompressedSize(); - } - - private function buildDataPage(Codec $codec, Compressions $compression): PageContainer - { - $rleBitPackedHybrid = new RLEBitPackedHybrid(); - $packer = new RLEBitPackedPacker($rleBitPackedHybrid, $this->byteOrder); - - $pageBuffer = ''; - $pageWriter = new BinaryBufferWriter($pageBuffer); - - if ($this->column->maxRepetitionsLevel() > 0) { - $pageWriter->append($packer->packWithLength( - BitWidth::calculate($this->column->maxRepetitionsLevel()), - $this->repetitionLevels, - )); - } - - if ($this->column->maxDefinitionsLevel() > 0) { - $pageWriter->append($packer->packWithLength( - BitWidth::calculate($this->column->maxDefinitionsLevel()), - $this->definitionLevels, - )); - } - - $pageWriter->append($this->valueStorage->getBuffer()); - - $compressedBuffer = $codec->compress($pageBuffer, $compression); - - $pageHeader = new PageHeader( - Type::DATA_PAGE, - strlen($compressedBuffer), - strlen($pageBuffer), - dataPageHeader: new DataPageHeader( - encoding: Encodings::DELTA_BINARY_PACKED, - repetitionLevelEncoding: Encodings::RLE, - definitionLevelEncoding: Encodings::RLE, - valuesCount: count($this->definitionLevels), - statistics: $this->statistics->pageHeader($this->pageStatistics), - ), - dataPageHeaderV2: null, - dictionaryPageHeader: null, - ); - - return new PageContainer($compressedBuffer, $pageHeader); - } - - private function buildDataPageV2(Codec $codec, Compressions $compression): PageContainer - { - $statistics = $this->statistics->pageHeader($this->pageStatistics); - - $rleBitPackedHybrid = new RLEBitPackedHybrid(); - $packer = new RLEBitPackedPacker($rleBitPackedHybrid, $this->byteOrder); - - if ($this->column->maxRepetitionsLevel() > 0) { - $repetitionsBuffer = $packer->pack( - BitWidth::calculate($this->column->maxRepetitionsLevel()), - $this->repetitionLevels, - ); - $repetitionsLength = strlen($repetitionsBuffer); - } else { - $repetitionsBuffer = ''; - $repetitionsLength = 0; - } - - if ($this->column->maxDefinitionsLevel() > 0) { - $definitionsBuffer = $packer->pack( - BitWidth::calculate($this->column->maxDefinitionsLevel()), - $this->definitionLevels, - ); - $definitionsLength = strlen($definitionsBuffer); - } else { - $definitionsBuffer = ''; - $definitionsLength = 0; - } - - $encodedValues = $this->valueStorage->getBuffer(); - $compressedBuffer = $codec->compress($encodedValues, $compression); - - $pageHeader = new PageHeader( - Type::DATA_PAGE_V2, - strlen($compressedBuffer) + $repetitionsLength + $definitionsLength, - strlen($encodedValues) + $repetitionsLength + $definitionsLength, - dataPageHeader: null, - dataPageHeaderV2: new DataPageHeaderV2( - valuesCount: count($this->definitionLevels), - nullsCount: $this->nullCount, - rowsCount: $this->rowsCount, - encoding: Encodings::DELTA_BINARY_PACKED, - definitionsByteLength: $definitionsLength, - repetitionsByteLength: $repetitionsLength, - isCompressed: !($compression === Compressions::UNCOMPRESSED), - statistics: $statistics, - ), - dictionaryPageHeader: null, - ); - - return new PageContainer($repetitionsBuffer . $definitionsBuffer . $compressedBuffer, $pageHeader); - } - - private function currentPageUncompressedSize(): int - { - return ( - $this->valueStorage->size() + (count($this->repetitionLevels) * 4) + (count($this->definitionLevels) * 4) - ); - } -} diff --git a/src/lib/parquet/src/Flow/Parquet/Writer/ColumnChunkBuilder/PlainFlatColumnChunkBuilder.php b/src/lib/parquet/src/Flow/Parquet/Writer/ColumnChunkBuilder/FlatColumnChunkBuilder.php similarity index 91% rename from src/lib/parquet/src/Flow/Parquet/Writer/ColumnChunkBuilder/PlainFlatColumnChunkBuilder.php rename to src/lib/parquet/src/Flow/Parquet/Writer/ColumnChunkBuilder/FlatColumnChunkBuilder.php index 151fb5d4f0..b8211a18d5 100644 --- a/src/lib/parquet/src/Flow/Parquet/Writer/ColumnChunkBuilder/PlainFlatColumnChunkBuilder.php +++ b/src/lib/parquet/src/Flow/Parquet/Writer/ColumnChunkBuilder/FlatColumnChunkBuilder.php @@ -21,7 +21,6 @@ use Flow\Parquet\ParquetFile\RowGroup\ColumnChunk; use Flow\Parquet\ParquetFile\Schema\Column; use Flow\Parquet\ParquetFile\Schema\FlatColumn; -use Flow\Parquet\ParquetFile\Schema\PhysicalType; use Flow\Parquet\Writer\ColumnChunkBuilder; use Flow\Parquet\Writer\ColumnChunkContainer; use Flow\Parquet\Writer\PageBuilder\RLEBitPackedPacker; @@ -30,15 +29,13 @@ use Flow\Parquet\Writer\PageIndexBuilder; use Flow\Parquet\Writer\StatisticsCounter; use Flow\Parquet\Writer\StatisticsOptions; -use Flow\Parquet\Writer\ValueStorage\BooleanValueStorage; -use Flow\Parquet\Writer\ValueStorage\BufferValueStorage; use Flow\Parquet\Writer\ValueStorage\ValueStorage; use RuntimeException; use function count; use function strlen; -final class PlainFlatColumnChunkBuilder implements ColumnChunkBuilder +final class FlatColumnChunkBuilder implements ColumnChunkBuilder { private readonly ByteOrder $byteOrder; @@ -66,23 +63,18 @@ final class PlainFlatColumnChunkBuilder implements ColumnChunkBuilder private int $rowsCount = 0; - private readonly ValueStorage $valueStorage; - public function __construct( private readonly FlatColumn $column, private readonly Options $options, private readonly Compressions $compression, private readonly StatisticsOptions $statistics, + private readonly ValueStorage $valueStorage, ) { $this->pageIndex = $this->statistics->pageIndex($this->column); $this->pages = new PageContainers(); $this->chunkStatistics = $this->statistics->counter($this->column); $this->pageStatistics = $this->statistics->counter($this->column); $this->byteOrder = ByteOrder::LITTLE_ENDIAN; - - $this->valueStorage = $this->column->type() === PhysicalType::BOOLEAN - ? new BooleanValueStorage() - : new BufferValueStorage(); } public function addColumn(WriteFlatColumnValues $columnValues): void @@ -196,7 +188,7 @@ public function flush(int $fileOffset): array public function isEmpty(): bool { return ( - $this->valueStorage->size() === 0 + $this->valueStorage->isEmpty() && count($this->definitionLevels) === 0 && count($this->repetitionLevels) === 0 ); @@ -204,7 +196,7 @@ public function isEmpty(): bool public function isFull(): bool { - return $this->valueStorage->size() >= $this->options->getInt(Option::PAGE_SIZE_BYTES); + return $this->valueStorage->plainSize() >= $this->options->getInt(Option::PAGE_SIZE_BYTES); } public function uncompressedSize(): int @@ -243,7 +235,7 @@ private function buildDataPage(Codec $codec, Compressions $compression): PageCon strlen($compressedBuffer), strlen($pageBuffer), dataPageHeader: new DataPageHeader( - encoding: Encodings::PLAIN, + encoding: $this->valueStorage->encoding(), repetitionLevelEncoding: Encodings::RLE, definitionLevelEncoding: Encodings::RLE, valuesCount: count($this->definitionLevels), @@ -285,18 +277,19 @@ private function buildDataPageV2(Codec $codec, Compressions $compression): PageC $definitionsLength = 0; } - $compressedBuffer = $codec->compress($this->valueStorage->getBuffer(), $compression); + $encodedValues = $this->valueStorage->getBuffer(); + $compressedBuffer = $codec->compress($encodedValues, $compression); $pageHeader = new PageHeader( Type::DATA_PAGE_V2, strlen($compressedBuffer) + $repetitionsLength + $definitionsLength, - strlen($this->valueStorage->getBuffer()) + $repetitionsLength + $definitionsLength, + strlen($encodedValues) + $repetitionsLength + $definitionsLength, dataPageHeader: null, dataPageHeaderV2: new DataPageHeaderV2( valuesCount: count($this->definitionLevels), nullsCount: $this->nullCount, rowsCount: $this->rowsCount, - encoding: Encodings::PLAIN, + encoding: $this->valueStorage->encoding(), definitionsByteLength: $definitionsLength, repetitionsByteLength: $repetitionsLength, isCompressed: !($compression === Compressions::UNCOMPRESSED), @@ -311,7 +304,9 @@ private function buildDataPageV2(Codec $codec, Compressions $compression): PageC private function currentPageUncompressedSize(): int { return ( - $this->valueStorage->size() + (count($this->repetitionLevels) * 4) + (count($this->definitionLevels) * 4) + $this->valueStorage->plainSize() + + (count($this->repetitionLevels) * 4) + + (count($this->definitionLevels) * 4) ); } } diff --git a/src/lib/parquet/src/Flow/Parquet/Writer/ColumnChunkBuilder/RLEDictionaryChunkBuilder.php b/src/lib/parquet/src/Flow/Parquet/Writer/ColumnChunkBuilder/RLEDictionaryChunkBuilder.php index 1d485bb0ef..e0ad0c0406 100644 --- a/src/lib/parquet/src/Flow/Parquet/Writer/ColumnChunkBuilder/RLEDictionaryChunkBuilder.php +++ b/src/lib/parquet/src/Flow/Parquet/Writer/ColumnChunkBuilder/RLEDictionaryChunkBuilder.php @@ -24,6 +24,7 @@ use Flow\Parquet\ParquetFile\RowGroup\ColumnChunk; use Flow\Parquet\ParquetFile\Schema\Column; use Flow\Parquet\ParquetFile\Schema\FlatColumn; +use Flow\Parquet\ParquetFile\Schema\PhysicalType; use Flow\Parquet\Writer\ColumnChunkBuilder; use Flow\Parquet\Writer\ColumnChunkContainer; use Flow\Parquet\Writer\PageBuilder\Dictionary; @@ -34,9 +35,15 @@ use Flow\Parquet\Writer\PageIndexBuilder; use Flow\Parquet\Writer\StatisticsCounter; use Flow\Parquet\Writer\StatisticsOptions; +use Flow\Parquet\Writer\ValueStorage\ValueStorage; +use function array_filter; +use function array_push; use function array_slice; use function count; +use function implode; +use function intdiv; +use function is_string; use function strlen; final class RLEDictionaryChunkBuilder implements ColumnChunkBuilder @@ -60,6 +67,11 @@ final class RLEDictionaryChunkBuilder implements ColumnChunkBuilder */ private int $dictionaryBytes = 0; + /** + * Built once the dictionary overflows, when it can no longer change, so its values are freed before the flush. + */ + private ?PageContainer $dictionaryPage = null; + private int $nullCount = 0; private PageContainers $pages; @@ -81,16 +93,22 @@ final class RLEDictionaryChunkBuilder implements ColumnChunkBuilder private int $rowsCount = 0; /** - * Set once a page would grow the dictionary past Option::DICTIONARY_PAGE_SIZE: that page and the rest of the chunk - * are PLAIN, as arrow-rs falls back. + * Set once a page would grow the dictionary to Option::DICTIONARY_PAGE_SIZE: that page and the rest of the chunk + * use the fallback storage's encoding, as arrow-rs falls back. + */ + private bool $overflowed = false; + + /** + * Set once a page of the chunk is written with dictionary indices, which needs a dictionary page even when empty. */ - private bool $plain = false; + private bool $indexed = false; public function __construct( private readonly FlatColumn $column, private readonly Options $options, private readonly Compressions $compression, private readonly StatisticsOptions $statistics, + private readonly ValueStorage $fallback, ) { $this->pageIndex = $this->statistics->pageIndex($this->column); $this->pages = new PageContainers(); @@ -118,7 +136,12 @@ public function addColumn(WriteFlatColumnValues $columnValues): void } } - array_push($this->pageValues, ...$columnValues->values()); + if ($this->overflowed) { + $this->fallback->addValues($this->column, $columnValues->values()); + } else { + array_push($this->pageValues, ...$columnValues->values()); + } + $this->pageStatistics->addBatch($columnValues->values()); if ($nullsInBatch > 0) { @@ -135,7 +158,11 @@ public function closePage(): void } $codec = new Codec($this->options); - $indices = $this->plain ? null : $this->dictionaryIndices(); + $indices = $this->overflowed ? null : $this->dictionaryIndices(); + + if ($indices !== null) { + $this->indexed = true; + } $pageContainer = match ($writerVersion = $this->options->getInt(Option::WRITER_VERSION)) { 1 => $this->buildDataPage($codec, $this->compression, $indices), @@ -153,6 +180,7 @@ public function closePage(): void $this->repetitionLevels = []; $this->definitionLevels = []; $this->pageValues = []; + $this->fallback->reset(); $this->rowsCount = 0; $this->nullCount = 0; $this->pageStatistics = $this->statistics->counter($this->column); @@ -167,7 +195,9 @@ public function flush(int $fileOffset): array { $this->closePage(); - if ($this->dictionary->dictionary !== []) { + if ($this->dictionaryPage !== null) { + $this->pages->add($this->dictionaryPage); + } elseif ($this->indexed) { $this->pages->add($this->buildDictionaryPage(new Codec($this->options), $this->compression)); } @@ -203,9 +233,12 @@ public function flush(int $fileOffset): array $this->definitionLevels = []; $this->repetitionLevels = []; $this->pageValues = []; + $this->fallback->reset(); $this->dictionary = new Dictionary([], []); $this->dictionaryBytes = 0; - $this->plain = false; + $this->dictionaryPage = null; + $this->overflowed = false; + $this->indexed = false; $this->rowsCount = 0; $this->nullCount = 0; @@ -216,6 +249,7 @@ public function isEmpty(): bool { return ( count($this->pageValues) === 0 + && $this->fallback->isEmpty() && count($this->definitionLevels) === 0 && count($this->repetitionLevels) === 0 ); @@ -223,7 +257,7 @@ public function isEmpty(): bool public function isFull(): bool { - return (count($this->pageValues) * 4) >= $this->options->getInt(Option::PAGE_SIZE_BYTES); + return $this->pageValuesSize() >= $this->options->getInt(Option::PAGE_SIZE_BYTES); } public function uncompressedSize(): int @@ -232,7 +266,7 @@ public function uncompressedSize(): int } /** - * @param null|array $indices the page's dictionary indices, null for a PLAIN page + * @param null|array $indices the page's dictionary indices, null for a fallback page */ private function buildDataPage(Codec $codec, Compressions $compression, ?array $indices): PageContainer { @@ -265,7 +299,7 @@ private function buildDataPage(Codec $codec, Compressions $compression, ?array $ strlen($compressedBuffer), strlen($pageBuffer), dataPageHeader: new DataPageHeader( - encoding: $indices === null ? Encodings::PLAIN : Encodings::RLE_DICTIONARY, + encoding: $indices === null ? $this->fallback->encoding() : Encodings::RLE_DICTIONARY, repetitionLevelEncoding: Encodings::RLE, definitionLevelEncoding: Encodings::RLE, valuesCount: count($this->definitionLevels), @@ -279,7 +313,7 @@ private function buildDataPage(Codec $codec, Compressions $compression, ?array $ } /** - * @param null|array $indices the page's dictionary indices, null for a PLAIN page + * @param null|array $indices the page's dictionary indices, null for a fallback page */ private function buildDataPageV2(Codec $codec, Compressions $compression, ?array $indices): PageContainer { @@ -322,7 +356,7 @@ private function buildDataPageV2(Codec $codec, Compressions $compression, ?array valuesCount: count($this->definitionLevels), nullsCount: $this->nullCount, rowsCount: $this->rowsCount, - encoding: $indices === null ? Encodings::PLAIN : Encodings::RLE_DICTIONARY, + encoding: $indices === null ? $this->fallback->encoding() : Encodings::RLE_DICTIONARY, definitionsByteLength: $definitionsLength, repetitionsByteLength: $repetitionsLength, isCompressed: !($compression === Compressions::UNCOMPRESSED), @@ -356,8 +390,8 @@ private function buildDictionaryPage(Codec $codec, Compressions $compression): P } /** - * The page's values in the chunk dictionary, extending it; null when that would grow it past - * Option::DICTIONARY_PAGE_SIZE, which turns this page and the rest of the chunk PLAIN. + * The page's values in the chunk dictionary, extending it; null when that would grow it to + * Option::DICTIONARY_PAGE_SIZE, which turns this page and the rest of the chunk to the fallback encoding. * * @return null|array */ @@ -373,20 +407,22 @@ private function dictionaryIndices(): ?array ), $this->dictionary, ); - $added = ''; - (new PlainValuesPacker(new BinaryBufferWriter($added), $this->byteOrder))->packValues( - $this->column, - array_slice($extended->dictionary, count($this->dictionary->dictionary)), - ); + $addedBytes = $this->plainSize(array_slice($extended->dictionary, count($this->dictionary->dictionary))); + + if (($this->dictionaryBytes + $addedBytes) >= $this->options->getInt(Option::DICTIONARY_PAGE_SIZE)) { + $this->overflowed = true; + + if ($this->indexed) { + $this->dictionaryPage = $this->buildDictionaryPage(new Codec($this->options), $this->compression); + } - if (($this->dictionaryBytes + strlen($added)) > $this->options->getInt(Option::DICTIONARY_PAGE_SIZE)) { - $this->plain = true; + $this->dictionary = new Dictionary([], []); return null; } $this->dictionary = $extended; - $this->dictionaryBytes += strlen($added); + $this->dictionaryBytes += $addedBytes; return $extended->indices; } @@ -397,22 +433,37 @@ private function dictionaryIndices(): ?array private function values(RLEBitPackedPacker $packer, ?array $indices): string { if ($indices === null) { - $values = ''; - (new PlainValuesPacker(new BinaryBufferWriter($values), $this->byteOrder))->packValues( - $this->column, - $this->pageValues, - ); + $this->fallback->addValues($this->column, $this->pageValues); - return $values; + return $this->fallback->getBuffer(); } - return $indices === [] ? '' : $packer->packWithBitWidth(BitWidth::fromArray($indices), $indices); + return $packer->packWithBitWidth(BitWidth::fromArray($indices), $indices); } private function currentPageUncompressedSize(): int { - return ( - (count($this->pageValues) * 4) + (count($this->repetitionLevels) * 4) + (count($this->definitionLevels) * 4) - ); + return $this->pageValuesSize() + (count($this->repetitionLevels) * 4) + (count($this->definitionLevels) * 4); + } + + private function pageValuesSize(): int + { + return $this->overflowed ? $this->fallback->plainSize() : count($this->pageValues) * 4; + } + + /** + * @param array $values + */ + private function plainSize(array $values): int + { + return match ($this->column->type()) { + PhysicalType::BYTE_ARRAY => (count($values) * 4) + + strlen(implode('', array_filter($values, is_string(...)))), + PhysicalType::FIXED_LEN_BYTE_ARRAY => strlen(implode('', array_filter($values, is_string(...)))), + PhysicalType::BOOLEAN => intdiv(count($values) + 7, 8), + PhysicalType::INT32, PhysicalType::FLOAT => count($values) * 4, + PhysicalType::INT64, PhysicalType::DOUBLE => count($values) * 8, + PhysicalType::INT96 => count($values) * 12, + }; } } diff --git a/src/lib/parquet/src/Flow/Parquet/Writer/ColumnChunkBuilderFactory.php b/src/lib/parquet/src/Flow/Parquet/Writer/ColumnChunkBuilderFactory.php index 3044ae862e..f8f5f4c956 100644 --- a/src/lib/parquet/src/Flow/Parquet/Writer/ColumnChunkBuilderFactory.php +++ b/src/lib/parquet/src/Flow/Parquet/Writer/ColumnChunkBuilderFactory.php @@ -4,16 +4,21 @@ namespace Flow\Parquet\Writer; -use Flow\Parquet\Exception\InvalidArgumentException; use Flow\Parquet\Option; use Flow\Parquet\Options; use Flow\Parquet\ParquetFile\Compressions; use Flow\Parquet\ParquetFile\Encodings; use Flow\Parquet\ParquetFile\Schema\FlatColumn; use Flow\Parquet\ParquetFile\Schema\PhysicalType; -use Flow\Parquet\Writer\ColumnChunkBuilder\DeltaBinaryPackedColumnChunkBuilder; -use Flow\Parquet\Writer\ColumnChunkBuilder\PlainFlatColumnChunkBuilder; +use Flow\Parquet\Writer\ColumnChunkBuilder\FlatColumnChunkBuilder; use Flow\Parquet\Writer\ColumnChunkBuilder\RLEDictionaryChunkBuilder; +use Flow\Parquet\Writer\ValueStorage\BooleanValueStorage; +use Flow\Parquet\Writer\ValueStorage\BufferValueStorage; +use Flow\Parquet\Writer\ValueStorage\DeltaBinaryPackedValueStorage; +use Flow\Parquet\Writer\ValueStorage\DeltaByteArrayValueStorage; +use Flow\Parquet\Writer\ValueStorage\DeltaLengthByteArrayValueStorage; +use Flow\Parquet\Writer\ValueStorage\RLEBooleanValueStorage; +use Flow\Parquet\Writer\ValueStorage\ValueStorage; use function array_key_exists; @@ -42,75 +47,52 @@ public static function createBuilder( } } + $policy = new ColumnEncodingPolicy( + $options->getInt(Option::WRITER_VERSION), + $options->getBool(Option::DICTIONARY_ENABLED), + ); + $requested = null; + if ($options->has(Option::COLUMNS_ENCODINGS)) { $columnsEncodings = $options->getArray(Option::COLUMNS_ENCODINGS); if ($columnsEncodings !== null && array_key_exists($flatPath, $columnsEncodings)) { // @mago-ignore analysis:mixed-assignment - $encoding = $columnsEncodings[$flatPath]; + $columnEncoding = $columnsEncodings[$flatPath]; - if ($encoding instanceof Encodings) { - return self::createForEncoding($column, $encoding, $options, $columnCompression, $statistics); + if ($columnEncoding instanceof Encodings) { + $requested = $columnEncoding; } } } - if ( - ($column->type() === PhysicalType::INT32 || $column->type() === PhysicalType::INT64) - && $options->getInt(Option::WRITER_VERSION) === 2 - ) { - return new DeltaBinaryPackedColumnChunkBuilder($column, $options, $columnCompression, $statistics); - } - - return new PlainFlatColumnChunkBuilder($column, $options, $columnCompression, $statistics); - } - - private static function createForEncoding( - FlatColumn $column, - Encodings $encoding, - Options $options, - Compressions $compressions, - StatisticsOptions $statistics, - ): ColumnChunkBuilder { - self::validateEncodingForColumn($column, $encoding); + $encoding = $policy->encodingFor($column, $requested); - return match ($encoding) { - Encodings::PLAIN => new PlainFlatColumnChunkBuilder($column, $options, $compressions, $statistics), - Encodings::RLE_DICTIONARY => new RLEDictionaryChunkBuilder($column, $options, $compressions, $statistics), - Encodings::DELTA_BINARY_PACKED => new DeltaBinaryPackedColumnChunkBuilder( + return $encoding === Encodings::RLE_DICTIONARY + ? new RLEDictionaryChunkBuilder( $column, $options, - $compressions, + $columnCompression, $statistics, - ), - default => throw new InvalidArgumentException("Unsupported encoding for column builder: {$encoding->name}"), - }; + self::valueStorage($policy->fallbackEncoding($column->type()), $column), + ) + : new FlatColumnChunkBuilder( + $column, + $options, + $columnCompression, + $statistics, + self::valueStorage($encoding, $column), + ); } - private static function validateEncodingForColumn(FlatColumn $column, Encodings $encoding): void + private static function valueStorage(Encodings $encoding, FlatColumn $column): ValueStorage { - $columnType = $column->type(); - $encodingName = $encoding->name; - $flatPath = $column->flatPath(); - - match ($encoding) { - Encodings::DELTA_BINARY_PACKED => $columnType !== PhysicalType::INT32 && $columnType !== PhysicalType::INT64 - ? throw new InvalidArgumentException( - 'DELTA_BINARY_PACKED encoding is only supported for INT32 and INT64 columns. ' - . "Column '{$flatPath}' has type: {$columnType->name}", - ) - : null, - Encodings::RLE_DICTIONARY => $columnType === PhysicalType::FIXED_LEN_BYTE_ARRAY - ? throw new InvalidArgumentException( - 'RLE_DICTIONARY encoding is not supported for FIXED_LEN_BYTE_ARRAY columns. ' - . "Column '{$flatPath}' has type: {$columnType->name}", - ) - : null, - Encodings::PLAIN => null, - default => throw new InvalidArgumentException( - "Encoding '{$encodingName}' is not implemented. " - . 'Supported encodings: PLAIN, RLE_DICTIONARY, DELTA_BINARY_PACKED', - ), + return match ($encoding) { + Encodings::DELTA_BINARY_PACKED => new DeltaBinaryPackedValueStorage(), + Encodings::DELTA_LENGTH_BYTE_ARRAY => new DeltaLengthByteArrayValueStorage(), + Encodings::DELTA_BYTE_ARRAY => new DeltaByteArrayValueStorage(), + Encodings::RLE => new RLEBooleanValueStorage(), + default => $column->type() === PhysicalType::BOOLEAN ? new BooleanValueStorage() : new BufferValueStorage(), }; } } diff --git a/src/lib/parquet/src/Flow/Parquet/Writer/ColumnEncodingPolicy.php b/src/lib/parquet/src/Flow/Parquet/Writer/ColumnEncodingPolicy.php new file mode 100644 index 0000000000..3611b36a43 --- /dev/null +++ b/src/lib/parquet/src/Flow/Parquet/Writer/ColumnEncodingPolicy.php @@ -0,0 +1,111 @@ +dictionaryEnabled && $this->supportsDictionary($column->type()) + ? Encodings::RLE_DICTIONARY + : $this->fallbackEncoding($column->type()); + } + + $this->assertAllowed($column, $requested); + + return $requested; + } + + public function fallbackEncoding(PhysicalType $type): Encodings + { + return match (true) { + $this->writerVersion !== 2 => Encodings::PLAIN, + $type === PhysicalType::BOOLEAN => Encodings::RLE, + $type === PhysicalType::INT32, $type === PhysicalType::INT64 => Encodings::DELTA_BINARY_PACKED, + $type === PhysicalType::BYTE_ARRAY, + $type === PhysicalType::FIXED_LEN_BYTE_ARRAY, + => Encodings::DELTA_BYTE_ARRAY, + default => Encodings::PLAIN, + }; + } + + public function supportsDictionary(PhysicalType $type): bool + { + return match ($type) { + PhysicalType::BOOLEAN => false, + PhysicalType::FIXED_LEN_BYTE_ARRAY => $this->writerVersion === 2, + default => true, + }; + } + + private function assertAllowed(FlatColumn $column, Encodings $encoding): void + { + $type = $column->type(); + + if (!in_array($encoding, self::SUPPORTED, true)) { + throw new InvalidArgumentException( + "Encoding '{$encoding->name}' is not implemented. Supported encodings: " + . implode(', ', array_map( + static fn(Encodings $supported): string => $supported->name, + self::SUPPORTED, + )), + ); + } + + $requirement = match ($encoding) { + Encodings::RLE => $type === PhysicalType::BOOLEAN ? null : 'is only supported for BOOLEAN columns', + Encodings::RLE_DICTIONARY => match (true) { + $this->supportsDictionary($type) => null, + $type === PhysicalType::FIXED_LEN_BYTE_ARRAY + => 'is not supported for FIXED_LEN_BYTE_ARRAY columns under WRITER_VERSION 1', + default => "is not supported for {$type->name} columns", + }, + Encodings::DELTA_BINARY_PACKED => in_array($type, [PhysicalType::INT32, PhysicalType::INT64], true) + ? null + : 'is only supported for INT32 and INT64 columns', + Encodings::DELTA_LENGTH_BYTE_ARRAY => $type === PhysicalType::BYTE_ARRAY + ? null + : 'is only supported for BYTE_ARRAY columns', + Encodings::DELTA_BYTE_ARRAY => in_array( + $type, + [PhysicalType::BYTE_ARRAY, PhysicalType::FIXED_LEN_BYTE_ARRAY], + true, + ) + ? null + : 'is only supported for BYTE_ARRAY and FIXED_LEN_BYTE_ARRAY columns', + default => null, + }; + + if ($requirement !== null) { + throw new InvalidArgumentException( + "{$encoding->name} encoding {$requirement}. Column '{$column->flatPath()}' has type: {$type->name}", + ); + } + } +} diff --git a/src/lib/parquet/src/Flow/Parquet/Writer/PageBuilder/DictionaryBuilder.php b/src/lib/parquet/src/Flow/Parquet/Writer/PageBuilder/DictionaryBuilder.php index ca2fa2d88d..fdbc7f32bb 100644 --- a/src/lib/parquet/src/Flow/Parquet/Writer/PageBuilder/DictionaryBuilder.php +++ b/src/lib/parquet/src/Flow/Parquet/Writer/PageBuilder/DictionaryBuilder.php @@ -34,20 +34,11 @@ public function build( PhysicalType::BOOLEAN => (new ScalarDictionaryBuilder())->build($data, $dictionary), PhysicalType::FLOAT, PhysicalType::DOUBLE => (new FloatDictionaryBuilder())->build($data, $dictionary), PhysicalType::FIXED_LEN_BYTE_ARRAY, PhysicalType::BYTE_ARRAY => match ($column->logicalType()?->name()) { - LogicalType::STRING, - LogicalType::JSON, - LogicalType::BSON, - LogicalType::UUID, - LogicalType::ENUM, - LogicalType::DECIMAL, - => (new ScalarDictionaryBuilder())->build($data, $dictionary), LogicalType::DATE, LogicalType::TIME, LogicalType::TIMESTAMP => (new ObjectDictionaryBuilder())->build( $data, $dictionary, ), - default => throw new RuntimeException( - 'Building dictionary for "' . ($column->logicalType()?->name() ?? 'null') . '" is not supported', - ), + default => (new ScalarDictionaryBuilder())->build($data, $dictionary), }, default => throw new RuntimeException( 'Building dictionary for "' . $column->type()->name . '" is not supported', diff --git a/src/lib/parquet/src/Flow/Parquet/Writer/PageContainers.php b/src/lib/parquet/src/Flow/Parquet/Writer/PageContainers.php index 1fc754a1ec..418f1e6025 100644 --- a/src/lib/parquet/src/Flow/Parquet/Writer/PageContainers.php +++ b/src/lib/parquet/src/Flow/Parquet/Writer/PageContainers.php @@ -20,6 +20,8 @@ final class PageContainers private ?PageContainer $dictionaryPageContainer = null; + private int $uncompressedSize = 0; + /** * @param array $containers */ @@ -38,11 +40,13 @@ public function add(PageContainer $container): void } $this->dictionaryPageContainer = $container; + $this->uncompressedSize += $container->totalUncompressedSize(); return; } $this->dataPageContainers[] = $container; + $this->uncompressedSize += $container->totalUncompressedSize(); } public function buffer(): string @@ -113,6 +117,8 @@ public function encodings(): array $dataPageHeaderV2 = $pageContainer->pageHeader->dataPageHeaderV2(); if ($dataPageHeaderV2 !== null) { + // v2 levels are always RLE, arrow-rs lists it for every chunk + $encodings[] = Encodings::RLE->value; $encodings[] = $dataPageHeaderV2->encoding()->value; } } @@ -124,17 +130,7 @@ public function encodings(): array public function uncompressedSize(): int { - $size = 0; - - if ($this->dictionaryPageContainer) { - $size += $this->dictionaryPageContainer->totalUncompressedSize(); - } - - foreach ($this->dataPageContainers as $pageContainer) { - $size += $pageContainer->totalUncompressedSize(); - } - - return $size; + return $this->uncompressedSize; } public function valuesCount(): int diff --git a/src/lib/parquet/src/Flow/Parquet/Writer/ValueStorage/BooleanValueStorage.php b/src/lib/parquet/src/Flow/Parquet/Writer/ValueStorage/BooleanValueStorage.php index ff8c96cefa..7853192172 100644 --- a/src/lib/parquet/src/Flow/Parquet/Writer/ValueStorage/BooleanValueStorage.php +++ b/src/lib/parquet/src/Flow/Parquet/Writer/ValueStorage/BooleanValueStorage.php @@ -5,9 +5,11 @@ namespace Flow\Parquet\Writer\ValueStorage; use Flow\Parquet\BinaryWriter\BinaryBufferWriter; +use Flow\Parquet\ParquetFile\Encodings; use Flow\Parquet\ParquetFile\Schema\FlatColumn; use function count; +use function intdiv; use function is_bool; final class BooleanValueStorage implements ValueStorage @@ -27,6 +29,11 @@ public function addValues(FlatColumn $column, array $values): void } } + public function encoding(): Encodings + { + return Encodings::PLAIN; + } + public function getBuffer(): string { if (empty($this->values)) { @@ -55,8 +62,8 @@ public function reset(): void $this->values = []; } - public function size(): int + public function plainSize(): int { - return count($this->values); + return intdiv(count($this->values) + 7, 8); } } diff --git a/src/lib/parquet/src/Flow/Parquet/Writer/ValueStorage/BufferValueStorage.php b/src/lib/parquet/src/Flow/Parquet/Writer/ValueStorage/BufferValueStorage.php index 0d526da9af..04f00ac38f 100644 --- a/src/lib/parquet/src/Flow/Parquet/Writer/ValueStorage/BufferValueStorage.php +++ b/src/lib/parquet/src/Flow/Parquet/Writer/ValueStorage/BufferValueStorage.php @@ -7,6 +7,7 @@ use Flow\Parquet\Binary\ByteOrder; use Flow\Parquet\BinaryWriter\BinaryBufferWriter; use Flow\Parquet\Data\PlainValuesPacker; +use Flow\Parquet\ParquetFile\Encodings; use Flow\Parquet\ParquetFile\Schema\FlatColumn; use function strlen; @@ -32,6 +33,11 @@ public function addValues(FlatColumn $column, array $values): void $this->size += strlen($localBuffer); } + public function encoding(): Encodings + { + return Encodings::PLAIN; + } + public function getBuffer(): string { return $this->buffer; @@ -48,7 +54,7 @@ public function reset(): void $this->size = 0; } - public function size(): int + public function plainSize(): int { return $this->size; } diff --git a/src/lib/parquet/src/Flow/Parquet/Writer/ValueStorage/DeltaBinaryPackedValueStorage.php b/src/lib/parquet/src/Flow/Parquet/Writer/ValueStorage/DeltaBinaryPackedValueStorage.php index 558beb773a..557463e65e 100644 --- a/src/lib/parquet/src/Flow/Parquet/Writer/ValueStorage/DeltaBinaryPackedValueStorage.php +++ b/src/lib/parquet/src/Flow/Parquet/Writer/ValueStorage/DeltaBinaryPackedValueStorage.php @@ -6,10 +6,10 @@ use Flow\Parquet\Data\DeltaBinaryPackedEncoder; use Flow\Parquet\Exception\InvalidArgumentException; +use Flow\Parquet\ParquetFile\Encodings; use Flow\Parquet\ParquetFile\Schema\FlatColumn; use Flow\Parquet\ParquetFile\Schema\PhysicalType; -use function count; use function gettype; use function sprintf; @@ -20,12 +20,28 @@ final class DeltaBinaryPackedValueStorage implements ValueStorage */ private array $values = []; + private PhysicalType $physicalType = PhysicalType::INT64; + + private int $size = 0; + + public function __construct( + private readonly DeltaBinaryPackedEncoder $int32Encoder = new DeltaBinaryPackedEncoder( + physicalType: PhysicalType::INT32, + ), + private readonly DeltaBinaryPackedEncoder $int64Encoder = new DeltaBinaryPackedEncoder( + physicalType: PhysicalType::INT64, + ), + ) {} + public function addValues(FlatColumn $column, array $values): void { if (!in_array($column->type(), [PhysicalType::INT32, PhysicalType::INT64], true)) { throw new InvalidArgumentException('Delta encoding only supports INT32 and INT64 physical types'); } + $this->physicalType = $column->type(); + $valueSize = $this->physicalType === PhysicalType::INT32 ? 4 : 8; + // @mago-ignore analysis:mixed-assignment foreach ($values as $value) { if ($value !== null) { @@ -37,17 +53,21 @@ public function addValues(FlatColumn $column, array $values): void } $this->values[] = $value; + $this->size += $valueSize; } } } - public function getBuffer(): string + public function encoding(): Encodings { - if (!count($this->values)) { - return ''; - } + return Encodings::DELTA_BINARY_PACKED; + } - return (new DeltaBinaryPackedEncoder())->encode($this->values); + public function getBuffer(): string + { + return ($this->physicalType === PhysicalType::INT32 + ? $this->int32Encoder + : $this->int64Encoder)->encode($this->values); } public function isEmpty(): bool @@ -58,10 +78,11 @@ public function isEmpty(): bool public function reset(): void { $this->values = []; + $this->size = 0; } - public function size(): int + public function plainSize(): int { - return count($this->values); + return $this->size; } } diff --git a/src/lib/parquet/src/Flow/Parquet/Writer/ValueStorage/DeltaByteArrayValueStorage.php b/src/lib/parquet/src/Flow/Parquet/Writer/ValueStorage/DeltaByteArrayValueStorage.php new file mode 100644 index 0000000000..965c052a13 --- /dev/null +++ b/src/lib/parquet/src/Flow/Parquet/Writer/ValueStorage/DeltaByteArrayValueStorage.php @@ -0,0 +1,92 @@ + + */ + private array $values = []; + + public function __construct( + private readonly DeltaByteArrayEncoder $encoder = new DeltaByteArrayEncoder(), + ) {} + + public function addValues(FlatColumn $column, array $values): void + { + if (!in_array($column->type(), [PhysicalType::BYTE_ARRAY, PhysicalType::FIXED_LEN_BYTE_ARRAY], true)) { + throw new InvalidArgumentException( + 'DELTA_BYTE_ARRAY encoding only supports BYTE_ARRAY and FIXED_LEN_BYTE_ARRAY physical types', + ); + } + + $strings = array_filter($values, is_string(...)); + + if ((count($strings) + count(array_filter($values, is_null(...)))) !== count($values)) { + throw new InvalidArgumentException(sprintf( + 'DELTA_BYTE_ARRAY encoding requires string values, got %s', + gettype( + array_values(array_filter( + $values, + static fn(mixed $value): bool => $value !== null && !is_string($value), + ))[0], + ), + )); + } + + // PLAIN prefixes every BYTE_ARRAY value with its 4-byte length, FIXED_LEN_BYTE_ARRAY values with nothing + $lengthPrefix = $column->type() === PhysicalType::BYTE_ARRAY ? 4 : 0; + + array_push($this->values, ...array_values($strings)); + $this->size += (count($strings) * $lengthPrefix) + strlen(implode('', $strings)); + } + + public function encoding(): Encodings + { + return Encodings::DELTA_BYTE_ARRAY; + } + + public function getBuffer(): string + { + return $this->encoder->encode($this->values); + } + + public function isEmpty(): bool + { + return $this->values === []; + } + + public function reset(): void + { + $this->values = []; + $this->size = 0; + } + + public function plainSize(): int + { + return $this->size; + } +} diff --git a/src/lib/parquet/src/Flow/Parquet/Writer/ValueStorage/DeltaLengthByteArrayValueStorage.php b/src/lib/parquet/src/Flow/Parquet/Writer/ValueStorage/DeltaLengthByteArrayValueStorage.php new file mode 100644 index 0000000000..c6cb1d02be --- /dev/null +++ b/src/lib/parquet/src/Flow/Parquet/Writer/ValueStorage/DeltaLengthByteArrayValueStorage.php @@ -0,0 +1,88 @@ + + */ + private array $values = []; + + public function __construct( + private readonly DeltaLengthByteArrayEncoder $encoder = new DeltaLengthByteArrayEncoder(), + ) {} + + public function addValues(FlatColumn $column, array $values): void + { + if ($column->type() !== PhysicalType::BYTE_ARRAY) { + throw new InvalidArgumentException( + 'DELTA_LENGTH_BYTE_ARRAY encoding only supports BYTE_ARRAY physical type', + ); + } + + $strings = array_filter($values, is_string(...)); + + if ((count($strings) + count(array_filter($values, is_null(...)))) !== count($values)) { + throw new InvalidArgumentException(sprintf( + 'DELTA_LENGTH_BYTE_ARRAY encoding requires string values, got %s', + gettype( + array_values(array_filter( + $values, + static fn(mixed $value): bool => $value !== null && !is_string($value), + ))[0], + ), + )); + } + + array_push($this->values, ...array_values($strings)); + $this->size += (count($strings) * 4) + strlen(implode('', $strings)); + } + + public function encoding(): Encodings + { + return Encodings::DELTA_LENGTH_BYTE_ARRAY; + } + + public function getBuffer(): string + { + return $this->encoder->encode($this->values); + } + + public function isEmpty(): bool + { + return $this->values === []; + } + + public function reset(): void + { + $this->values = []; + $this->size = 0; + } + + public function plainSize(): int + { + return $this->size; + } +} diff --git a/src/lib/parquet/src/Flow/Parquet/Writer/ValueStorage/RLEBooleanValueStorage.php b/src/lib/parquet/src/Flow/Parquet/Writer/ValueStorage/RLEBooleanValueStorage.php new file mode 100644 index 0000000000..d953f9c63c --- /dev/null +++ b/src/lib/parquet/src/Flow/Parquet/Writer/ValueStorage/RLEBooleanValueStorage.php @@ -0,0 +1,61 @@ + + */ + private array $bits = []; + + public function __construct( + private readonly RLEBitPackedPacker $packer = new RLEBitPackedPacker(new RLEBitPackedHybrid()), + ) {} + + public function addValues(FlatColumn $column, array $values): void + { + array_push( + $this->bits, + ...array_map(static fn(bool $value): int => $value ? 1 : 0, array_filter($values, is_bool(...))), + ); + } + + public function encoding(): Encodings + { + return Encodings::RLE; + } + + public function getBuffer(): string + { + return $this->packer->packWithLength(1, $this->bits); + } + + public function isEmpty(): bool + { + return $this->bits === []; + } + + public function reset(): void + { + $this->bits = []; + } + + public function plainSize(): int + { + return intdiv(count($this->bits) + 7, 8); + } +} diff --git a/src/lib/parquet/src/Flow/Parquet/Writer/ValueStorage/ValueStorage.php b/src/lib/parquet/src/Flow/Parquet/Writer/ValueStorage/ValueStorage.php index c3b4ec4372..0bcc8df671 100644 --- a/src/lib/parquet/src/Flow/Parquet/Writer/ValueStorage/ValueStorage.php +++ b/src/lib/parquet/src/Flow/Parquet/Writer/ValueStorage/ValueStorage.php @@ -4,6 +4,7 @@ namespace Flow\Parquet\Writer\ValueStorage; +use Flow\Parquet\ParquetFile\Encodings; use Flow\Parquet\ParquetFile\Schema\FlatColumn; interface ValueStorage @@ -13,11 +14,13 @@ interface ValueStorage */ public function addValues(FlatColumn $column, array $values): void; + public function encoding(): Encodings; + public function getBuffer(): string; public function isEmpty(): bool; public function reset(): void; - public function size(): int; + public function plainSize(): int; } diff --git a/src/lib/parquet/tests/Flow/Parquet/Tests/Context/MemoryParquetFile.php b/src/lib/parquet/tests/Flow/Parquet/Tests/Context/MemoryParquetFile.php index 9d33a43ff3..18e8f11a82 100644 --- a/src/lib/parquet/tests/Flow/Parquet/Tests/Context/MemoryParquetFile.php +++ b/src/lib/parquet/tests/Flow/Parquet/Tests/Context/MemoryParquetFile.php @@ -97,25 +97,17 @@ public static function dictionaryColumn(array $values, Options $options): string */ public static function pagedColumns(): string { - return self::written( - Writer::php(options: (new Options())->set(Option::PAGE_SIZE_BYTES, 16)->set( - Option::PAGE_SIZE_CHECK_INTERVAL, - 2, - )->set(Option::COLUMNS_ENCODINGS, ['word' => Encodings::RLE_DICTIONARY])), - Schema::with( - FlatColumn::int64('id'), - FlatColumn::string('word'), - NestedColumn::list('tags', ListElement::int64()), - ), - array_map( - static fn(int $id): array => [ - 'id' => $id, - 'word' => 'w' . ($id % 3), - 'tags' => array_map(static fn(): int => $id, range(0, $id % 3)), - ], - range(0, 11), - ), - ); + return self::written(Writer::php(options: (new Options()) + ->set(Option::DICTIONARY_ENABLED, false) + ->set(Option::PAGE_SIZE_BYTES, 16) + ->set(Option::PAGE_SIZE_CHECK_INTERVAL, 2) + ->set(Option::COLUMNS_ENCODINGS, [ + 'word' => Encodings::RLE_DICTIONARY, + ])), Schema::with(FlatColumn::int64('id'), FlatColumn::string('word'), NestedColumn::list('tags', ListElement::int64())), array_map(static fn(int $id): array => [ + 'id' => $id, + 'word' => 'w' . ($id % 3), + 'tags' => array_map(static fn(): int => $id, range(0, $id % 3)), + ], range(0, 11))); } /** diff --git a/src/lib/parquet/tests/Flow/Parquet/Tests/Integration/IO/DeltaEncodingsTest.php b/src/lib/parquet/tests/Flow/Parquet/Tests/Integration/IO/DeltaEncodingsTest.php new file mode 100644 index 0000000000..9ebdf3fcb0 --- /dev/null +++ b/src/lib/parquet/tests/Flow/Parquet/Tests/Integration/IO/DeltaEncodingsTest.php @@ -0,0 +1,240 @@ + [$writerEngine, $readerEngine]) { + yield "{$pair} v1" => [$writerEngine, $readerEngine, 1]; + yield "{$pair} v2" => [$writerEngine, $readerEngine, 2]; + } + } + + #[DataProvider('engine_pair_provider')] + public function test_a_dictionary_column_of_nulls_round_trips(string $writerEngine, string $readerEngine): void + { + $path = TestParquetFile::path($this); + $rows = [['s' => null], ['s' => null]]; + + (new Writer(options: Options::default()->set(Option::COLUMNS_ENCODINGS, [ + 's' => Encodings::RLE_DICTIONARY, + ]), engine: ParquetEngineMother::create($writerEngine)))->write( + $path, + Schema::with(FlatColumn::string('s')), + $rows, + ); + + static::assertSame($rows, iterator_to_array( + (new Reader(engine: ParquetEngineMother::create($readerEngine))) + ->read($path) + ->values(), + false, + )); + } + + #[DataProvider('engine_pair_writer_version_provider')] + public function test_explicit_delta_string_encodings_round_trip( + string $writerEngine, + string $readerEngine, + int $writerVersion, + ): void { + $path = TestParquetFile::path($this); + $rows = [ + ['dlba' => 'axis', 'dba' => 'axis', 'dbp' => 1, 'required' => 3], + ['dlba' => null, 'dba' => null, 'dbp' => null, 'required' => -2], + ['dlba' => '', 'dba' => 'axle', 'dbp' => -5, 'required' => 7], + ]; + + (new Writer(options: Options::default()->set( + Option::WRITER_VERSION, + $writerVersion, + )->set(Option::COLUMNS_ENCODINGS, [ + 'dlba' => Encodings::DELTA_LENGTH_BYTE_ARRAY, + 'dba' => Encodings::DELTA_BYTE_ARRAY, + 'dbp' => Encodings::DELTA_BINARY_PACKED, + 'required' => Encodings::DELTA_BINARY_PACKED, + ]), engine: ParquetEngineMother::create($writerEngine)))->write( + $path, + Schema::with( + FlatColumn::string('dlba'), + FlatColumn::string('dba'), + FlatColumn::int32('dbp'), + FlatColumn::int64('required', Repetition::REQUIRED), + ), + $rows, + ); + + $file = (new Reader(engine: ParquetEngineMother::create($readerEngine)))->read($path); + + static::assertSame($rows, iterator_to_array($file->values(), false)); + [$dlba, $dba, $dbp, $required] = $file->metadata()->columnChunks(); + static::assertContains(Encodings::DELTA_LENGTH_BYTE_ARRAY, $dlba->encodings()); + static::assertContains(Encodings::DELTA_BYTE_ARRAY, $dba->encodings()); + static::assertContains(Encodings::DELTA_BINARY_PACKED, $dbp->encodings()); + static::assertContains(Encodings::DELTA_BINARY_PACKED, $required->encodings()); + } + + #[DataProvider('engine_provider')] + public function test_reading_delta_byte_array_written_by_arrow_rs(string $engineClass): void + { + static::assertSame( + [ + ['s' => 'axis', 'f' => 'aaaa'], + ['s' => 'axle', 'f' => null], + ['s' => 'babble', 'f' => 'aabb'], + ['s' => 'babyhood', 'f' => 'abcd'], + ['s' => null, 'f' => 'zzzz'], + ], + iterator_to_array( + (new Reader(engine: ParquetEngineMother::create($engineClass))) + ->read(__DIR__ . '/Fixtures/delta_byte_array_arrow_v2.parquet') + ->values(), + false, + ), + ); + } + + #[DataProvider('engine_provider')] + public function test_reading_delta_length_byte_array_written_by_duckdb(string $engineClass): void + { + static::assertSame( + [ + ['s' => 'axis', 'n' => 'a'], + ['s' => 'axle', 'n' => null], + ['s' => 'babble', 'n' => ''], + ['s' => 'babyhood', 'n' => 'b'], + ['s' => '', 'n' => null], + ], + iterator_to_array( + (new Reader(engine: ParquetEngineMother::create($engineClass))) + ->read(__DIR__ . '/Fixtures/delta_length_byte_array_duckdb_v2.parquet') + ->values(), + false, + ), + ); + } + + #[DataProvider('engine_pair_provider')] + public function test_writer_version_2_round_trips_strings_fixed_length_and_ints( + string $writerEngine, + string $readerEngine, + ): void { + $path = TestParquetFile::path($this); + $rows = [ + ['s' => null, 'f' => null, 'i' => null], + ['s' => null, 'f' => null, 'i' => null], + ['s' => 'axis', 'f' => 'aaaa', 'i' => 1], + ['s' => 'axle', 'f' => 'aabb', 'i' => 2], + ['s' => 'babble', 'f' => 'abcd', 'i' => 3], + ]; + + (new Writer( + options: Options::default() + ->set(Option::DICTIONARY_ENABLED, false) + ->set(Option::WRITER_VERSION, 2) + ->set(Option::PAGE_MAXIMUM_ROWS_COUNT, 2) + ->set(Option::PAGE_SIZE_CHECK_INTERVAL, 2), + engine: ParquetEngineMother::create($writerEngine), + ))->write( + $path, + Schema::with(FlatColumn::string('s'), FlatColumn::fixedSizeByteArray('f', 4), FlatColumn::int64('i')), + $rows, + ); + + $file = (new Reader(engine: ParquetEngineMother::create($readerEngine)))->read($path); + + static::assertSame($rows, iterator_to_array($file->values(), false)); + + if ($writerEngine === PhpParquetEngine::class) { + static::assertSame( + [2, 2, 1, 2, 2, 1, 2, 2, 1], + array_map( + static fn(ColumnPageHeader $page): int => $page->pageHeader->dataValuesCount() ?? 0, + iterator_to_array($file->pageHeaders(), false), + ), + ); + static::assertSame( + [ + [Encodings::RLE, Encodings::DELTA_BYTE_ARRAY], + [Encodings::RLE, Encodings::DELTA_BYTE_ARRAY], + [Encodings::RLE, Encodings::DELTA_BINARY_PACKED], + ], + array_map( + static fn(ColumnChunk $chunk): array => $chunk->encodings(), + $file->metadata()->columnChunks(), + ), + ); + } + } + + #[DataProvider('engine_pair_provider')] + public function test_int64_values_with_deltas_near_large_powers_of_two_round_trip( + string $writerEngine, + string $readerEngine, + ): void { + $path = TestParquetFile::path($this); + $rows = array_map(static fn(int $i): array => ['i' => ($i % 2) === 0 ? 0 : (1 << 59) + $i], range(0, 129)); + + (new Writer( + options: Options::default()->set(Option::DICTIONARY_ENABLED, false)->set(Option::WRITER_VERSION, 2), + engine: ParquetEngineMother::create($writerEngine), + ))->write($path, Schema::with(FlatColumn::int64('i')), $rows); + + static::assertSame($rows, iterator_to_array( + (new Reader(engine: ParquetEngineMother::create($readerEngine))) + ->read($path) + ->values(), + false, + )); + } + + #[DataProvider('engine_pair_provider')] + public function test_negative_and_sequential_ints_round_trip_gzip_compressed( + string $writerEngine, + string $readerEngine, + ): void { + $path = TestParquetFile::path($this); + $rows = array_map( + static fn(int $i): array => ['i32' => -$i * 7, 'i64' => 1_700_000_000_000 + $i], + range(0, 199), + ); + + (new Writer( + compression: Compressions::GZIP, + options: Options::default()->set(Option::DICTIONARY_ENABLED, false)->set(Option::WRITER_VERSION, 2), + engine: ParquetEngineMother::create($writerEngine), + ))->write($path, Schema::with(FlatColumn::int32('i32'), FlatColumn::int64('i64')), $rows); + + static::assertSame($rows, iterator_to_array( + (new Reader(engine: ParquetEngineMother::create($readerEngine))) + ->read($path) + ->values(), + false, + )); + } +} diff --git a/src/lib/parquet/tests/Flow/Parquet/Tests/Integration/IO/Fixtures/delta_byte_array_arrow_v2.parquet b/src/lib/parquet/tests/Flow/Parquet/Tests/Integration/IO/Fixtures/delta_byte_array_arrow_v2.parquet new file mode 100644 index 0000000000..a2e35b0433 Binary files /dev/null and b/src/lib/parquet/tests/Flow/Parquet/Tests/Integration/IO/Fixtures/delta_byte_array_arrow_v2.parquet differ diff --git a/src/lib/parquet/tests/Flow/Parquet/Tests/Integration/IO/Fixtures/delta_length_byte_array_duckdb_v2.parquet b/src/lib/parquet/tests/Flow/Parquet/Tests/Integration/IO/Fixtures/delta_length_byte_array_duckdb_v2.parquet new file mode 100644 index 0000000000..9cf8698575 Binary files /dev/null and b/src/lib/parquet/tests/Flow/Parquet/Tests/Integration/IO/Fixtures/delta_length_byte_array_duckdb_v2.parquet differ diff --git a/src/lib/parquet/tests/Flow/Parquet/Tests/Integration/IO/WriterTest.php b/src/lib/parquet/tests/Flow/Parquet/Tests/Integration/IO/WriterTest.php index 6ac0165e15..beba3585a9 100644 --- a/src/lib/parquet/tests/Flow/Parquet/Tests/Integration/IO/WriterTest.php +++ b/src/lib/parquet/tests/Flow/Parquet/Tests/Integration/IO/WriterTest.php @@ -12,14 +12,18 @@ use Flow\Parquet\BloomFilter\BloomHash; use Flow\Parquet\Engine\PhpParquetEngine; use Flow\Parquet\Engine\RustParquetEngine; +use Flow\Parquet\Exception\InvalidArgumentException; use Flow\Parquet\Option; use Flow\Parquet\Options; +use Flow\Parquet\ParquetEngine; use Flow\Parquet\ParquetFile; use Flow\Parquet\ParquetFile\ColumnOrder; +use Flow\Parquet\ParquetFile\Encodings; use Flow\Parquet\ParquetFile\Page\ColumnPageHeader; use Flow\Parquet\ParquetFile\PageIndex\BoundaryOrder; use Flow\Parquet\ParquetFile\PageIndex\ColumnIndex; use Flow\Parquet\ParquetFile\PageIndex\PageLocation; +use Flow\Parquet\ParquetFile\RowGroup\ColumnChunk; use Flow\Parquet\ParquetFile\Schema; use Flow\Parquet\ParquetFile\Schema\ConvertedType; use Flow\Parquet\ParquetFile\Schema\FlatColumn; @@ -42,9 +46,12 @@ use Flow\Parquet\Writer\StatisticsLevel; use Generator; use PHPUnit\Framework\Attributes\DataProvider; +use PHPUnit\Framework\Attributes\RequiresPhpExtension; +use PHPUnit\Framework\Attributes\TestWith; use RuntimeException; use function array_combine; +use function array_filter; use function array_map; use function array_unique; use function array_values; @@ -90,6 +97,125 @@ public function test_a_close_that_throws_leaves_the_writer_closed(string $engine $writer->close(); } + #[RequiresPhpExtension('arrow')] + #[TestWith([1, true])] + #[TestWith([2, true])] + #[TestWith([1, false])] + #[TestWith([2, false])] + public function test_both_engines_write_the_same_default_encodings( + int $writerVersion, + bool $dictionaryEnabled, + ): void { + $schema = Schema::with( + FlatColumn::boolean('boolean'), + FlatColumn::int32('int32'), + FlatColumn::int64('int64'), + FlatColumn::float('float'), + FlatColumn::double('double'), + FlatColumn::string('string'), + FlatColumn::fixedSizeByteArray('flba', 4), + ); + $rows = [ + [ + 'boolean' => true, + 'int32' => 1, + 'int64' => 1, + 'float' => 1.5, + 'double' => 1.5, + 'string' => 'a', + 'flba' => 'aaaa', + ], + [ + 'boolean' => null, + 'int32' => null, + 'int64' => null, + 'float' => null, + 'double' => null, + 'string' => null, + 'flba' => null, + ], + [ + 'boolean' => false, + 'int32' => 1, + 'int64' => 2, + 'float' => 1.5, + 'double' => 2.5, + 'string' => 'a', + 'flba' => 'bbbb', + ], + ]; + $options = Options::default()->set(Option::WRITER_VERSION, $writerVersion)->set( + Option::DICTIONARY_ENABLED, + $dictionaryEnabled, + ); + $php = MemoryParquetFile::read(Reader::php(), MemoryParquetFile::written( + new Writer(options: $options, engine: new PhpParquetEngine()), + $schema, + $rows, + )); + $arrow = MemoryParquetFile::read(Reader::php(), MemoryParquetFile::written( + new Writer(options: $options, engine: new RustParquetEngine()), + $schema, + $rows, + )); + $encodings = static fn(ParquetFile $file): array => array_map(static fn(ColumnChunk $chunk): array => array_map( + static fn(Encodings $encoding): string => $encoding->name, + $chunk->encodings(), + ), $file->metadata()->columnChunks()); + + foreach ($encodings($php) as $index => $chunkEncodings) { + static::assertEqualsCanonicalizing($encodings($arrow)[$index], $chunkEncodings, "column {$index}"); + } + + static::assertCount(7, $encodings($php)); + static::assertSame(iterator_to_array($arrow->values(), false), iterator_to_array($php->values(), false)); + } + + public function test_a_write_that_throws_leaves_no_file(): void + { + $path = TestParquetFile::path($this); + + try { + Writer::php(options: Options::default()->set(Option::COLUMNS_ENCODINGS, [ + 's' => Encodings::DELTA_BINARY_PACKED, + ]))->write($path, Schema::with(FlatColumn::string('s')), [['s' => 'a']]); + static::fail('write() was expected to throw'); + } catch (InvalidArgumentException) { + } + + static::assertFileDoesNotExist($path); + } + + public function test_an_open_that_throws_leaves_no_file(): void + { + $path = TestParquetFile::path($this); + + try { + Writer::php(options: Options::default()->set(Option::COLUMNS_ENCODINGS, [ + 's' => Encodings::DELTA_BINARY_PACKED, + ]))->open($path, Schema::with(FlatColumn::string('s'))); + static::fail('open() was expected to throw'); + } catch (InvalidArgumentException) { + } + + static::assertFileDoesNotExist($path); + } + + public function test_opening_a_stream_with_invalid_options_writes_nothing_to_it(): void + { + $memory = memory_filesystem(); + + try { + Writer::php(options: Options::default()->set(Option::COLUMNS_ENCODINGS, [ + 's' => Encodings::DELTA_BINARY_PACKED, + ]))->openForStream($memory->writeTo(path('memory://file.parquet')), Schema::with(FlatColumn::string('s'))); + static::fail('openForStream() was expected to throw'); + } catch (InvalidArgumentException) { + } + + static::assertSame('', $memory->readFrom(path('memory://file.parquet'))->content()); + } + #[DataProvider('engine_provider')] public function test_closing_not_open_writer(string $engineClass): void { @@ -536,6 +662,10 @@ public function test_writing_data_page_v2_statistics(): void foreach ((new Reader(options: $options, engine: new PhpParquetEngine())) ->read($path) ->pageHeaders() as $pageHeader) { + if (!$pageHeader->pageHeader->type()->isDataPage()) { + continue; + } + $dataPageHeaderV2 = $pageHeader->pageHeader->dataPageHeaderV2(); static::assertNotNull($dataPageHeaderV2); $statistics = $dataPageHeaderV2->statistics(); @@ -726,11 +856,28 @@ public function test_writing_to_file(string $engineClass): void static::assertFileExists($path); } - #[DataProvider('engine_provider')] - public function test_writing_to_file_v2(string $engineClass): void + /** + * @return Generator, bool}> + */ + public static function engine_dictionary_provider(): Generator + { + foreach (self::engine_provider() as $engine => [$engineClass]) { + yield "{$engine} dictionary" => [$engineClass, true]; + yield "{$engine} without dictionary" => [$engineClass, false]; + } + } + + #[DataProvider('engine_dictionary_provider')] + public function test_writing_to_file_v2(string $engineClass, bool $dictionaryEnabled): void { $engine = ParquetEngineMother::create($engineClass); - $writer = new Writer(options: Options::default()->set(Option::WRITER_VERSION, 2), engine: $engine); + $writer = new Writer( + options: Options::default()->set(Option::WRITER_VERSION, 2)->set( + Option::DICTIONARY_ENABLED, + $dictionaryEnabled, + ), + engine: $engine, + ); $path = TestParquetFile::path($this); @@ -1056,14 +1203,17 @@ public function test_page_header_statistics_are_written_only_when_asked(): void { $headers = static fn(Options $options): array => array_map( static fn(ColumnPageHeader $header) => $header->pageHeader->dataPageHeader()?->statistics(), - iterator_to_array( - MemoryParquetFile::read(Reader::php(), MemoryParquetFile::written( - Writer::php(options: $options), - Schema::with(FlatColumn::int64('id')), - [['id' => 1], ['id' => 2]], - ))->pageHeaders(), - false, - ), + array_values(array_filter( + iterator_to_array( + MemoryParquetFile::read(Reader::php(), MemoryParquetFile::written( + Writer::php(options: $options), + Schema::with(FlatColumn::int64('id')), + [['id' => 1], ['id' => 2]], + ))->pageHeaders(), + false, + ), + static fn(ColumnPageHeader $header): bool => $header->pageHeader->type()->isDataPage(), + )), ); static::assertSame([null], $headers(new Options())); diff --git a/src/lib/parquet/tests/Flow/Parquet/Tests/Integration/Writer/ColumnChunkBuilder/DeltaBinaryPackedColumnChunkBuilderTest.php b/src/lib/parquet/tests/Flow/Parquet/Tests/Integration/Writer/ColumnChunkBuilder/DeltaBinaryPackedColumnChunkBuilderTest.php deleted file mode 100644 index 1fe024e5d9..0000000000 --- a/src/lib/parquet/tests/Flow/Parquet/Tests/Integration/Writer/ColumnChunkBuilder/DeltaBinaryPackedColumnChunkBuilderTest.php +++ /dev/null @@ -1,240 +0,0 @@ -set(Option::WRITER_VERSION, 2); - $column = new FlatColumn('negative_col', PhysicalType::INT32); - $builder = new DeltaBinaryPackedColumnChunkBuilder( - $column, - $options, - Compressions::UNCOMPRESSED, - StatisticsOptions::fromOptions($options), - ); - - $negativeValues = [-10, -8, -6, -4, -2, 0, 2, 4, 6, 8]; - - foreach ($negativeValues as $value) { - $builder->addColumn(new WriteFlatColumnValues($column, [0], [1], [$value])); - } - - $containers = $builder->flush(0); - - static::assertCount(1, $containers); - static::assertEquals(0, $builder->uncompressedSize()); - - $container = $containers[0]; - static::assertNotEmpty($container->binaryBuffer); - static::assertGreaterThan(0, strlen($container->binaryBuffer)); - } - - public function test_delta_encoding_with_sequential_values(): void - { - $options = (new Options())->set(Option::WRITER_VERSION, 2); - $column = new FlatColumn('sequential_col', PhysicalType::INT32); - $builder = new DeltaBinaryPackedColumnChunkBuilder( - $column, - $options, - Compressions::UNCOMPRESSED, - StatisticsOptions::fromOptions($options), - ); - - $sequentialValues = range(1, 100); - - foreach ($sequentialValues as $value) { - $builder->addColumn(new WriteFlatColumnValues($column, [0], [1], [$value])); - } - - $containers = $builder->flush(0); - - static::assertCount(1, $containers); - static::assertEquals(0, $builder->uncompressedSize()); - - $container = $containers[0]; - static::assertNotEmpty($container->binaryBuffer); - static::assertGreaterThan(0, strlen($container->binaryBuffer)); - } - - public function test_delta_encoding_with_timestamp_sequence(): void - { - $options = (new Options())->set(Option::WRITER_VERSION, 2); - $column = new FlatColumn('timestamp_col', PhysicalType::INT64); - $builder = new DeltaBinaryPackedColumnChunkBuilder( - $column, - $options, - Compressions::UNCOMPRESSED, - StatisticsOptions::fromOptions($options), - ); - - $baseTimestamp = 1609459200; // 2021-01-01 00:00:00 - - for ($i = 0; $i < 100; $i++) { - $timestamp = $baseTimestamp + ($i * 60); - $builder->addColumn(new WriteFlatColumnValues($column, [0], [1], [$timestamp])); - } - - $containers = $builder->flush(0); - - static::assertCount(1, $containers); - static::assertEquals(0, $builder->uncompressedSize()); - - $container = $containers[0]; - static::assertNotEmpty($container->binaryBuffer); - static::assertGreaterThan(0, strlen($container->binaryBuffer)); - } - - public function test_delta_encoding_workflow(): void - { - $options = (new Options())->set(Option::WRITER_VERSION, 2); - $column = new FlatColumn('workflow_col', PhysicalType::INT32); - $builder = new DeltaBinaryPackedColumnChunkBuilder( - $column, - $options, - Compressions::UNCOMPRESSED, - StatisticsOptions::fromOptions($options), - ); - - $values = range(1, 50); - - foreach ($values as $value) { - $builder->addColumn(new WriteFlatColumnValues($column, [0], [1], [$value])); - } - - static::assertFalse($builder->isFull()); - - $containers = $builder->flush(0); - static::assertCount(1, $containers); - - $uncompressedSize = $builder->uncompressedSize(); - static::assertEquals(0, $uncompressedSize); - - $container = $containers[0]; - static::assertNotEmpty($container->binaryBuffer); - static::assertNotNull($container->columnChunk); - static::assertSame($column->type(), $container->columnChunk->type()); - } - - public function test_round_trip_int32_sequential_values(): void - { - $column = new FlatColumn('test_col', PhysicalType::INT32); - $options = (new Options())->set(Option::WRITER_VERSION, 2); - $builder = new DeltaBinaryPackedColumnChunkBuilder( - $column, - $options, - Compressions::UNCOMPRESSED, - StatisticsOptions::fromOptions($options), - ); - - $values = [1, 2, 3, 4, 5, 6, 7, 8, 9, 10]; - - foreach ($values as $value) { - $builder->addColumn(new WriteFlatColumnValues($column, [0], [1], [$value])); - } - - $containers = $builder->flush(0); - $container = $containers[0]; - - static::assertNotNull($container); - static::assertGreaterThan(0, strlen($container->binaryBuffer)); - static::assertSame($column->type(), $container->columnChunk->type()); - - $encodings = $container->columnChunk->encodings(); - static::assertContains(Encodings::DELTA_BINARY_PACKED, $encodings); - } - - public function test_round_trip_int64_timestamp_sequence(): void - { - $column = new FlatColumn('timestamp_col', PhysicalType::INT64); - $options = (new Options())->set(Option::WRITER_VERSION, 2); - $builder = new DeltaBinaryPackedColumnChunkBuilder( - $column, - $options, - Compressions::UNCOMPRESSED, - StatisticsOptions::fromOptions($options), - ); - - $baseTimestamp = 1609459200; // 2021-01-01 00:00:00 - $values = []; - - for ($i = 0; $i < 10; $i++) { - $values[] = $baseTimestamp + ($i * 60); // Every minute - } - - foreach ($values as $value) { - $builder->addColumn(new WriteFlatColumnValues($column, [0], [1], [$value])); - } - - $containers = $builder->flush(0); - $container = $containers[0]; - - static::assertNotNull($container); - static::assertGreaterThan(0, strlen($container->binaryBuffer)); - static::assertSame($column->type(), $container->columnChunk->type()); - } - - public function test_round_trip_negative_values(): void - { - $column = new FlatColumn('negative_col', PhysicalType::INT32); - $options = (new Options())->set(Option::WRITER_VERSION, 2); - $builder = new DeltaBinaryPackedColumnChunkBuilder( - $column, - $options, - Compressions::UNCOMPRESSED, - StatisticsOptions::fromOptions($options), - ); - - $values = [-10, -8, -6, -4, -2, 0, 2, 4, 6, 8]; - - foreach ($values as $value) { - $builder->addColumn(new WriteFlatColumnValues($column, [0], [1], [$value])); - } - - $containers = $builder->flush(0); - $container = $containers[0]; - - static::assertNotNull($container); - static::assertGreaterThan(0, strlen($container->binaryBuffer)); - static::assertSame($column->type(), $container->columnChunk->type()); - } - - public function test_round_trip_with_different_compression(): void - { - $column = new FlatColumn('compressed_col', PhysicalType::INT32); - $options = (new Options())->set(Option::WRITER_VERSION, 2); - $builder = new DeltaBinaryPackedColumnChunkBuilder( - $column, - $options, - Compressions::GZIP, - StatisticsOptions::fromOptions($options), - ); - - $values = range(100, 200); - - foreach ($values as $value) { - $builder->addColumn(new WriteFlatColumnValues($column, [0], [1], [$value])); - } - - $containers = $builder->flush(0); - $container = $containers[0]; - - static::assertNotNull($container); - static::assertGreaterThan(0, strlen($container->binaryBuffer)); - static::assertSame($column->type(), $container->columnChunk->type()); - } -} diff --git a/src/lib/parquet/tests/Flow/Parquet/Tests/Integration/Writer/ColumnChunkBuilder/RLEDictionaryChunkBuilderTest.php b/src/lib/parquet/tests/Flow/Parquet/Tests/Integration/Writer/ColumnChunkBuilder/RLEDictionaryChunkBuilderTest.php index 20ed597d87..b9ba0e96da 100644 --- a/src/lib/parquet/tests/Flow/Parquet/Tests/Integration/Writer/ColumnChunkBuilder/RLEDictionaryChunkBuilderTest.php +++ b/src/lib/parquet/tests/Flow/Parquet/Tests/Integration/Writer/ColumnChunkBuilder/RLEDictionaryChunkBuilderTest.php @@ -74,7 +74,7 @@ public function test_the_chunk_lists_both_encodings_after_a_fallback_to_plain(): $file->metadata()->rowGroups()->all()[0]->columnChunks()[0]->encodings(), ); static::assertSame( - ['DICTIONARY_PAGE', 'RLE_DICTIONARY', 'RLE_DICTIONARY', 'PLAIN'], + ['DICTIONARY_PAGE', 'RLE_DICTIONARY', 'PLAIN', 'PLAIN'], array_map( static fn(ColumnPageHeader $header): string => $header->pageHeader->type()->isDataPage() ? $header->pageHeader->dataPageHeader()?->encoding()->name ?? '' diff --git a/src/lib/parquet/tests/Flow/Parquet/Tests/Unit/Data/BitWidthTest.php b/src/lib/parquet/tests/Flow/Parquet/Tests/Unit/Data/BitWidthTest.php new file mode 100644 index 0000000000..7e7ac6bcf4 --- /dev/null +++ b/src/lib/parquet/tests/Flow/Parquet/Tests/Unit/Data/BitWidthTest.php @@ -0,0 +1,32 @@ +encode($values1); $encoded2 = $encoder->encode($values2); - $decoded1 = $decoder->decode($encoded1, count($values1)); - $decoded2 = $decoder->decode($encoded2, count($values2)); + $decoded1 = $decoder->decode(new BinaryBufferReader($encoded1), count($values1)); + $decoded2 = $decoder->decode(new BinaryBufferReader($encoded2), count($values2)); static::assertSame($values1, $decoded1); static::assertSame($values2, $decoded2); @@ -62,13 +64,13 @@ public function test_decode_empty_data_with_non_zero_count_throws_error(): void $this->expectException(RuntimeException::class); $this->expectExceptionMessage('Cannot decode empty data when value count is greater than 0'); - $decoder->decode('', 5); + $decoder->decode(new BinaryBufferReader(''), 5); } public function test_decode_empty_data_with_zero_count(): void { $decoder = new DeltaBinaryPackedDecoder(); - $result = $decoder->decode('', 0); + $result = $decoder->decode(new BinaryBufferReader(''), 0); static::assertSame([], $result); } @@ -85,7 +87,7 @@ public function test_decode_large_dataset(): void } $encoded = $encoder->encode($values); - $decoded = $decoder->decode($encoded, count($values)); + $decoded = $decoder->decode(new BinaryBufferReader($encoded), count($values)); // For now, let's just test the first 128 values to ensure basic functionality works static::assertSame(array_slice($values, 0, 128), array_slice($decoded, 0, 128)); @@ -101,11 +103,42 @@ public function test_decode_large_values(): void $values = [1000000, 1000001, 1000002, 1000003, 1000004, 1000005, 1000006, 1000007]; $encoded = $encoder->encode($values); - $decoded = $decoder->decode($encoded, count($values)); + $decoded = $decoder->decode(new BinaryBufferReader($encoded), count($values)); static::assertSame($values, $decoded); } + #[TestWith([3])] + #[TestWith([129])] + #[TestWith([200])] + public function test_decode_leaves_the_reader_at_the_end_of_its_stream(int $count): void + { + $reader = new BinaryBufferReader((new DeltaBinaryPackedEncoder())->encode(range(1, $count)) . 'tail'); + + static::assertSame(range(1, $count), (new DeltaBinaryPackedDecoder())->decode($reader, $count)); + static::assertSame('tail', $reader->readBytes(4)); + } + + #[TestWith([2])] + #[TestWith([8])] + #[TestWith([9])] + #[TestWith([32])] + #[TestWith([33])] + #[TestWith([56])] + #[TestWith([57])] + #[TestWith([61])] + #[TestWith([62])] + #[TestWith([63])] + public function test_decode_round_trips_every_miniblock_bit_width(int $bitWidth): void + { + $values = array_map(static fn(int $i): int => ($i % 2) === 0 ? 0 : 1 << ($bitWidth - 2), range(0, 129)); + + static::assertSame($values, (new DeltaBinaryPackedDecoder())->decode( + new BinaryBufferReader((new DeltaBinaryPackedEncoder())->encode($values)), + count($values), + )); + } + public function test_decode_negative_values(): void { $encoder = new DeltaBinaryPackedEncoder(); @@ -113,7 +146,7 @@ public function test_decode_negative_values(): void $values = [-10, -8, -6, -4, -2, 0, 2, 4]; $encoded = $encoder->encode($values); - $decoded = $decoder->decode($encoded, count($values)); + $decoded = $decoder->decode(new BinaryBufferReader($encoded), count($values)); static::assertSame($values, $decoded); } @@ -126,8 +159,8 @@ public function test_decode_produces_consistent_output(): void $values = [1, 3, 5, 7, 9, 11, 13, 15]; $encoded = $encoder->encode($values); - $decoded1 = $decoder->decode($encoded, count($values)); - $decoded2 = $decoder->decode($encoded, count($values)); + $decoded1 = $decoder->decode(new BinaryBufferReader($encoded), count($values)); + $decoded2 = $decoder->decode(new BinaryBufferReader($encoded), count($values)); static::assertSame($decoded1, $decoded2); static::assertSame($values, $decoded1); @@ -140,7 +173,7 @@ public function test_decode_random_pattern(): void $values = [10, 15, 12, 18, 14, 20, 16, 22]; $encoded = $encoder->encode($values); - $decoded = $decoder->decode($encoded, count($values)); + $decoded = $decoder->decode(new BinaryBufferReader($encoded), count($values)); static::assertSame($values, $decoded); } @@ -152,7 +185,7 @@ public function test_decode_sequential_values(): void $values = [1, 2, 3, 4, 5, 6, 7, 8]; $encoded = $encoder->encode($values); - $decoded = $decoder->decode($encoded, count($values)); + $decoded = $decoder->decode(new BinaryBufferReader($encoded), count($values)); static::assertSame($values, $decoded); } @@ -164,7 +197,7 @@ public function test_decode_single_value(): void $values = [42]; $encoded = $encoder->encode($values); - $decoded = $decoder->decode($encoded, 1); + $decoded = $decoder->decode(new BinaryBufferReader($encoded), 1); static::assertSame($values, $decoded); } @@ -182,7 +215,7 @@ public function test_decode_timestamp_like_sequence(): void } $encoded = $encoder->encode($values); - $decoded = $decoder->decode($encoded, count($values)); + $decoded = $decoder->decode(new BinaryBufferReader($encoded), count($values)); static::assertSame($values, $decoded); } @@ -198,7 +231,7 @@ public function test_decode_value_count_mismatch_throws_error(): void $this->expectException(RuntimeException::class); $this->expectExceptionMessage('Value count mismatch: expected 10, got 5'); - $decoder->decode($encoded, 10); + $decoder->decode(new BinaryBufferReader($encoded), 10); } public function test_decode_values_with_negative_deltas(): void @@ -208,7 +241,7 @@ public function test_decode_values_with_negative_deltas(): void $values = [100, 90, 80, 70, 60, 50, 40, 30]; $encoded = $encoder->encode($values); - $decoded = $decoder->decode($encoded, count($values)); + $decoded = $decoder->decode(new BinaryBufferReader($encoded), count($values)); static::assertSame($values, $decoded); } @@ -220,7 +253,7 @@ public function test_decode_values_with_varying_deltas(): void $values = [100, 102, 98, 105, 95, 110, 90, 115]; $encoded = $encoder->encode($values); - $decoded = $decoder->decode($encoded, count($values)); + $decoded = $decoder->decode(new BinaryBufferReader($encoded), count($values)); static::assertSame($values, $decoded); } @@ -232,7 +265,7 @@ public function test_decode_values_with_zero_deltas(): void $values = [42, 42, 42, 42, 42, 42, 42, 42]; $encoded = $encoder->encode($values); - $decoded = $decoder->decode($encoded, count($values)); + $decoded = $decoder->decode(new BinaryBufferReader($encoded), count($values)); static::assertSame($values, $decoded); } @@ -244,7 +277,7 @@ public function test_decode_with_custom_block_size(): void $values = range(1, 100); $encoded = $encoder->encode($values); - $decoded = $decoder->decode($encoded, count($values)); + $decoded = $decoder->decode(new BinaryBufferReader($encoded), count($values)); static::assertSame($values, $decoded); } diff --git a/src/lib/parquet/tests/Flow/Parquet/Tests/Unit/Data/DeltaBinaryPackedEncoderTest.php b/src/lib/parquet/tests/Flow/Parquet/Tests/Unit/Data/DeltaBinaryPackedEncoderTest.php index d59a95b2d6..074cf045ee 100644 --- a/src/lib/parquet/tests/Flow/Parquet/Tests/Unit/Data/DeltaBinaryPackedEncoderTest.php +++ b/src/lib/parquet/tests/Flow/Parquet/Tests/Unit/Data/DeltaBinaryPackedEncoderTest.php @@ -51,7 +51,7 @@ public function test_encode_empty_array(): void $encoder = new DeltaBinaryPackedEncoder(); $result = $encoder->encode([]); - static::assertSame('', $result); + static::assertSame("\x80\x01\x04\x00\x00", $result); } public function test_encode_extreme_values(): void diff --git a/src/lib/parquet/tests/Flow/Parquet/Tests/Unit/Data/DeltaBinaryPackedEncodingRoundtripTest.php b/src/lib/parquet/tests/Flow/Parquet/Tests/Unit/Data/DeltaBinaryPackedEncodingRoundtripTest.php index 885ab4c88f..2ddf10c7ec 100644 --- a/src/lib/parquet/tests/Flow/Parquet/Tests/Unit/Data/DeltaBinaryPackedEncodingRoundtripTest.php +++ b/src/lib/parquet/tests/Flow/Parquet/Tests/Unit/Data/DeltaBinaryPackedEncodingRoundtripTest.php @@ -4,6 +4,7 @@ namespace Flow\Parquet\Tests\Unit\Data; +use Flow\Parquet\BinaryReader\BinaryBufferReader; use Flow\Parquet\Data\DeltaBinaryPackedDecoder; use Flow\Parquet\Data\DeltaBinaryPackedEncoder; use PHPUnit\Framework\Attributes\DataProvider; @@ -35,7 +36,7 @@ public static function problematicInt64ValuesProvider(): array public function test_roundtrip_problematic_int64_values(array $values): void { $encoded = (new DeltaBinaryPackedEncoder())->encode($values); - $decoded = (new DeltaBinaryPackedDecoder())->decode($encoded, count($values)); + $decoded = (new DeltaBinaryPackedDecoder())->decode(new BinaryBufferReader($encoded), count($values)); static::assertSame($values, $decoded, 'Values should roundtrip correctly through delta encoding'); } diff --git a/src/lib/parquet/tests/Flow/Parquet/Tests/Unit/Data/DeltaBinaryPackedInt64Test.php b/src/lib/parquet/tests/Flow/Parquet/Tests/Unit/Data/DeltaBinaryPackedInt64Test.php index ff240732ac..8c7fe9a9e3 100644 --- a/src/lib/parquet/tests/Flow/Parquet/Tests/Unit/Data/DeltaBinaryPackedInt64Test.php +++ b/src/lib/parquet/tests/Flow/Parquet/Tests/Unit/Data/DeltaBinaryPackedInt64Test.php @@ -4,6 +4,7 @@ namespace Flow\Parquet\Tests\Unit\Data; +use Flow\Parquet\BinaryReader\BinaryBufferReader; use Flow\Parquet\Data\DeltaBinaryPackedDecoder; use Flow\Parquet\Data\DeltaBinaryPackedEncoder; use PHPUnit\Framework\Attributes\DataProvider; @@ -59,7 +60,7 @@ public function test_delta_encoding_with_known_working_values(): void foreach ($workingValues as $values) { $encoded = (new DeltaBinaryPackedEncoder())->encode($values); - $decoded = (new DeltaBinaryPackedDecoder())->decode($encoded, count($values)); + $decoded = (new DeltaBinaryPackedDecoder())->decode(new BinaryBufferReader($encoded), count($values)); static::assertSame( $values, @@ -76,7 +77,7 @@ public function test_delta_encoding_with_known_working_values(): void public function test_int64_delta_encoding_roundtrip(array $values): void { $encoded = (new DeltaBinaryPackedEncoder())->encode($values); - $decoded = (new DeltaBinaryPackedDecoder())->decode($encoded, count($values)); + $decoded = (new DeltaBinaryPackedDecoder())->decode(new BinaryBufferReader($encoded), count($values)); $testName = $this->dataName(); diff --git a/src/lib/parquet/tests/Flow/Parquet/Tests/Unit/Data/DeltaByteArrayDecoderTest.php b/src/lib/parquet/tests/Flow/Parquet/Tests/Unit/Data/DeltaByteArrayDecoderTest.php new file mode 100644 index 0000000000..d7c0d2c07a --- /dev/null +++ b/src/lib/parquet/tests/Flow/Parquet/Tests/Unit/Data/DeltaByteArrayDecoderTest.php @@ -0,0 +1,79 @@ +encode(['ab', 'abc']) . 'tail'); + + static::assertSame(['ab', 'abc'], (new DeltaByteArrayDecoder())->decode($reader, 2)); + static::assertSame('tail', $reader->readBytes(4)); + } + + public function test_decode_round_trips_values_crossing_block_and_miniblock_edges(): void + { + $values = array_map(static fn(int $i): string => sprintf('key_%05d', $i), range(1, 300)); + + static::assertSame($values, (new DeltaByteArrayDecoder())->decode( + new BinaryBufferReader((new DeltaByteArrayEncoder())->encode($values)), + 300, + )); + } + + public function test_decode_spec_example(): void + { + $encoder = new DeltaBinaryPackedEncoder(); + + static::assertSame( + ['axis', 'axle', 'babble', 'babyhood'], + (new DeltaByteArrayDecoder())->decode(new BinaryBufferReader($encoder->encode([0, 2, 0, 3]) + . $encoder->encode([4, 2, 6, 5]) + . 'axislebabbleyhood'), 4), + ); + } + + public function test_decode_throws_on_a_prefix_longer_than_the_previous_value(): void + { + $encoder = new DeltaBinaryPackedEncoder(); + + $this->expectException(RuntimeException::class); + $this->expectExceptionMessage( + 'Invalid DELTA_BYTE_ARRAY prefix length 5 of value 1, the previous value has 2 bytes', + ); + + (new DeltaByteArrayDecoder())->decode( + new BinaryBufferReader($encoder->encode([0, 5]) . $encoder->encode([2, 1]) . 'abc'), + 2, + ); + } + + public function test_decode_throws_on_a_negative_prefix_length(): void + { + $encoder = new DeltaBinaryPackedEncoder(); + + $this->expectException(RuntimeException::class); + $this->expectExceptionMessage( + 'Invalid DELTA_BYTE_ARRAY prefix length -1 of value 0, the previous value has 0 bytes', + ); + + (new DeltaByteArrayDecoder())->decode( + new BinaryBufferReader($encoder->encode([-1]) . $encoder->encode([1]) . 'a'), + 1, + ); + } +} diff --git a/src/lib/parquet/tests/Flow/Parquet/Tests/Unit/Data/DeltaByteArrayEncoderTest.php b/src/lib/parquet/tests/Flow/Parquet/Tests/Unit/Data/DeltaByteArrayEncoderTest.php new file mode 100644 index 0000000000..078403cbe0 --- /dev/null +++ b/src/lib/parquet/tests/Flow/Parquet/Tests/Unit/Data/DeltaByteArrayEncoderTest.php @@ -0,0 +1,53 @@ +, list, list}> + */ + public static function values_provider(): array + { + return [ + 'spec example' => [ + ['axis', 'axle', 'babble', 'babyhood'], + [0, 2, 0, 3], + ['axis', 'le', 'babble', 'yhood'], + ], + 'repeated value' => [['a', 'a'], [0, 1], ['a', '']], + 'shorter value under a full prefix' => [['abc', 'ab'], [0, 2], ['abc', '']], + 'nul bytes in the prefix' => [["\0\0a", "\0\0b"], [0, 2], ["\0\0a", 'b']], + 'prefix stopping at a nul byte' => [["a\0", 'a'], [0, 1], ["a\0", '']], + 'empty first value' => [['', 'a'], [0, 0], ['', 'a']], + ]; + } + + /** + * @param list $values + * @param list $prefixLengths + * @param list $suffixes + */ + #[DataProvider('values_provider')] + public function test_encode(array $values, array $prefixLengths, array $suffixes): void + { + $encoder = new DeltaBinaryPackedEncoder(); + + static::assertSame( + $encoder->encode($prefixLengths) . $encoder->encode(array_map(strlen(...), $suffixes)) + . implode('', $suffixes), + (new DeltaByteArrayEncoder())->encode($values), + ); + } +} diff --git a/src/lib/parquet/tests/Flow/Parquet/Tests/Unit/Data/DeltaCalculatorTest.php b/src/lib/parquet/tests/Flow/Parquet/Tests/Unit/Data/DeltaCalculatorTest.php index 8a9ee085bc..d90072f87b 100644 --- a/src/lib/parquet/tests/Flow/Parquet/Tests/Unit/Data/DeltaCalculatorTest.php +++ b/src/lib/parquet/tests/Flow/Parquet/Tests/Unit/Data/DeltaCalculatorTest.php @@ -5,6 +5,7 @@ namespace Flow\Parquet\Tests\Unit\Data; use Flow\Parquet\Data\DeltaCalculator; +use PHPUnit\Framework\Attributes\TestWith; use PHPUnit\Framework\TestCase; final class DeltaCalculatorTest extends TestCase @@ -337,4 +338,59 @@ public function test_roundtrip_preserves_values_with_extreme_cases(): void ); } } + + /** + * @param list $deltas + * @param list $expected + */ + #[TestWith([[5, 7, 9], 5, [0, 2, 4]])] + #[TestWith([[2305843009213693952, -1], -1, [2305843009213693953, 0]])] + #[TestWith([[2305843009213693953, -1], -1, [2305843009213693954, 0]])] + #[TestWith([[PHP_INT_MAX, PHP_INT_MIN], PHP_INT_MIN, [-1, 0]])] + public function test_calculate_relative_deltas(array $deltas, int $minDelta, array $expected): void + { + static::assertSame($expected, (new DeltaCalculator())->calculateRelativeDeltas($deltas, $minDelta)); + } + + /** + * @param list $relativeDeltas + * @param list $expected + */ + #[TestWith([[0, 2, 4], 5, [5, 7, 9]])] + #[TestWith([[2305843009213693952, 0], 1, [2305843009213693953, 1]])] + #[TestWith([[2305843009213693953, 0], 1, [2305843009213693954, 1]])] + #[TestWith([[-1, 0], PHP_INT_MIN, [PHP_INT_MAX, PHP_INT_MIN]])] + public function test_calculate_absolute_deltas(array $relativeDeltas, int $minDelta, array $expected): void + { + static::assertSame($expected, (new DeltaCalculator())->calculateAbsoluteDeltas($relativeDeltas, $minDelta)); + } + + public function test_calculate_absolute_and_relative_deltas_of_nothing(): void + { + static::assertSame([], (new DeltaCalculator())->calculateAbsoluteDeltas([], 3)); + static::assertSame([], (new DeltaCalculator())->calculateRelativeDeltas([], 3)); + } + + /** + * @param list $values + * @param list $expected + */ + #[TestWith([[2305843009213693952, -2305843009213693952], [-4611686018427387904]])] + #[TestWith([[2305843009213693953, -2305843009213693953], [-4611686018427387906]])] + public function test_calculate_deltas_at_the_overflow_free_bound(array $values, array $expected): void + { + static::assertSame($expected, (new DeltaCalculator())->calculateDeltas($values)); + } + + #[TestWith([0, 0])] + #[TestWith([2147483647, 2147483647])] + #[TestWith([2147483648, -2147483648])] + #[TestWith([-2147483649, 2147483647])] + #[TestWith([4294967295, -1])] + #[TestWith([PHP_INT_MAX, -1])] + #[TestWith([PHP_INT_MIN, 0])] + public function test_to_int32_wraps_to_signed_32_bits(int $value, int $expected): void + { + static::assertSame($expected, (new DeltaCalculator())->toInt32($value)); + } } diff --git a/src/lib/parquet/tests/Flow/Parquet/Tests/Unit/Data/DeltaLengthByteArrayDecoderTest.php b/src/lib/parquet/tests/Flow/Parquet/Tests/Unit/Data/DeltaLengthByteArrayDecoderTest.php new file mode 100644 index 0000000000..6e1936322c --- /dev/null +++ b/src/lib/parquet/tests/Flow/Parquet/Tests/Unit/Data/DeltaLengthByteArrayDecoderTest.php @@ -0,0 +1,48 @@ +encode([5, 5, 6, 6]) + . 'HelloWorldFoobarABCDEFtail'); + + static::assertSame( + ['Hello', 'World', 'Foobar', 'ABCDEF'], + (new DeltaLengthByteArrayDecoder())->decode($reader, 4), + ); + static::assertSame('tail', $reader->readBytes(4)); + } + + public function test_decode_throws_on_a_negative_length(): void + { + $this->expectException(RuntimeException::class); + $this->expectExceptionMessage('Invalid DELTA_LENGTH_BYTE_ARRAY length -1 of value 0'); + + (new DeltaLengthByteArrayDecoder())->decode( + new BinaryBufferReader((new DeltaBinaryPackedEncoder())->encode([-1]) . 'x'), + 1, + ); + } + + public function test_decode_throws_when_the_data_ends_early(): void + { + $this->expectException(RuntimeException::class); + $this->expectExceptionMessage('DELTA_LENGTH_BYTE_ARRAY data ends before value 0'); + + (new DeltaLengthByteArrayDecoder())->decode( + new BinaryBufferReader((new DeltaBinaryPackedEncoder())->encode([5]) . 'abc'), + 1, + ); + } +} diff --git a/src/lib/parquet/tests/Flow/Parquet/Tests/Unit/Data/DeltaLengthByteArrayEncoderTest.php b/src/lib/parquet/tests/Flow/Parquet/Tests/Unit/Data/DeltaLengthByteArrayEncoderTest.php new file mode 100644 index 0000000000..2efc287e21 --- /dev/null +++ b/src/lib/parquet/tests/Flow/Parquet/Tests/Unit/Data/DeltaLengthByteArrayEncoderTest.php @@ -0,0 +1,33 @@ +encode([])); + } + + public function test_encode_empty_strings(): void + { + static::assertSame( + (new DeltaBinaryPackedEncoder())->encode([0, 1, 0]) . 'a', + (new DeltaLengthByteArrayEncoder())->encode(['', 'a', '']), + ); + } + + public function test_encode_spec_example(): void + { + static::assertSame( + (new DeltaBinaryPackedEncoder())->encode([5, 5, 6, 6]) . 'HelloWorldFoobarABCDEF', + (new DeltaLengthByteArrayEncoder())->encode(['Hello', 'World', 'Foobar', 'ABCDEF']), + ); + } +} diff --git a/src/lib/parquet/tests/Flow/Parquet/Tests/Unit/Engine/Arrow/OptionsConverterTest.php b/src/lib/parquet/tests/Flow/Parquet/Tests/Unit/Engine/Arrow/OptionsConverterTest.php index e7281fd97a..6eae3c545b 100644 --- a/src/lib/parquet/tests/Flow/Parquet/Tests/Unit/Engine/Arrow/OptionsConverterTest.php +++ b/src/lib/parquet/tests/Flow/Parquet/Tests/Unit/Engine/Arrow/OptionsConverterTest.php @@ -38,6 +38,16 @@ public function test_engine_options_are_read_from_the_options(): void ); } + public function test_dictionary_enabled_is_mapped(): void + { + static::assertFalse( + OptionsConverter::toExtension((new Options())->set( + Option::DICTIONARY_ENABLED, + false, + ))['DICTIONARY_ENABLED'], + ); + } + public function test_page_maximum_rows_count_is_mapped(): void { static::assertSame( @@ -150,6 +160,7 @@ public function test_default_options_produces_expected_keys(): void static::assertArrayHasKey('ZSTD_COMPRESSION_LEVEL', $result); static::assertArrayHasKey('ROW_GROUP_SIZE_BYTES', $result); static::assertArrayHasKey('PAGE_SIZE_BYTES', $result); + static::assertTrue($result['DICTIONARY_ENABLED']); static::assertArrayHasKey('DICTIONARY_PAGE_SIZE', $result); static::assertArrayHasKey('WRITER_VERSION', $result); static::assertArrayNotHasKey('COLUMNS_COMPRESSIONS', $result); diff --git a/src/lib/parquet/tests/Flow/Parquet/Tests/Unit/ParquetFile/Page/Header/DataPageHeaderV2Test.php b/src/lib/parquet/tests/Flow/Parquet/Tests/Unit/ParquetFile/Page/Header/DataPageHeaderV2Test.php new file mode 100644 index 0000000000..f160951da8 --- /dev/null +++ b/src/lib/parquet/tests/Flow/Parquet/Tests/Unit/ParquetFile/Page/Header/DataPageHeaderV2Test.php @@ -0,0 +1,24 @@ +isCompressed(), + ); + } +} diff --git a/src/lib/parquet/tests/Flow/Parquet/Tests/Unit/ParquetFileTest.php b/src/lib/parquet/tests/Flow/Parquet/Tests/Unit/ParquetFileTest.php index 87a8a6a7e2..1319786287 100644 --- a/src/lib/parquet/tests/Flow/Parquet/Tests/Unit/ParquetFileTest.php +++ b/src/lib/parquet/tests/Flow/Parquet/Tests/Unit/ParquetFileTest.php @@ -324,7 +324,7 @@ public function test_page_headers_are_each_column_chunk_pages_labelled_with_its_ )); static::assertSame( - [['id', 'id'], ['name', 'name']], + [['id', 'id'], ['id', 'id'], ['name', 'name'], ['name', 'name']], array_map( static fn(ColumnPageHeader $header): array => [ $header->column->flatPath(), diff --git a/src/lib/parquet/tests/Flow/Parquet/Tests/Unit/Reader/ColumnDataDecoderTest.php b/src/lib/parquet/tests/Flow/Parquet/Tests/Unit/Reader/ColumnDataDecoderTest.php new file mode 100644 index 0000000000..3ebec75057 --- /dev/null +++ b/src/lib/parquet/tests/Flow/Parquet/Tests/Unit/Reader/ColumnDataDecoderTest.php @@ -0,0 +1,117 @@ + + */ + public static function delta_string_pages_provider(): array + { + return [ + 'DELTA_LENGTH_BYTE_ARRAY' => [ + Encodings::DELTA_LENGTH_BYTE_ARRAY, + (new DeltaLengthByteArrayEncoder())->encode(['axis', 'axle', '']), + ], + 'DELTA_BYTE_ARRAY' => [ + Encodings::DELTA_BYTE_ARRAY, + (new DeltaByteArrayEncoder())->encode(['axis', 'axle', '']), + ], + ]; + } + + #[DataProvider('delta_string_pages_provider')] + public function test_delta_string_page_v1_decodes(Encodings $encoding, string $buffer): void + { + static::assertSame( + ['axis', 'axle', ''], + iterator_to_array( + (new ColumnDataDecoder()) + ->decodeData( + $buffer, + FlatColumn::string('s', Repetition::REQUIRED), + new DataPageHeader($encoding, Encodings::RLE, Encodings::RLE, 3), + ) + ->rawValues(), + false, + ), + ); + } + + #[DataProvider('delta_string_pages_provider')] + public function test_delta_string_page_v2_decodes(Encodings $encoding, string $buffer): void + { + static::assertSame( + ['axis', 'axle', ''], + iterator_to_array( + (new ColumnDataDecoder()) + ->decodeDataV2( + $buffer, + FlatColumn::string('s', Repetition::REQUIRED), + new DataPageHeaderV2(3, 0, 3, $encoding, 0, 0, false, null), + ) + ->rawValues(), + false, + ), + ); + } + + public function test_an_rle_boolean_page_v1_decodes(): void + { + $column = FlatColumn::boolean('b', Repetition::REQUIRED); + $storage = new RLEBooleanValueStorage(); + $storage->addValues($column, [true, false, true]); + + static::assertSame( + [true, false, true], + iterator_to_array( + (new ColumnDataDecoder()) + ->decodeData( + $storage->getBuffer(), + $column, + new DataPageHeader(Encodings::RLE, Encodings::RLE, Encodings::RLE, 3), + ) + ->rawValues(), + false, + ), + ); + } + + public function test_an_rle_boolean_page_v2_decodes(): void + { + $column = FlatColumn::boolean('b', Repetition::REQUIRED); + $storage = new RLEBooleanValueStorage(); + $storage->addValues($column, [true, false, true]); + + static::assertSame( + [true, false, true], + iterator_to_array( + (new ColumnDataDecoder()) + ->decodeDataV2( + $storage->getBuffer(), + $column, + new DataPageHeaderV2(3, 0, 3, Encodings::RLE, 0, 0, false, null), + ) + ->rawValues(), + false, + ), + ); + } +} diff --git a/src/lib/parquet/tests/Flow/Parquet/Tests/Unit/Reader/PageValuesDecoderTest.php b/src/lib/parquet/tests/Flow/Parquet/Tests/Unit/Reader/PageValuesDecoderTest.php new file mode 100644 index 0000000000..51511305dd --- /dev/null +++ b/src/lib/parquet/tests/Flow/Parquet/Tests/Unit/Reader/PageValuesDecoderTest.php @@ -0,0 +1,226 @@ +decode( + $reader, + FlatColumn::string('s', Repetition::REQUIRED), + Encodings::RLE_DICTIONARY, + 0, + new Dictionary(['a']), + ), + false, + ), + ); + static::assertSame('tail', $reader->readBytes(4)); + } + + public function test_decodes_delta_binary_packed_int32_wrapping_at_32_bits(): void + { + $values = [-2147483648, 2147483647, 0, -1, 1]; + + static::assertSame($values, iterator_to_array( + (new PageValuesDecoder())->decode( + new BinaryBufferReader((new DeltaBinaryPackedEncoder(physicalType: PhysicalType::INT32))->encode( + $values, + )), + FlatColumn::int32('i', Repetition::REQUIRED), + Encodings::DELTA_BINARY_PACKED, + 5, + ), + false, + )); + } + + public function test_decodes_delta_byte_array(): void + { + static::assertSame( + ['axis', 'axle'], + iterator_to_array( + (new PageValuesDecoder())->decode( + new BinaryBufferReader((new DeltaByteArrayEncoder())->encode(['axis', 'axle'])), + FlatColumn::fixedSizeByteArray('f', 4, Repetition::REQUIRED), + Encodings::DELTA_BYTE_ARRAY, + 2, + ), + false, + ), + ); + } + + public function test_decodes_delta_length_byte_array(): void + { + static::assertSame( + ['a', ''], + iterator_to_array( + (new PageValuesDecoder())->decode( + new BinaryBufferReader((new DeltaLengthByteArrayEncoder())->encode(['a', ''])), + FlatColumn::string('s', Repetition::REQUIRED), + Encodings::DELTA_LENGTH_BYTE_ARRAY, + 2, + ), + false, + ), + ); + } + + public function test_decodes_dictionary_indices_to_values_and_unknown_indices_to_null(): void + { + static::assertSame( + ['b', 'a', null], + iterator_to_array( + (new PageValuesDecoder())->decode( + new BinaryBufferReader((new RLEBitPackedPacker(new RLEBitPackedHybrid()))->packWithBitWidth(2, [ + 1, + 0, + 2, + ])), + FlatColumn::string('s', Repetition::REQUIRED), + Encodings::RLE_DICTIONARY, + 3, + new Dictionary(['a', 'b']), + ), + false, + ), + ); + } + + public function test_decodes_plain(): void + { + $column = FlatColumn::int32('i', Repetition::REQUIRED); + $storage = new BufferValueStorage(); + $storage->addValues($column, [7, -3]); + + static::assertSame( + [7, -3], + iterator_to_array( + (new PageValuesDecoder())->decode( + new BinaryBufferReader($storage->getBuffer()), + $column, + Encodings::PLAIN, + 2, + ), + false, + ), + ); + } + + public function test_decodes_rle_booleans(): void + { + $column = FlatColumn::boolean('b', Repetition::REQUIRED); + $storage = new RLEBooleanValueStorage(); + $storage->addValues($column, [true, false, true]); + + static::assertSame( + [true, false, true], + iterator_to_array( + (new PageValuesDecoder())->decode( + new BinaryBufferReader($storage->getBuffer()), + $column, + Encodings::RLE, + 3, + ), + false, + ), + ); + } + + public function test_delta_binary_packed_rejects_a_string_column(): void + { + $this->expectException(RuntimeException::class); + $this->expectExceptionMessage('Delta encoding only supports INT32 and INT64 physical types'); + + iterator_to_array((new PageValuesDecoder())->decode( + new BinaryBufferReader((new DeltaBinaryPackedEncoder())->encode([1])), + FlatColumn::string('s', Repetition::REQUIRED), + Encodings::DELTA_BINARY_PACKED, + 1, + )); + } + + public function test_delta_byte_array_rejects_a_fixed_len_byte_array_value_of_the_wrong_length(): void + { + $this->expectException(RuntimeException::class); + $this->expectExceptionMessage( + "DELTA_BYTE_ARRAY value 1 of FIXED_LEN_BYTE_ARRAY column 'f' has 3 bytes, expected 4", + ); + + iterator_to_array((new PageValuesDecoder())->decode( + new BinaryBufferReader((new DeltaByteArrayEncoder())->encode(['abcd', 'abc'])), + FlatColumn::fixedSizeByteArray('f', 4, Repetition::REQUIRED), + Encodings::DELTA_BYTE_ARRAY, + 2, + )); + } + + public function test_delta_byte_array_rejects_an_int32_column(): void + { + $this->expectException(RuntimeException::class); + $this->expectExceptionMessage( + 'DELTA_BYTE_ARRAY encoding only supports BYTE_ARRAY and FIXED_LEN_BYTE_ARRAY physical types', + ); + + iterator_to_array((new PageValuesDecoder())->decode( + new BinaryBufferReader((new DeltaByteArrayEncoder())->encode(['a'])), + FlatColumn::int32('i', Repetition::REQUIRED), + Encodings::DELTA_BYTE_ARRAY, + 1, + )); + } + + public function test_delta_length_byte_array_rejects_an_int32_column(): void + { + $this->expectException(RuntimeException::class); + $this->expectExceptionMessage('DELTA_LENGTH_BYTE_ARRAY encoding only supports BYTE_ARRAY physical type'); + + iterator_to_array((new PageValuesDecoder())->decode( + new BinaryBufferReader((new DeltaLengthByteArrayEncoder())->encode(['a'])), + FlatColumn::int32('i', Repetition::REQUIRED), + Encodings::DELTA_LENGTH_BYTE_ARRAY, + 1, + )); + } + + public function test_rle_on_a_non_boolean_column_is_not_supported(): void + { + $this->expectException(RuntimeException::class); + $this->expectExceptionMessage('Encoding RLE not supported'); + + iterator_to_array((new PageValuesDecoder())->decode( + new BinaryBufferReader("\x00\x00\x00\x00"), + FlatColumn::int32('i', Repetition::REQUIRED), + Encodings::RLE, + 0, + )); + } +} diff --git a/src/lib/parquet/tests/Flow/Parquet/Tests/Unit/Writer/ColumnChunkBuilder/DeltaBinaryPackedColumnChunkBuilderTest.php b/src/lib/parquet/tests/Flow/Parquet/Tests/Unit/Writer/ColumnChunkBuilder/DeltaBinaryPackedColumnChunkBuilderTest.php deleted file mode 100644 index 5d2e82764b..0000000000 --- a/src/lib/parquet/tests/Flow/Parquet/Tests/Unit/Writer/ColumnChunkBuilder/DeltaBinaryPackedColumnChunkBuilderTest.php +++ /dev/null @@ -1,270 +0,0 @@ - [Compressions::UNCOMPRESSED]; - yield 'gzip' => [Compressions::GZIP]; - yield 'snappy' => [Compressions::SNAPPY]; - } - - public static function page_size_provider(): Generator - { - yield 'small page' => [1024]; - yield 'medium page' => [8192]; - yield 'large page' => [65536]; - } - - public static function physical_types_provider(): Generator - { - yield 'int32' => [PhysicalType::INT32, 42]; - yield 'int64' => [PhysicalType::INT64, 1234567890123]; - } - - public static function writer_version_provider(): Generator - { - yield 'version 1' => [1]; - yield 'version 2' => [2]; - } - - public function test_constructor_rejects_unsupported_types(): void - { - $this->expectException(InvalidArgumentException::class); - $this->expectExceptionMessage('Delta encoding only supports INT32 and INT64 physical types'); - - new DeltaBinaryPackedColumnChunkBuilder( - new FlatColumn('test_col', PhysicalType::FLOAT), - new Options(), - Compressions::UNCOMPRESSED, - StatisticsOptions::fromOptions(new Options()), - ); - } - - public function test_empty_builder_properties(): void - { - $options = (new Options())->set(Option::WRITER_VERSION, 2); - $column = new FlatColumn('test_col', PhysicalType::INT32); - $builder = new DeltaBinaryPackedColumnChunkBuilder( - $column, - $options, - Compressions::UNCOMPRESSED, - StatisticsOptions::fromOptions($options), - ); - - static::assertSame($column, $builder->column()); - static::assertFalse($builder->isFull()); - static::assertSame(0, $builder->uncompressedSize()); - } - - public function test_flush_cleans_up_builder_state(): void - { - $column = new FlatColumn('test_col', PhysicalType::INT32); - $options = new Options(); - $compression = Compressions::UNCOMPRESSED; - $builder = new DeltaBinaryPackedColumnChunkBuilder( - $column, - $options, - $compression, - StatisticsOptions::fromOptions($options), - ); - - $builder->addColumn(new WriteFlatColumnValues($column, [0, 0], [1, 1], [42, 84])); - - static::assertFalse($builder->isEmpty()); - static::assertGreaterThan(0, $builder->uncompressedSize()); - - $containers = $builder->flush(0); - static::assertCount(1, $containers); - - static::assertTrue($builder->isEmpty()); - static::assertEquals(0, $builder->uncompressedSize()); - - $builder->addColumn(new WriteFlatColumnValues($column, [0], [1], [126])); - - static::assertFalse($builder->isEmpty()); - static::assertGreaterThan(0, $builder->uncompressedSize()); - } - - public function test_flush_empty_builder(): void - { - $options = (new Options())->set(Option::WRITER_VERSION, 2); - $column = new FlatColumn('test_col', PhysicalType::INT32); - $builder = new DeltaBinaryPackedColumnChunkBuilder( - $column, - $options, - Compressions::UNCOMPRESSED, - StatisticsOptions::fromOptions($options), - ); - - $containers = $builder->flush(0); - - static::assertIsArray($containers); - static::assertCount(1, $containers); - static::assertInstanceOf(ColumnChunkContainer::class, $containers[0]); - } - - public function test_flush_with_data(): void - { - $options = (new Options())->set(Option::WRITER_VERSION, 2); - $column = new FlatColumn('test_col', PhysicalType::INT32); - $builder = new DeltaBinaryPackedColumnChunkBuilder( - $column, - $options, - Compressions::UNCOMPRESSED, - StatisticsOptions::fromOptions($options), - ); - - $values = [1, 2, 3, 4, 5]; - - foreach ($values as $value) { - $builder->addColumn(new WriteFlatColumnValues($column, [0], [1], [$value])); - } - - $containers = $builder->flush(0); - - static::assertIsArray($containers); - static::assertCount(1, $containers); - static::assertInstanceOf(ColumnChunkContainer::class, $containers[0]); - static::assertEquals(0, $builder->uncompressedSize()); - } - - public function test_is_full_calculation_for_int32(): void - { - $options = (new Options())->set(Option::PAGE_SIZE_BYTES, 100)->set(Option::WRITER_VERSION, 2); - - $column = new FlatColumn('test_col', PhysicalType::INT32); - $builder = new DeltaBinaryPackedColumnChunkBuilder( - $column, - $options, - Compressions::UNCOMPRESSED, - StatisticsOptions::fromOptions($options), - ); - - for ($i = 0; $i < 30; $i++) { - $builder->addColumn(new WriteFlatColumnValues($column, [0], [1], [$i])); - } - - static::assertTrue($builder->isFull()); - } - - public function test_is_full_calculation_for_int64(): void - { - $options = (new Options())->set(Option::PAGE_SIZE_BYTES, 100)->set(Option::WRITER_VERSION, 2); - - $column = new FlatColumn('test_col', PhysicalType::INT64); - $builder = new DeltaBinaryPackedColumnChunkBuilder( - $column, - $options, - Compressions::UNCOMPRESSED, - StatisticsOptions::fromOptions($options), - ); - - for ($i = 0; $i < 15; $i++) { - $builder->addColumn(new WriteFlatColumnValues($column, [0], [1], [$i])); - } - - static::assertTrue($builder->isFull()); - } - - #[DataProvider('compression_types_provider')] - public function test_supports_different_compression_types(Compressions $compression): void - { - $options = (new Options())->set(Option::WRITER_VERSION, 2); - $column = new FlatColumn('test_col', PhysicalType::INT32); - $builder = new DeltaBinaryPackedColumnChunkBuilder( - $column, - $options, - $compression, - StatisticsOptions::fromOptions($options), - ); - - static::assertSame($column, $builder->column()); - static::assertInstanceOf(DeltaBinaryPackedColumnChunkBuilder::class, $builder); - } - - #[DataProvider('writer_version_provider')] - public function test_supports_different_writer_versions(int $version): void - { - $options = (new Options())->set(Option::WRITER_VERSION, $version); - - $column = new FlatColumn('test_col', PhysicalType::INT32); - $builder = new DeltaBinaryPackedColumnChunkBuilder( - $column, - $options, - Compressions::UNCOMPRESSED, - StatisticsOptions::fromOptions($options), - ); - - static::assertSame($column, $builder->column()); - static::assertInstanceOf(DeltaBinaryPackedColumnChunkBuilder::class, $builder); - } - - #[DataProvider('physical_types_provider')] - public function test_supports_integer_types(PhysicalType $type, int $value): void - { - $options = (new Options())->set(Option::WRITER_VERSION, 2); - $column = new FlatColumn('test_col', $type); - $builder = new DeltaBinaryPackedColumnChunkBuilder( - $column, - $options, - Compressions::UNCOMPRESSED, - StatisticsOptions::fromOptions($options), - ); - - static::assertSame($column, $builder->column()); - static::assertInstanceOf(DeltaBinaryPackedColumnChunkBuilder::class, $builder); - } - - public function test_each_page_gets_a_column_index_entry_and_an_offset_index_location(): void - { - $column = FlatColumn::int64('id'); - $builder = new DeltaBinaryPackedColumnChunkBuilder( - $column, - new Options(), - Compressions::UNCOMPRESSED, - StatisticsOptions::fromOptions(new Options()), - ); - $builder->addColumn(new WriteFlatColumnValues($column, [0, 0], [1, 1], [1, 2])); - $builder->closePage(); - $builder->addColumn(new WriteFlatColumnValues($column, [0], [0], [])); - $container = $builder->flush(4)[0]; - $last = $container->offsetIndex->pages[1]; - - static::assertEquals( - new ColumnIndex([false, true], [pack('P', 1), ''], [pack('P', 2), ''], BoundaryOrder::ASCENDING, [0, 1]), - $container->columnIndex, - ); - static::assertSame( - [0, 2], - array_map(static fn(PageLocation $page): int => $page->firstRowIndex, $container->offsetIndex->pages), - ); - static::assertSame($container->columnChunk->dataPageOffset(), $container->offsetIndex->pages[0]->offset); - static::assertSame(4 + strlen($container->binaryBuffer), $last->offset + $last->compressedPageSize); - } -} diff --git a/src/lib/parquet/tests/Flow/Parquet/Tests/Unit/Writer/ColumnChunkBuilder/PlainFlatColumnChunkBuilderTest.php b/src/lib/parquet/tests/Flow/Parquet/Tests/Unit/Writer/ColumnChunkBuilder/FlatColumnChunkBuilderTest.php similarity index 79% rename from src/lib/parquet/tests/Flow/Parquet/Tests/Unit/Writer/ColumnChunkBuilder/PlainFlatColumnChunkBuilderTest.php rename to src/lib/parquet/tests/Flow/Parquet/Tests/Unit/Writer/ColumnChunkBuilder/FlatColumnChunkBuilderTest.php index ab8c8c2be4..96b451d7f4 100644 --- a/src/lib/parquet/tests/Flow/Parquet/Tests/Unit/Writer/ColumnChunkBuilder/PlainFlatColumnChunkBuilderTest.php +++ b/src/lib/parquet/tests/Flow/Parquet/Tests/Unit/Writer/ColumnChunkBuilder/FlatColumnChunkBuilderTest.php @@ -14,9 +14,15 @@ use Flow\Parquet\ParquetFile\Schema\FlatColumn; use Flow\Parquet\ParquetFile\Schema\LogicalType; use Flow\Parquet\ParquetFile\Schema\PhysicalType; -use Flow\Parquet\Writer\ColumnChunkBuilder\PlainFlatColumnChunkBuilder; +use Flow\Parquet\Writer\ColumnChunkBuilder\FlatColumnChunkBuilder; use Flow\Parquet\Writer\ColumnChunkContainer; use Flow\Parquet\Writer\StatisticsOptions; +use Flow\Parquet\Writer\ValueStorage\BooleanValueStorage; +use Flow\Parquet\Writer\ValueStorage\BufferValueStorage; +use Flow\Parquet\Writer\ValueStorage\DeltaBinaryPackedValueStorage; +use Flow\Parquet\Writer\ValueStorage\DeltaByteArrayValueStorage; +use Flow\Parquet\Writer\ValueStorage\DeltaLengthByteArrayValueStorage; +use Flow\Parquet\Writer\ValueStorage\ValueStorage; use Generator; use PHPUnit\Framework\Attributes\DataProvider; use PHPUnit\Framework\TestCase; @@ -26,7 +32,7 @@ use function pack; use function strlen; -final class PlainFlatColumnChunkBuilderTest extends TestCase +final class FlatColumnChunkBuilderTest extends TestCase { public static function compression_types_provider(): Generator { @@ -52,6 +58,49 @@ public static function physical_types_provider(): Generator yield 'byte_array' => [PhysicalType::BYTE_ARRAY, 'test_string']; } + public static function delta_binary_packed_is_full_provider(): Generator + { + yield 'int32' => [PhysicalType::INT32, 25]; + yield 'int64' => [PhysicalType::INT64, 13]; + } + + public static function page_index_storages_provider(): Generator + { + yield 'PLAIN' => [new BufferValueStorage()]; + yield 'DELTA_BINARY_PACKED' => [new DeltaBinaryPackedValueStorage()]; + } + + public static function storage_encodings_provider(): Generator + { + foreach ([1, 2] as $writerVersion) { + yield "PLAIN v{$writerVersion}" => [FlatColumn::int32('c'), new BufferValueStorage(), 7, $writerVersion]; + yield "PLAIN boolean v{$writerVersion}" => [ + FlatColumn::boolean('c'), + new BooleanValueStorage(), + true, + $writerVersion, + ]; + yield "DELTA_BINARY_PACKED v{$writerVersion}" => [ + FlatColumn::int64('c'), + new DeltaBinaryPackedValueStorage(), + 7, + $writerVersion, + ]; + yield "DELTA_LENGTH_BYTE_ARRAY v{$writerVersion}" => [ + FlatColumn::string('c'), + new DeltaLengthByteArrayValueStorage(), + 'a', + $writerVersion, + ]; + yield "DELTA_BYTE_ARRAY v{$writerVersion}" => [ + FlatColumn::string('c'), + new DeltaByteArrayValueStorage(), + 'a', + $writerVersion, + ]; + } + } + public static function writer_version_provider(): Generator { yield 'version 1' => [1]; @@ -63,11 +112,12 @@ public function test_add_multiple_rows(): void $column = new FlatColumn('test_col', PhysicalType::INT32); $options = new Options(); $compression = Compressions::UNCOMPRESSED; - $builder = new PlainFlatColumnChunkBuilder( + $builder = new FlatColumnChunkBuilder( $column, $options, $compression, StatisticsOptions::fromOptions($options), + new BufferValueStorage(), ); for ($i = 0; $i < 5; $i++) { @@ -83,11 +133,12 @@ public function test_add_row_with_boolean_values(): void $column = new FlatColumn('bool_col', PhysicalType::BOOLEAN); $options = new Options(); $compression = Compressions::UNCOMPRESSED; - $builder = new PlainFlatColumnChunkBuilder( + $builder = new FlatColumnChunkBuilder( $column, $options, $compression, StatisticsOptions::fromOptions($options), + new BooleanValueStorage(), ); $builder->addColumn(new WriteFlatColumnValues($column, [0, 0], [1, 1], [true, false])); @@ -101,11 +152,12 @@ public function test_add_row_with_empty_data(): void $column = new FlatColumn('test_col', PhysicalType::INT32); $options = new Options(); $compression = Compressions::UNCOMPRESSED; - $builder = new PlainFlatColumnChunkBuilder( + $builder = new FlatColumnChunkBuilder( $column, $options, $compression, StatisticsOptions::fromOptions($options), + new BufferValueStorage(), ); $builder->addColumn(new WriteFlatColumnValues($column, [], [], [])); @@ -119,11 +171,12 @@ public function test_add_row_with_extreme_values(): void $column = new FlatColumn('test_col', PhysicalType::INT64); $options = new Options(); $compression = Compressions::UNCOMPRESSED; - $builder = new PlainFlatColumnChunkBuilder( + $builder = new FlatColumnChunkBuilder( $column, $options, $compression, StatisticsOptions::fromOptions($options), + new BufferValueStorage(), ); $builder->addColumn(new WriteFlatColumnValues($column, [0, 0], [1, 1], [PHP_INT_MAX, PHP_INT_MIN])); @@ -140,11 +193,12 @@ public function test_add_row_with_multiple_values(): void $column = new FlatColumn('test_col', PhysicalType::INT32); $options = new Options(); $compression = Compressions::UNCOMPRESSED; - $builder = new PlainFlatColumnChunkBuilder( + $builder = new FlatColumnChunkBuilder( $column, $options, $compression, StatisticsOptions::fromOptions($options), + new BufferValueStorage(), ); $builder->addColumn(new WriteFlatColumnValues($column, [0, 0, 0], [1, 1, 0], [42, 100])); @@ -158,11 +212,12 @@ public function test_add_row_with_null_values(): void $column = new FlatColumn('test_col', PhysicalType::INT32); $options = new Options(); $compression = Compressions::UNCOMPRESSED; - $builder = new PlainFlatColumnChunkBuilder( + $builder = new FlatColumnChunkBuilder( $column, $options, $compression, StatisticsOptions::fromOptions($options), + new BufferValueStorage(), ); $builder->addColumn(new WriteFlatColumnValues($column, [0], [0], [])); @@ -176,11 +231,12 @@ public function test_add_row_with_repetition_levels(): void $column = new FlatColumn('test_col', PhysicalType::INT32); $options = new Options(); $compression = Compressions::UNCOMPRESSED; - $builder = new PlainFlatColumnChunkBuilder( + $builder = new FlatColumnChunkBuilder( $column, $options, $compression, StatisticsOptions::fromOptions($options), + new BufferValueStorage(), ); $builder->addColumn(new WriteFlatColumnValues($column, [0, 1], [1, 1], [42, 100])); @@ -197,11 +253,12 @@ public function test_add_row_with_single_value(): void $column = new FlatColumn('test_col', PhysicalType::INT32); $options = new Options(); $compression = Compressions::UNCOMPRESSED; - $builder = new PlainFlatColumnChunkBuilder( + $builder = new FlatColumnChunkBuilder( $column, $options, $compression, StatisticsOptions::fromOptions($options), + new BufferValueStorage(), ); $builder->addColumn(new WriteFlatColumnValues($column, [0], [1], [42])); @@ -215,11 +272,12 @@ public function test_add_row_with_string_values(): void $column = new FlatColumn('str_col', PhysicalType::BYTE_ARRAY, logicalType: LogicalType::string()); $options = new Options(); $compression = Compressions::UNCOMPRESSED; - $builder = new PlainFlatColumnChunkBuilder( + $builder = new FlatColumnChunkBuilder( $column, $options, $compression, StatisticsOptions::fromOptions($options), + new BufferValueStorage(), ); $builder->addColumn(new WriteFlatColumnValues($column, [0, 0], [1, 1], ['hello', 'world'])); @@ -233,11 +291,12 @@ public function test_add_row_with_very_large_definition_levels(): void $column = new FlatColumn('test_col', PhysicalType::INT32); $options = new Options(); $compression = Compressions::UNCOMPRESSED; - $builder = new PlainFlatColumnChunkBuilder( + $builder = new FlatColumnChunkBuilder( $column, $options, $compression, StatisticsOptions::fromOptions($options), + new BufferValueStorage(), ); $builder->addColumn(new WriteFlatColumnValues($column, [0], [255], [42])); @@ -254,11 +313,12 @@ public function test_close_page_resets_internal_state(): void $column = new FlatColumn('test_col', PhysicalType::INT32); $options = new Options(); $compression = Compressions::UNCOMPRESSED; - $builder = new PlainFlatColumnChunkBuilder( + $builder = new FlatColumnChunkBuilder( $column, $options, $compression, StatisticsOptions::fromOptions($options), + new BufferValueStorage(), ); $builder->addColumn(new WriteFlatColumnValues($column, [0], [1], [42])); @@ -275,11 +335,12 @@ public function test_close_page_with_data(): void $column = new FlatColumn('test_col', PhysicalType::INT32); $options = new Options(); $compression = Compressions::UNCOMPRESSED; - $builder = new PlainFlatColumnChunkBuilder( + $builder = new FlatColumnChunkBuilder( $column, $options, $compression, StatisticsOptions::fromOptions($options), + new BufferValueStorage(), ); $builder->addColumn(new WriteFlatColumnValues($column, [0], [1], [42])); @@ -297,11 +358,12 @@ public function test_close_page_with_different_writer_versions(int $writerVersio $options = new Options(); $options->set(Option::WRITER_VERSION, $writerVersion); $compression = Compressions::UNCOMPRESSED; - $builder = new PlainFlatColumnChunkBuilder( + $builder = new FlatColumnChunkBuilder( $column, $options, $compression, StatisticsOptions::fromOptions($options), + new BufferValueStorage(), ); $builder->addColumn(new WriteFlatColumnValues($column, [0], [1], [42])); @@ -316,11 +378,12 @@ public function test_close_page_with_empty_data(): void $column = new FlatColumn('test_col', PhysicalType::INT32); $options = new Options(); $compression = Compressions::UNCOMPRESSED; - $builder = new PlainFlatColumnChunkBuilder( + $builder = new FlatColumnChunkBuilder( $column, $options, $compression, StatisticsOptions::fromOptions($options), + new BufferValueStorage(), ); $builder->closePage(); @@ -335,11 +398,12 @@ public function test_close_page_with_unsupported_writer_version_throws_exception $options = new Options(); $options->set(Option::WRITER_VERSION, 3); $compression = Compressions::UNCOMPRESSED; - $builder = new PlainFlatColumnChunkBuilder( + $builder = new FlatColumnChunkBuilder( $column, $options, $compression, StatisticsOptions::fromOptions($options), + new BufferValueStorage(), ); $builder->addColumn(new WriteFlatColumnValues($column, [0], [1], [42])); @@ -357,11 +421,12 @@ public function test_column_returns_correct_column(): void $column = new FlatColumn('test_col', PhysicalType::INT32); $options = new Options(); $compression = Compressions::UNCOMPRESSED; - $builder = new PlainFlatColumnChunkBuilder( + $builder = new FlatColumnChunkBuilder( $column, $options, $compression, StatisticsOptions::fromOptions($options), + new BufferValueStorage(), ); $result = $builder->column(); @@ -375,50 +440,35 @@ public function test_constructor_initializes_correctly(): void $options = new Options(); $compression = Compressions::UNCOMPRESSED; - $builder = new PlainFlatColumnChunkBuilder( + $builder = new FlatColumnChunkBuilder( $column, $options, $compression, StatisticsOptions::fromOptions($options), + new BufferValueStorage(), ); - static::assertInstanceOf(PlainFlatColumnChunkBuilder::class, $builder); + static::assertInstanceOf(FlatColumnChunkBuilder::class, $builder); static::assertSame($column, $builder->column()); static::assertFalse($builder->isFull()); static::assertSame(0, $builder->uncompressedSize()); } - public function test_constructor_with_boolean_column_uses_boolean_storage(): void - { - $column = new FlatColumn('bool_col', PhysicalType::BOOLEAN); - $options = new Options(); - $compression = Compressions::UNCOMPRESSED; - - $builder = new PlainFlatColumnChunkBuilder( - $column, - $options, - $compression, - StatisticsOptions::fromOptions($options), - ); - - static::assertInstanceOf(PlainFlatColumnChunkBuilder::class, $builder); - static::assertSame($column, $builder->column()); - } - public function test_constructor_with_byte_array_logical_type(): void { $column = new FlatColumn('binary_col', PhysicalType::BYTE_ARRAY); $options = new Options(); $compression = Compressions::UNCOMPRESSED; - $builder = new PlainFlatColumnChunkBuilder( + $builder = new FlatColumnChunkBuilder( $column, $options, $compression, StatisticsOptions::fromOptions($options), + new BufferValueStorage(), ); - static::assertInstanceOf(PlainFlatColumnChunkBuilder::class, $builder); + static::assertInstanceOf(FlatColumnChunkBuilder::class, $builder); static::assertSame($column, $builder->column()); } @@ -428,14 +478,15 @@ public function test_constructor_with_different_compressions(Compressions $compr $column = new FlatColumn('test_col', PhysicalType::INT32); $options = new Options(); - $builder = new PlainFlatColumnChunkBuilder( + $builder = new FlatColumnChunkBuilder( $column, $options, $compression, StatisticsOptions::fromOptions($options), + new BufferValueStorage(), ); - static::assertInstanceOf(PlainFlatColumnChunkBuilder::class, $builder); + static::assertInstanceOf(FlatColumnChunkBuilder::class, $builder); } #[DataProvider('physical_types_provider')] @@ -445,14 +496,15 @@ public function test_constructor_with_different_physical_types(PhysicalType $phy $options = new Options(); $compression = Compressions::UNCOMPRESSED; - $builder = new PlainFlatColumnChunkBuilder( + $builder = new FlatColumnChunkBuilder( $column, $options, $compression, StatisticsOptions::fromOptions($options), + new BufferValueStorage(), ); - static::assertInstanceOf(PlainFlatColumnChunkBuilder::class, $builder); + static::assertInstanceOf(FlatColumnChunkBuilder::class, $builder); static::assertSame($column, $builder->column()); } @@ -461,11 +513,12 @@ public function test_edge_case_boolean_false_value(): void $column = new FlatColumn('bool_col', PhysicalType::BOOLEAN); $options = new Options(); $compression = Compressions::UNCOMPRESSED; - $builder = new PlainFlatColumnChunkBuilder( + $builder = new FlatColumnChunkBuilder( $column, $options, $compression, StatisticsOptions::fromOptions($options), + new BooleanValueStorage(), ); $builder->addColumn(new WriteFlatColumnValues($column, [0], [1], [false])); @@ -482,11 +535,12 @@ public function test_edge_case_empty_string_value(): void $column = new FlatColumn('str_col', PhysicalType::BYTE_ARRAY, logicalType: LogicalType::string()); $options = new Options(); $compression = Compressions::UNCOMPRESSED; - $builder = new PlainFlatColumnChunkBuilder( + $builder = new FlatColumnChunkBuilder( $column, $options, $compression, StatisticsOptions::fromOptions($options), + new BufferValueStorage(), ); $builder->addColumn(new WriteFlatColumnValues($column, [0], [1], [''])); @@ -503,11 +557,12 @@ public function test_edge_case_large_string_value(): void $column = new FlatColumn('str_col', PhysicalType::BYTE_ARRAY, logicalType: LogicalType::string()); $options = new Options(); $compression = Compressions::UNCOMPRESSED; - $builder = new PlainFlatColumnChunkBuilder( + $builder = new FlatColumnChunkBuilder( $column, $options, $compression, StatisticsOptions::fromOptions($options), + new BufferValueStorage(), ); $largeString = str_repeat('x', 10000); @@ -525,11 +580,12 @@ public function test_edge_case_zero_values(): void $column = new FlatColumn('test_col', PhysicalType::INT32); $options = new Options(); $compression = Compressions::UNCOMPRESSED; - $builder = new PlainFlatColumnChunkBuilder( + $builder = new FlatColumnChunkBuilder( $column, $options, $compression, StatisticsOptions::fromOptions($options), + new BufferValueStorage(), ); $builder->addColumn(new WriteFlatColumnValues($column, [0], [1], [0])); @@ -546,11 +602,12 @@ public function test_flush_automatically_closes_page_when_data_exists(): void $column = new FlatColumn('test_col', PhysicalType::INT32); $options = new Options(); $compression = Compressions::UNCOMPRESSED; - $builder = new PlainFlatColumnChunkBuilder( + $builder = new FlatColumnChunkBuilder( $column, $options, $compression, StatisticsOptions::fromOptions($options), + new BufferValueStorage(), ); $builder->addColumn(new WriteFlatColumnValues($column, [0], [1], [42])); @@ -568,11 +625,12 @@ public function test_flush_cleans_up_builder_state(): void $column = new FlatColumn('test_col', PhysicalType::INT32); $options = new Options(); $compression = Compressions::UNCOMPRESSED; - $builder = new PlainFlatColumnChunkBuilder( + $builder = new FlatColumnChunkBuilder( $column, $options, $compression, StatisticsOptions::fromOptions($options), + new BufferValueStorage(), ); $builder->addColumn(new WriteFlatColumnValues($column, [0, 0], [1, 1], [42, 84])); @@ -597,11 +655,12 @@ public function test_flush_preserves_column_chunk_metadata(): void $column = new FlatColumn('test_col', PhysicalType::INT32); $options = new Options(); $compression = Compressions::GZIP; - $builder = new PlainFlatColumnChunkBuilder( + $builder = new FlatColumnChunkBuilder( $column, $options, $compression, StatisticsOptions::fromOptions($options), + new BufferValueStorage(), ); $builder->addColumn(new WriteFlatColumnValues($column, [0], [1], [42])); @@ -620,11 +679,12 @@ public function test_flush_with_data(): void $column = new FlatColumn('test_col', PhysicalType::INT32); $options = new Options(); $compression = Compressions::UNCOMPRESSED; - $builder = new PlainFlatColumnChunkBuilder( + $builder = new FlatColumnChunkBuilder( $column, $options, $compression, StatisticsOptions::fromOptions($options), + new BufferValueStorage(), ); $builder->addColumn(new WriteFlatColumnValues($column, [0], [1], [42])); @@ -643,11 +703,12 @@ public function test_flush_with_different_file_offsets(): void $column = new FlatColumn('test_col', PhysicalType::INT32); $options = new Options(); $compression = Compressions::UNCOMPRESSED; - $builder = new PlainFlatColumnChunkBuilder( + $builder = new FlatColumnChunkBuilder( $column, $options, $compression, StatisticsOptions::fromOptions($options), + new BufferValueStorage(), ); $offsets = [0, 100, 1000, 50000]; @@ -666,11 +727,12 @@ public function test_flush_with_empty_data(): void $column = new FlatColumn('test_col', PhysicalType::INT32); $options = new Options(); $compression = Compressions::UNCOMPRESSED; - $builder = new PlainFlatColumnChunkBuilder( + $builder = new FlatColumnChunkBuilder( $column, $options, $compression, StatisticsOptions::fromOptions($options), + new BufferValueStorage(), ); $containers = $builder->flush(0); @@ -686,11 +748,12 @@ public function test_flush_with_negative_file_offset(): void $column = new FlatColumn('test_col', PhysicalType::INT32); $options = new Options(); $compression = Compressions::UNCOMPRESSED; - $builder = new PlainFlatColumnChunkBuilder( + $builder = new FlatColumnChunkBuilder( $column, $options, $compression, StatisticsOptions::fromOptions($options), + new BufferValueStorage(), ); $containers = $builder->flush(-100); @@ -705,11 +768,12 @@ public function test_is_full_initially_false(): void $column = new FlatColumn('test_col', PhysicalType::INT32); $options = new Options(); $compression = Compressions::UNCOMPRESSED; - $builder = new PlainFlatColumnChunkBuilder( + $builder = new FlatColumnChunkBuilder( $column, $options, $compression, StatisticsOptions::fromOptions($options), + new BufferValueStorage(), ); static::assertFalse($builder->isFull()); @@ -722,11 +786,12 @@ public function test_is_full_respects_page_size_option(int $pageSize): void $options = new Options(); $options->set(Option::PAGE_SIZE_BYTES, $pageSize); $compression = Compressions::UNCOMPRESSED; - $builder = new PlainFlatColumnChunkBuilder( + $builder = new FlatColumnChunkBuilder( $column, $options, $compression, StatisticsOptions::fromOptions($options), + new BufferValueStorage(), ); static::assertFalse($builder->isFull()); @@ -748,11 +813,12 @@ public function test_is_full_with_zero_page_size(): void $options = new Options(); $options->set(Option::PAGE_SIZE_BYTES, 0); $compression = Compressions::UNCOMPRESSED; - $builder = new PlainFlatColumnChunkBuilder( + $builder = new FlatColumnChunkBuilder( $column, $options, $compression, StatisticsOptions::fromOptions($options), + new BufferValueStorage(), ); $builder->addColumn(new WriteFlatColumnValues($column, [0], [1], [42])); @@ -765,11 +831,12 @@ public function test_mixed_null_and_non_null_values(): void $column = new FlatColumn('test_col', PhysicalType::INT32); $options = new Options(); $compression = Compressions::UNCOMPRESSED; - $builder = new PlainFlatColumnChunkBuilder( + $builder = new FlatColumnChunkBuilder( $column, $options, $compression, StatisticsOptions::fromOptions($options), + new BufferValueStorage(), ); $builder->addColumn(new WriteFlatColumnValues($column, [0, 0, 0], [1, 0, 1], [42, 100])); @@ -786,11 +853,12 @@ public function test_multiple_close_page_calls(): void $column = new FlatColumn('test_col', PhysicalType::INT32); $options = new Options(); $compression = Compressions::UNCOMPRESSED; - $builder = new PlainFlatColumnChunkBuilder( + $builder = new FlatColumnChunkBuilder( $column, $options, $compression, StatisticsOptions::fromOptions($options), + new BufferValueStorage(), ); $builder->addColumn(new WriteFlatColumnValues($column, [0], [1], [42])); @@ -807,11 +875,12 @@ public function test_multiple_flush_calls_maintain_consistency(): void $column = new FlatColumn('test_col', PhysicalType::INT32); $options = new Options(); $compression = Compressions::UNCOMPRESSED; - $builder = new PlainFlatColumnChunkBuilder( + $builder = new FlatColumnChunkBuilder( $column, $options, $compression, StatisticsOptions::fromOptions($options), + new BufferValueStorage(), ); $containers1 = $builder->flush(0); @@ -832,11 +901,12 @@ public function test_statistics_are_generated(): void $column = new FlatColumn('test_col', PhysicalType::INT32); $options = new Options(); $compression = Compressions::UNCOMPRESSED; - $builder = new PlainFlatColumnChunkBuilder( + $builder = new FlatColumnChunkBuilder( $column, $options, $compression, StatisticsOptions::fromOptions($options), + new BufferValueStorage(), ); $builder->addColumn(new WriteFlatColumnValues($column, [0, 0, 0], [1, 1, 1], [10, 50, 30])); @@ -852,11 +922,12 @@ public function test_statistics_track_null_count_for_all_null_chunk(): void $column = FlatColumn::string('all_null'); $options = new Options(); $compression = Compressions::UNCOMPRESSED; - $builder = new PlainFlatColumnChunkBuilder( + $builder = new FlatColumnChunkBuilder( $column, $options, $compression, StatisticsOptions::fromOptions($options), + new BufferValueStorage(), ); $builder->addColumn(new WriteFlatColumnValues($column, [0, 0, 0], [0, 0, 0], [])); @@ -874,11 +945,12 @@ public function test_statistics_track_null_count_for_mixed_chunk(): void $column = FlatColumn::string('mixed'); $options = new Options(); $compression = Compressions::UNCOMPRESSED; - $builder = new PlainFlatColumnChunkBuilder( + $builder = new FlatColumnChunkBuilder( $column, $options, $compression, StatisticsOptions::fromOptions($options), + new BufferValueStorage(), ); $builder->addColumn(new WriteFlatColumnValues($column, [0, 0, 0], [1, 0, 1], ['x', 'z'])); @@ -898,11 +970,12 @@ public function test_uncompressed_size_accumulates_multiple_pages(): void $column = new FlatColumn('test_col', PhysicalType::INT32); $options = new Options(); $compression = Compressions::UNCOMPRESSED; - $builder = new PlainFlatColumnChunkBuilder( + $builder = new FlatColumnChunkBuilder( $column, $options, $compression, StatisticsOptions::fromOptions($options), + new BufferValueStorage(), ); $sizes = []; @@ -923,11 +996,12 @@ public function test_uncompressed_size_increases_with_pages(): void $column = new FlatColumn('test_col', PhysicalType::INT32); $options = new Options(); $compression = Compressions::UNCOMPRESSED; - $builder = new PlainFlatColumnChunkBuilder( + $builder = new FlatColumnChunkBuilder( $column, $options, $compression, StatisticsOptions::fromOptions($options), + new BufferValueStorage(), ); $initialSize = $builder->uncompressedSize(); @@ -945,11 +1019,12 @@ public function test_uncompressed_size_initially_zero(): void $column = new FlatColumn('test_col', PhysicalType::INT32); $options = new Options(); $compression = Compressions::UNCOMPRESSED; - $builder = new PlainFlatColumnChunkBuilder( + $builder = new FlatColumnChunkBuilder( $column, $options, $compression, StatisticsOptions::fromOptions($options), + new BufferValueStorage(), ); static::assertSame(0, $builder->uncompressedSize()); @@ -960,11 +1035,12 @@ public function test_uncompressed_size_with_large_data(): void $column = new FlatColumn('str_col', PhysicalType::BYTE_ARRAY, logicalType: LogicalType::string()); $options = new Options(); $compression = Compressions::UNCOMPRESSED; - $builder = new PlainFlatColumnChunkBuilder( + $builder = new FlatColumnChunkBuilder( $column, $options, $compression, StatisticsOptions::fromOptions($options), + new BufferValueStorage(), ); $largeString = str_repeat('A', 50000); @@ -981,11 +1057,12 @@ public function test_workflow_add_close_flush(): void $column = new FlatColumn('test_col', PhysicalType::INT32); $options = new Options(); $compression = Compressions::UNCOMPRESSED; - $builder = new PlainFlatColumnChunkBuilder( + $builder = new FlatColumnChunkBuilder( $column, $options, $compression, StatisticsOptions::fromOptions($options), + new BufferValueStorage(), ); $builder->addColumn(new WriteFlatColumnValues($column, [0], [1], [42])); @@ -1004,11 +1081,12 @@ public function test_workflow_multiple_add_single_close_flush(): void $column = new FlatColumn('test_col', PhysicalType::INT32); $options = new Options(); $compression = Compressions::UNCOMPRESSED; - $builder = new PlainFlatColumnChunkBuilder( + $builder = new FlatColumnChunkBuilder( $column, $options, $compression, StatisticsOptions::fromOptions($options), + new BufferValueStorage(), ); for ($i = 0; $i < 5; $i++) { @@ -1030,11 +1108,12 @@ public function test_workflow_multiple_cycles(): void $column = new FlatColumn('test_col', PhysicalType::INT32); $options = new Options(); $compression = Compressions::UNCOMPRESSED; - $builder = new PlainFlatColumnChunkBuilder( + $builder = new FlatColumnChunkBuilder( $column, $options, $compression, StatisticsOptions::fromOptions($options), + new BufferValueStorage(), ); for ($cycle = 0; $cycle < 3; $cycle++) { @@ -1055,11 +1134,12 @@ public function test_workflow_with_all_null_values(): void $column = new FlatColumn('test_col', PhysicalType::INT32); $options = new Options(); $compression = Compressions::UNCOMPRESSED; - $builder = new PlainFlatColumnChunkBuilder( + $builder = new FlatColumnChunkBuilder( $column, $options, $compression, StatisticsOptions::fromOptions($options), + new BufferValueStorage(), ); for ($i = 0; $i < 5; $i++) { @@ -1073,14 +1153,16 @@ public function test_workflow_with_all_null_values(): void static::assertSame(5, $containers[0]->columnChunk->valuesCount()); } - public function test_each_page_gets_a_column_index_entry_and_an_offset_index_location(): void + #[DataProvider('page_index_storages_provider')] + public function test_each_page_gets_a_column_index_entry_and_an_offset_index_location(ValueStorage $storage): void { $column = FlatColumn::int64('id'); - $builder = new PlainFlatColumnChunkBuilder( + $builder = new FlatColumnChunkBuilder( $column, new Options(), Compressions::UNCOMPRESSED, StatisticsOptions::fromOptions(new Options()), + $storage, ); $builder->addColumn(new WriteFlatColumnValues($column, [0, 0], [1, 1], [1, 2])); $builder->closePage(); @@ -1099,4 +1181,70 @@ public function test_each_page_gets_a_column_index_entry_and_an_offset_index_loc static::assertSame($container->columnChunk->dataPageOffset(), $container->offsetIndex->pages[0]->offset); static::assertSame(4 + strlen($container->binaryBuffer), $last->offset + $last->compressedPageSize); } + + #[DataProvider('delta_binary_packed_is_full_provider')] + public function test_delta_binary_packed_storage_is_full_at_its_plain_size( + PhysicalType $type, + int $valuesToFill, + ): void { + $options = (new Options())->set(Option::PAGE_SIZE_BYTES, 100)->set(Option::WRITER_VERSION, 2); + $column = new FlatColumn('test_col', $type); + $builder = new FlatColumnChunkBuilder( + $column, + $options, + Compressions::UNCOMPRESSED, + StatisticsOptions::fromOptions($options), + new DeltaBinaryPackedValueStorage(), + ); + + for ($i = 1; $i < $valuesToFill; $i++) { + $builder->addColumn(new WriteFlatColumnValues($column, [0], [1], [$i])); + } + + static::assertFalse($builder->isFull()); + + $builder->addColumn(new WriteFlatColumnValues($column, [0], [1], [$valuesToFill])); + + static::assertTrue($builder->isFull()); + } + + #[DataProvider('compression_types_provider')] + public function test_delta_binary_packed_storage_writes_with_every_compression(Compressions $compression): void + { + $options = (new Options())->set(Option::WRITER_VERSION, 2); + $column = FlatColumn::int32('test_col'); + $builder = new FlatColumnChunkBuilder( + $column, + $options, + $compression, + StatisticsOptions::fromOptions($options), + new DeltaBinaryPackedValueStorage(), + ); + $builder->addColumn(new WriteFlatColumnValues($column, [0, 0], [1, 1], [42, 84])); + + $chunk = $builder->flush(0)[0]->columnChunk; + + static::assertSame($compression, $chunk->codec()); + static::assertSame(2, $chunk->valuesCount()); + } + + #[DataProvider('storage_encodings_provider')] + public function test_pages_carry_the_storage_encoding( + FlatColumn $column, + ValueStorage $storage, + bool|int|string $value, + int $writerVersion, + ): void { + $options = (new Options())->set(Option::WRITER_VERSION, $writerVersion); + $builder = new FlatColumnChunkBuilder( + $column, + $options, + Compressions::UNCOMPRESSED, + StatisticsOptions::fromOptions($options), + $storage, + ); + $builder->addColumn(new WriteFlatColumnValues($column, [0], [1], [$value])); + + static::assertContains($storage->encoding(), $builder->flush(0)[0]->columnChunk->encodings()); + } } diff --git a/src/lib/parquet/tests/Flow/Parquet/Tests/Unit/Writer/ColumnChunkBuilder/NestedColumnChunkBuilderTest.php b/src/lib/parquet/tests/Flow/Parquet/Tests/Unit/Writer/ColumnChunkBuilder/NestedColumnChunkBuilderTest.php index 75c4592660..11a4a74c0a 100644 --- a/src/lib/parquet/tests/Flow/Parquet/Tests/Unit/Writer/ColumnChunkBuilder/NestedColumnChunkBuilderTest.php +++ b/src/lib/parquet/tests/Flow/Parquet/Tests/Unit/Writer/ColumnChunkBuilder/NestedColumnChunkBuilderTest.php @@ -12,10 +12,12 @@ use Flow\Parquet\ParquetFile\Schema\NestedColumn; use Flow\Parquet\ParquetFile\Schema\PhysicalType; use Flow\Parquet\Writer\ColumnChunkBuilder; +use Flow\Parquet\Writer\ColumnChunkBuilder\FlatColumnChunkBuilder; use Flow\Parquet\Writer\ColumnChunkBuilder\NestedColumnChunkBuilder; -use Flow\Parquet\Writer\ColumnChunkBuilder\PlainFlatColumnChunkBuilder; use Flow\Parquet\Writer\ColumnChunkContainer; use Flow\Parquet\Writer\StatisticsOptions; +use Flow\Parquet\Writer\ValueStorage\BooleanValueStorage; +use Flow\Parquet\Writer\ValueStorage\BufferValueStorage; use Generator; use PHPUnit\Framework\Attributes\DataProvider; use PHPUnit\Framework\TestCase; @@ -62,11 +64,12 @@ public function test_add_multiple_rows(): void $options = new Options(); $compression = Compressions::UNCOMPRESSED; - $childBuilder = new PlainFlatColumnChunkBuilder( + $childBuilder = new FlatColumnChunkBuilder( $childColumn, $options, $compression, StatisticsOptions::fromOptions($options), + new BufferValueStorage(), ); $builder = new NestedColumnChunkBuilder($nestedColumn, [$childBuilder]); @@ -87,23 +90,26 @@ public function test_add_row_finds_correct_child_builder_by_flat_path(): void $options = new Options(); $compression = Compressions::UNCOMPRESSED; - $childBuilder1 = new PlainFlatColumnChunkBuilder( + $childBuilder1 = new FlatColumnChunkBuilder( $childColumn1, $options, $compression, StatisticsOptions::fromOptions($options), + new BufferValueStorage(), ); - $childBuilder2 = new PlainFlatColumnChunkBuilder( + $childBuilder2 = new FlatColumnChunkBuilder( $childColumn2, $options, $compression, StatisticsOptions::fromOptions($options), + new BufferValueStorage(), ); - $childBuilder3 = new PlainFlatColumnChunkBuilder( + $childBuilder3 = new FlatColumnChunkBuilder( $childColumn3, $options, $compression, StatisticsOptions::fromOptions($options), + new BooleanValueStorage(), ); $builder = new NestedColumnChunkBuilder($nestedColumn, [$childBuilder1, $childBuilder2, $childBuilder3]); @@ -119,11 +125,12 @@ public function test_add_row_with_empty_data(): void $options = new Options(); $compression = Compressions::UNCOMPRESSED; - $childBuilder = new PlainFlatColumnChunkBuilder( + $childBuilder = new FlatColumnChunkBuilder( $childColumn, $options, $compression, StatisticsOptions::fromOptions($options), + new BufferValueStorage(), ); $builder = new NestedColumnChunkBuilder($nestedColumn, [$childBuilder]); @@ -141,17 +148,19 @@ public function test_add_row_with_multiple_children_data(): void $options = new Options(); $compression = Compressions::UNCOMPRESSED; - $childBuilder1 = new PlainFlatColumnChunkBuilder( + $childBuilder1 = new FlatColumnChunkBuilder( $childColumn1, $options, $compression, StatisticsOptions::fromOptions($options), + new BufferValueStorage(), ); - $childBuilder2 = new PlainFlatColumnChunkBuilder( + $childBuilder2 = new FlatColumnChunkBuilder( $childColumn2, $options, $compression, StatisticsOptions::fromOptions($options), + new BufferValueStorage(), ); $builder = new NestedColumnChunkBuilder($nestedColumn, [$childBuilder1, $childBuilder2]); @@ -169,11 +178,12 @@ public function test_add_row_with_null_values(): void $options = new Options(); $compression = Compressions::UNCOMPRESSED; - $childBuilder = new PlainFlatColumnChunkBuilder( + $childBuilder = new FlatColumnChunkBuilder( $childColumn, $options, $compression, StatisticsOptions::fromOptions($options), + new BufferValueStorage(), ); $builder = new NestedColumnChunkBuilder($nestedColumn, [$childBuilder]); @@ -191,17 +201,19 @@ public function test_add_row_with_partial_child_data(): void $options = new Options(); $compression = Compressions::UNCOMPRESSED; - $childBuilder1 = new PlainFlatColumnChunkBuilder( + $childBuilder1 = new FlatColumnChunkBuilder( $childColumn1, $options, $compression, StatisticsOptions::fromOptions($options), + new BufferValueStorage(), ); - $childBuilder2 = new PlainFlatColumnChunkBuilder( + $childBuilder2 = new FlatColumnChunkBuilder( $childColumn2, $options, $compression, StatisticsOptions::fromOptions($options), + new BufferValueStorage(), ); $builder = new NestedColumnChunkBuilder($nestedColumn, [$childBuilder1, $childBuilder2]); @@ -218,11 +230,12 @@ public function test_add_row_with_single_child_data(): void $options = new Options(); $compression = Compressions::UNCOMPRESSED; - $childBuilder = new PlainFlatColumnChunkBuilder( + $childBuilder = new FlatColumnChunkBuilder( $childColumn, $options, $compression, StatisticsOptions::fromOptions($options), + new BufferValueStorage(), ); $builder = new NestedColumnChunkBuilder($nestedColumn, [$childBuilder]); @@ -251,17 +264,19 @@ public function test_close_page_with_multiple_children(): void $options = new Options(); $compression = Compressions::UNCOMPRESSED; - $childBuilder1 = new PlainFlatColumnChunkBuilder( + $childBuilder1 = new FlatColumnChunkBuilder( $childColumn1, $options, $compression, StatisticsOptions::fromOptions($options), + new BufferValueStorage(), ); - $childBuilder2 = new PlainFlatColumnChunkBuilder( + $childBuilder2 = new FlatColumnChunkBuilder( $childColumn2, $options, $compression, StatisticsOptions::fromOptions($options), + new BufferValueStorage(), ); $builder = new NestedColumnChunkBuilder($nestedColumn, [$childBuilder1, $childBuilder2]); @@ -281,11 +296,12 @@ public function test_close_page_with_single_child(): void $options = new Options(); $compression = Compressions::UNCOMPRESSED; - $childBuilder = new PlainFlatColumnChunkBuilder( + $childBuilder = new FlatColumnChunkBuilder( $childColumn, $options, $compression, StatisticsOptions::fromOptions($options), + new BufferValueStorage(), ); $builder = new NestedColumnChunkBuilder($nestedColumn, [$childBuilder]); @@ -304,11 +320,12 @@ public function test_column_returns_correct_nested_column(): void $options = new Options(); $compression = Compressions::UNCOMPRESSED; - $childBuilder = new PlainFlatColumnChunkBuilder( + $childBuilder = new FlatColumnChunkBuilder( $childColumn, $options, $compression, StatisticsOptions::fromOptions($options), + new BufferValueStorage(), ); $builder = new NestedColumnChunkBuilder($nestedColumn, [$childBuilder]); @@ -325,11 +342,12 @@ public function test_constructor_initializes_correctly(): void $options = new Options(); $compression = Compressions::UNCOMPRESSED; - $childBuilder = new PlainFlatColumnChunkBuilder( + $childBuilder = new FlatColumnChunkBuilder( $childColumn, $options, $compression, StatisticsOptions::fromOptions($options), + new BufferValueStorage(), ); $childrenBuilders = [$childBuilder]; @@ -354,11 +372,12 @@ public function test_constructor_with_different_nested_structures(array $childre $childrenBuilders = []; foreach ($children as $child) { - $childrenBuilders[] = new PlainFlatColumnChunkBuilder( + $childrenBuilders[] = new FlatColumnChunkBuilder( $child, $options, $compression, StatisticsOptions::fromOptions($options), + new BufferValueStorage(), ); } @@ -390,17 +409,19 @@ public function test_edge_case_mixed_null_and_non_null_across_children(): void $options = new Options(); $compression = Compressions::UNCOMPRESSED; - $childBuilder1 = new PlainFlatColumnChunkBuilder( + $childBuilder1 = new FlatColumnChunkBuilder( $childColumn1, $options, $compression, StatisticsOptions::fromOptions($options), + new BufferValueStorage(), ); - $childBuilder2 = new PlainFlatColumnChunkBuilder( + $childBuilder2 = new FlatColumnChunkBuilder( $childColumn2, $options, $compression, StatisticsOptions::fromOptions($options), + new BufferValueStorage(), ); $builder = new NestedColumnChunkBuilder($nestedColumn, [$childBuilder1, $childBuilder2]); @@ -423,17 +444,19 @@ public function test_edge_case_no_matching_flat_path(): void $options = new Options(); $compression = Compressions::UNCOMPRESSED; - $childBuilder1 = new PlainFlatColumnChunkBuilder( + $childBuilder1 = new FlatColumnChunkBuilder( $childColumn1, $options, $compression, StatisticsOptions::fromOptions($options), + new BufferValueStorage(), ); - $childBuilder2 = new PlainFlatColumnChunkBuilder( + $childBuilder2 = new FlatColumnChunkBuilder( $childColumn2, $options, $compression, StatisticsOptions::fromOptions($options), + new BufferValueStorage(), ); $builder = new NestedColumnChunkBuilder($nestedColumn, [$childBuilder1, $childBuilder2]); @@ -454,11 +477,12 @@ public function test_flush_automatically_closes_pages_when_data_exists(): void $options = new Options(); $compression = Compressions::UNCOMPRESSED; - $childBuilder = new PlainFlatColumnChunkBuilder( + $childBuilder = new FlatColumnChunkBuilder( $childColumn, $options, $compression, StatisticsOptions::fromOptions($options), + new BufferValueStorage(), ); $builder = new NestedColumnChunkBuilder($nestedColumn, [$childBuilder]); @@ -481,23 +505,26 @@ public function test_flush_calculates_correct_file_offsets(): void $options = new Options(); $compression = Compressions::UNCOMPRESSED; - $childBuilder1 = new PlainFlatColumnChunkBuilder( + $childBuilder1 = new FlatColumnChunkBuilder( $childColumn1, $options, $compression, StatisticsOptions::fromOptions($options), + new BufferValueStorage(), ); - $childBuilder2 = new PlainFlatColumnChunkBuilder( + $childBuilder2 = new FlatColumnChunkBuilder( $childColumn2, $options, $compression, StatisticsOptions::fromOptions($options), + new BufferValueStorage(), ); - $childBuilder3 = new PlainFlatColumnChunkBuilder( + $childBuilder3 = new FlatColumnChunkBuilder( $childColumn3, $options, $compression, StatisticsOptions::fromOptions($options), + new BooleanValueStorage(), ); $builder = new NestedColumnChunkBuilder($nestedColumn, [$childBuilder1, $childBuilder2, $childBuilder3]); @@ -532,11 +559,12 @@ public function test_flush_cleans_up_child_builder_state(): void $nestedColumn = NestedColumn::create('nested', [$childColumn]); $options = new Options(); - $childBuilder = new PlainFlatColumnChunkBuilder( + $childBuilder = new FlatColumnChunkBuilder( $childColumn, $options, $compression, StatisticsOptions::fromOptions($options), + new BufferValueStorage(), ); $builder = new NestedColumnChunkBuilder($nestedColumn, [$childBuilder]); @@ -564,11 +592,12 @@ public function test_flush_with_different_file_offsets(): void $options = new Options(); $compression = Compressions::UNCOMPRESSED; - $childBuilder = new PlainFlatColumnChunkBuilder( + $childBuilder = new FlatColumnChunkBuilder( $childColumn, $options, $compression, StatisticsOptions::fromOptions($options), + new BufferValueStorage(), ); $builder = new NestedColumnChunkBuilder($nestedColumn, [$childBuilder]); @@ -602,17 +631,19 @@ public function test_flush_with_multiple_children(): void $options = new Options(); $compression = Compressions::UNCOMPRESSED; - $childBuilder1 = new PlainFlatColumnChunkBuilder( + $childBuilder1 = new FlatColumnChunkBuilder( $childColumn1, $options, $compression, StatisticsOptions::fromOptions($options), + new BufferValueStorage(), ); - $childBuilder2 = new PlainFlatColumnChunkBuilder( + $childBuilder2 = new FlatColumnChunkBuilder( $childColumn2, $options, $compression, StatisticsOptions::fromOptions($options), + new BufferValueStorage(), ); $builder = new NestedColumnChunkBuilder($nestedColumn, [$childBuilder1, $childBuilder2]); @@ -636,11 +667,12 @@ public function test_flush_with_single_child(): void $options = new Options(); $compression = Compressions::UNCOMPRESSED; - $childBuilder = new PlainFlatColumnChunkBuilder( + $childBuilder = new FlatColumnChunkBuilder( $childColumn, $options, $compression, StatisticsOptions::fromOptions($options), + new BufferValueStorage(), ); $builder = new NestedColumnChunkBuilder($nestedColumn, [$childBuilder]); @@ -661,11 +693,12 @@ public function test_is_full_initially_false(): void $options = new Options(); $compression = Compressions::UNCOMPRESSED; - $childBuilder = new PlainFlatColumnChunkBuilder( + $childBuilder = new FlatColumnChunkBuilder( $childColumn, $options, $compression, StatisticsOptions::fromOptions($options), + new BufferValueStorage(), ); $builder = new NestedColumnChunkBuilder($nestedColumn, [$childBuilder]); @@ -680,17 +713,19 @@ public function test_is_full_returns_false_when_no_child_is_full(): void $options = new Options(); $compression = Compressions::UNCOMPRESSED; - $childBuilder1 = new PlainFlatColumnChunkBuilder( + $childBuilder1 = new FlatColumnChunkBuilder( $childColumn1, $options, $compression, StatisticsOptions::fromOptions($options), + new BufferValueStorage(), ); - $childBuilder2 = new PlainFlatColumnChunkBuilder( + $childBuilder2 = new FlatColumnChunkBuilder( $childColumn2, $options, $compression, StatisticsOptions::fromOptions($options), + new BufferValueStorage(), ); $builder = new NestedColumnChunkBuilder($nestedColumn, [$childBuilder1, $childBuilder2]); @@ -705,11 +740,12 @@ public function test_is_full_returns_true_when_any_child_is_full(): void $options = new Options(); $compression = Compressions::UNCOMPRESSED; - $childBuilder2 = new PlainFlatColumnChunkBuilder( + $childBuilder2 = new FlatColumnChunkBuilder( $childColumn2, $options, $compression, StatisticsOptions::fromOptions($options), + new BufferValueStorage(), ); $mockFullChildBuilder = $this->createStub(ColumnChunkBuilder::class); @@ -736,11 +772,12 @@ public function test_multiple_close_page_calls(): void $options = new Options(); $compression = Compressions::UNCOMPRESSED; - $childBuilder = new PlainFlatColumnChunkBuilder( + $childBuilder = new FlatColumnChunkBuilder( $childColumn, $options, $compression, StatisticsOptions::fromOptions($options), + new BufferValueStorage(), ); $builder = new NestedColumnChunkBuilder($nestedColumn, [$childBuilder]); @@ -760,11 +797,12 @@ public function test_uncompressed_size_initially_zero(): void $options = new Options(); $compression = Compressions::UNCOMPRESSED; - $childBuilder = new PlainFlatColumnChunkBuilder( + $childBuilder = new FlatColumnChunkBuilder( $childColumn, $options, $compression, StatisticsOptions::fromOptions($options), + new BufferValueStorage(), ); $builder = new NestedColumnChunkBuilder($nestedColumn, [$childBuilder]); @@ -779,17 +817,19 @@ public function test_uncompressed_size_sums_children_sizes(): void $options = new Options(); $compression = Compressions::UNCOMPRESSED; - $childBuilder1 = new PlainFlatColumnChunkBuilder( + $childBuilder1 = new FlatColumnChunkBuilder( $childColumn1, $options, $compression, StatisticsOptions::fromOptions($options), + new BufferValueStorage(), ); - $childBuilder2 = new PlainFlatColumnChunkBuilder( + $childBuilder2 = new FlatColumnChunkBuilder( $childColumn2, $options, $compression, StatisticsOptions::fromOptions($options), + new BufferValueStorage(), ); $builder = new NestedColumnChunkBuilder($nestedColumn, [$childBuilder1, $childBuilder2]); @@ -840,17 +880,19 @@ public function test_workflow_complete_add_close_flush_cycle(): void $options = new Options(); $compression = Compressions::UNCOMPRESSED; - $childBuilder1 = new PlainFlatColumnChunkBuilder( + $childBuilder1 = new FlatColumnChunkBuilder( $childColumn1, $options, $compression, StatisticsOptions::fromOptions($options), + new BufferValueStorage(), ); - $childBuilder2 = new PlainFlatColumnChunkBuilder( + $childBuilder2 = new FlatColumnChunkBuilder( $childColumn2, $options, $compression, StatisticsOptions::fromOptions($options), + new BufferValueStorage(), ); $builder = new NestedColumnChunkBuilder($nestedColumn, [$childBuilder1, $childBuilder2]); @@ -877,11 +919,12 @@ public function test_workflow_multiple_cycles(): void $options = new Options(); $compression = Compressions::UNCOMPRESSED; - $childBuilder = new PlainFlatColumnChunkBuilder( + $childBuilder = new FlatColumnChunkBuilder( $childColumn, $options, $compression, StatisticsOptions::fromOptions($options), + new BufferValueStorage(), ); $builder = new NestedColumnChunkBuilder($nestedColumn, [$childBuilder]); @@ -905,11 +948,12 @@ public function test_workflow_with_different_compressions(Compressions $compress $nestedColumn = NestedColumn::create('nested', [$childColumn]); $options = new Options(); - $childBuilder = new PlainFlatColumnChunkBuilder( + $childBuilder = new FlatColumnChunkBuilder( $childColumn, $options, $compression, StatisticsOptions::fromOptions($options), + new BufferValueStorage(), ); $builder = new NestedColumnChunkBuilder($nestedColumn, [$childBuilder]); diff --git a/src/lib/parquet/tests/Flow/Parquet/Tests/Unit/Writer/ColumnChunkBuilder/RLEDictionaryChunkBuilderTest.php b/src/lib/parquet/tests/Flow/Parquet/Tests/Unit/Writer/ColumnChunkBuilder/RLEDictionaryChunkBuilderTest.php index a20569cf6a..35cb36108a 100644 --- a/src/lib/parquet/tests/Flow/Parquet/Tests/Unit/Writer/ColumnChunkBuilder/RLEDictionaryChunkBuilderTest.php +++ b/src/lib/parquet/tests/Flow/Parquet/Tests/Unit/Writer/ColumnChunkBuilder/RLEDictionaryChunkBuilderTest.php @@ -4,6 +4,7 @@ namespace Flow\Parquet\Tests\Unit\Writer\ColumnChunkBuilder; +use Flow\Parquet\Data\RLEBitPackedHybrid; use Flow\Parquet\Dremel\ColumnData\WriteFlatColumnValues; use Flow\Parquet\Option; use Flow\Parquet\Options; @@ -17,13 +18,18 @@ use Flow\Parquet\ParquetFile\Schema\PhysicalType; use Flow\Parquet\Writer\ColumnChunkBuilder\RLEDictionaryChunkBuilder; use Flow\Parquet\Writer\ColumnChunkContainer; +use Flow\Parquet\Writer\PageBuilder\RLEBitPackedPacker; use Flow\Parquet\Writer\StatisticsOptions; +use Flow\Parquet\Writer\ValueStorage\BufferValueStorage; +use Flow\Parquet\Writer\ValueStorage\DeltaByteArrayValueStorage; +use Flow\Parquet\Writer\ValueStorage\ValueStorage; use Generator; use PHPUnit\Framework\Attributes\DataProvider; use PHPUnit\Framework\TestCase; use RuntimeException; use function array_map; +use function in_array; use function is_object; use function is_scalar; use function pack; @@ -66,6 +72,14 @@ public static function dictionary_data_provider(): Generator ]; } + public static function overflow_provider(): Generator + { + yield 'PLAIN after an indexed page' => [new BufferValueStorage(), null, true]; + yield 'PLAIN from the first page' => [new BufferValueStorage(), 'a', false]; + yield 'DELTA_BYTE_ARRAY after an indexed page' => [new DeltaByteArrayValueStorage(), null, true]; + yield 'DELTA_BYTE_ARRAY from the first page' => [new DeltaByteArrayValueStorage(), 'a', false]; + } + public static function page_size_provider(): Generator { yield 'small page' => [1024]; @@ -99,6 +113,7 @@ public function test_add_multiple_rows(): void $options, $compression, StatisticsOptions::fromOptions($options), + new BufferValueStorage(), ); for ($i = 0; $i < 5; $i++) { @@ -123,6 +138,7 @@ public function test_add_row_with_dictionary_data(array $values, string $descrip $options, $compression, StatisticsOptions::fromOptions($options), + new BufferValueStorage(), ); $repLevels = []; @@ -160,6 +176,7 @@ public function test_add_row_with_empty_data(): void $options, $compression, StatisticsOptions::fromOptions($options), + new BufferValueStorage(), ); $builder->addColumn(new WriteFlatColumnValues($column, [], [], [])); @@ -178,6 +195,7 @@ public function test_add_row_with_null_values(): void $options, $compression, StatisticsOptions::fromOptions($options), + new BufferValueStorage(), ); $builder->addColumn(new WriteFlatColumnValues($column, [0], [0], [])); @@ -196,6 +214,7 @@ public function test_add_row_with_repeated_values(): void $options, $compression, StatisticsOptions::fromOptions($options), + new BufferValueStorage(), ); $builder->addColumn( @@ -221,6 +240,7 @@ public function test_add_row_with_single_value(): void $options, $compression, StatisticsOptions::fromOptions($options), + new BufferValueStorage(), ); $builder->addColumn(new WriteFlatColumnValues($column, [0], [1], ['hello'])); @@ -229,7 +249,7 @@ public function test_add_row_with_single_value(): void static::assertGreaterThanOrEqual(0, $builder->uncompressedSize()); } - public function test_a_chunk_without_values_writes_no_dictionary_page(): void + public function test_a_chunk_of_nulls_writes_an_empty_dictionary_page(): void { $column = new FlatColumn('test_col', PhysicalType::BYTE_ARRAY, logicalType: LogicalType::string()); $builder = new RLEDictionaryChunkBuilder( @@ -237,11 +257,52 @@ public function test_a_chunk_without_values_writes_no_dictionary_page(): void new Options(), Compressions::UNCOMPRESSED, StatisticsOptions::fromOptions(new Options()), + new BufferValueStorage(), ); $builder->addColumn(new WriteFlatColumnValues($column, [0], [0], [])); - static::assertNull($builder->flush(4)[0]->columnChunk->dictionaryPageOffset()); + static::assertSame(4, $builder->flush(4)[0]->columnChunk->dictionaryPageOffset()); + } + + public function test_a_chunk_where_every_page_overflowed_writes_no_dictionary_page(): void + { + $column = FlatColumn::string('test_col'); + $options = (new Options())->set(Option::DICTIONARY_PAGE_SIZE, 1); + $builder = new RLEDictionaryChunkBuilder( + $column, + $options, + Compressions::UNCOMPRESSED, + StatisticsOptions::fromOptions($options), + new BufferValueStorage(), + ); + + $builder->addColumn(new WriteFlatColumnValues($column, [0], [1], ['a'])); + $builder->closePage(); + $builder->addColumn(new WriteFlatColumnValues($column, [0], [1], ['b'])); + $chunk = $builder->flush(4)[0]->columnChunk; + + static::assertNull($chunk->dictionaryPageOffset()); + static::assertNotContains(Encodings::RLE_DICTIONARY, $chunk->encodings()); + } + + public function test_an_all_null_page_writes_the_bit_width_byte(): void + { + $column = FlatColumn::string('test_col'); + $builder = new RLEDictionaryChunkBuilder( + $column, + new Options(), + Compressions::UNCOMPRESSED, + StatisticsOptions::fromOptions(new Options()), + new BufferValueStorage(), + ); + + $builder->addColumn(new WriteFlatColumnValues($column, [0, 0], [0, 0], [])); + + static::assertStringEndsWith( + (new RLEBitPackedPacker(new RLEBitPackedHybrid()))->packWithLength(1, [0, 0]) . "\x00", + $builder->flush(4)[0]->binaryBuffer, + ); } public function test_close_page_resets_internal_state(): void @@ -254,6 +315,7 @@ public function test_close_page_resets_internal_state(): void $options, $compression, StatisticsOptions::fromOptions($options), + new BufferValueStorage(), ); $builder->addColumn(new WriteFlatColumnValues($column, [0], [1], ['test'])); @@ -275,6 +337,7 @@ public function test_close_page_with_data_builds_dictionary(): void $options, $compression, StatisticsOptions::fromOptions($options), + new BufferValueStorage(), ); $builder->addColumn( @@ -299,6 +362,7 @@ public function test_close_page_with_different_writer_versions(int $writerVersio $options, $compression, StatisticsOptions::fromOptions($options), + new BufferValueStorage(), ); $builder->addColumn(new WriteFlatColumnValues($column, [0], [1], ['test'])); @@ -318,6 +382,7 @@ public function test_close_page_with_empty_data(): void $options, $compression, StatisticsOptions::fromOptions($options), + new BufferValueStorage(), ); $builder->closePage(); @@ -337,6 +402,7 @@ public function test_close_page_with_unsupported_writer_version_throws_exception $options, $compression, StatisticsOptions::fromOptions($options), + new BufferValueStorage(), ); $builder->addColumn(new WriteFlatColumnValues($column, [0], [1], ['test'])); @@ -359,6 +425,7 @@ public function test_column_returns_correct_column(): void $options, $compression, StatisticsOptions::fromOptions($options), + new BufferValueStorage(), ); $result = $builder->column(); @@ -377,6 +444,7 @@ public function test_constructor_initializes_correctly(): void $options, $compression, StatisticsOptions::fromOptions($options), + new BufferValueStorage(), ); static::assertInstanceOf(RLEDictionaryChunkBuilder::class, $builder); @@ -396,6 +464,7 @@ public function test_constructor_with_different_compressions(Compressions $compr $options, $compression, StatisticsOptions::fromOptions($options), + new BufferValueStorage(), ); static::assertInstanceOf(RLEDictionaryChunkBuilder::class, $builder); @@ -418,6 +487,7 @@ public function test_constructor_with_different_physical_types( $options, $compression, StatisticsOptions::fromOptions($options), + new BufferValueStorage(), ); static::assertInstanceOf(RLEDictionaryChunkBuilder::class, $builder); @@ -434,6 +504,7 @@ public function test_dictionary_compression_efficiency(): void $options, $compression, StatisticsOptions::fromOptions($options), + new BufferValueStorage(), ); $repeatedValues = ['apple', 'banana', 'cherry']; @@ -469,6 +540,7 @@ public function test_different_compressions_work_correctly(Compressions $compres $options, $compression, StatisticsOptions::fromOptions($options), + new BufferValueStorage(), ); $builder->addColumn(new WriteFlatColumnValues($column, [0], [1], ['test'])); @@ -490,6 +562,7 @@ public function test_edge_case_boolean_false_value(): void $options, $compression, StatisticsOptions::fromOptions($options), + new BufferValueStorage(), ); $builder->addColumn(new WriteFlatColumnValues($column, [0], [1], [false])); @@ -511,6 +584,7 @@ public function test_edge_case_empty_string_value(): void $options, $compression, StatisticsOptions::fromOptions($options), + new BufferValueStorage(), ); $builder->addColumn(new WriteFlatColumnValues($column, [0], [1], [''])); @@ -532,6 +606,7 @@ public function test_edge_case_large_string_value(): void $options, $compression, StatisticsOptions::fromOptions($options), + new BufferValueStorage(), ); $largeString = str_repeat('x', 10000); @@ -554,6 +629,7 @@ public function test_edge_case_zero_values(): void $options, $compression, StatisticsOptions::fromOptions($options), + new BufferValueStorage(), ); $builder->addColumn(new WriteFlatColumnValues($column, [0], [1], [0])); @@ -575,6 +651,7 @@ public function test_flush_automatically_closes_page_when_data_exists(): void $options, $compression, StatisticsOptions::fromOptions($options), + new BufferValueStorage(), ); $builder->addColumn(new WriteFlatColumnValues($column, [0], [1], ['test'])); @@ -597,6 +674,7 @@ public function test_flush_cleans_up_builder_state(): void $options, $compression, StatisticsOptions::fromOptions($options), + new BufferValueStorage(), ); $builder->addColumn(new WriteFlatColumnValues($column, [0, 0], [1, 1], ['hello', 'world'])); @@ -626,6 +704,7 @@ public function test_flush_preserves_column_chunk_metadata(): void $options, $compression, StatisticsOptions::fromOptions($options), + new BufferValueStorage(), ); $builder->addColumn(new WriteFlatColumnValues($column, [0], [1], ['test'])); @@ -649,6 +728,7 @@ public function test_flush_with_data(): void $options, $compression, StatisticsOptions::fromOptions($options), + new BufferValueStorage(), ); $builder->addColumn(new WriteFlatColumnValues($column, [0], [1], ['test'])); @@ -672,6 +752,7 @@ public function test_flush_with_dictionary_includes_dictionary_page(): void $options, $compression, StatisticsOptions::fromOptions($options), + new BufferValueStorage(), ); $builder->addColumn( @@ -699,6 +780,7 @@ public function test_flush_with_different_file_offsets(): void $options, $compression, StatisticsOptions::fromOptions($options), + new BufferValueStorage(), ); $offsets = [0, 100, 1000, 50000]; @@ -722,6 +804,7 @@ public function test_flush_with_empty_data(): void $options, $compression, StatisticsOptions::fromOptions($options), + new BufferValueStorage(), ); $containers = $builder->flush(0); @@ -742,6 +825,7 @@ public function test_is_full_initially_false(): void $options, $compression, StatisticsOptions::fromOptions($options), + new BufferValueStorage(), ); static::assertFalse($builder->isFull()); @@ -759,6 +843,7 @@ public function test_is_full_respects_page_size_option(int $pageSize): void $options, $compression, StatisticsOptions::fromOptions($options), + new BufferValueStorage(), ); static::assertFalse($builder->isFull()); @@ -785,6 +870,7 @@ public function test_is_full_uses_approximate_calculation(): void $options, $compression, StatisticsOptions::fromOptions($options), + new BufferValueStorage(), ); for ($i = 0; $i < 30; $i++) { @@ -804,6 +890,7 @@ public function test_mixed_null_and_non_null_values(): void $options, $compression, StatisticsOptions::fromOptions($options), + new BufferValueStorage(), ); $builder->addColumn(new WriteFlatColumnValues($column, [0, 0, 0], [1, 0, 1], ['hello', 'world'])); @@ -825,6 +912,7 @@ public function test_multiple_close_page_calls(): void $options, $compression, StatisticsOptions::fromOptions($options), + new BufferValueStorage(), ); $builder->addColumn(new WriteFlatColumnValues($column, [0], [1], ['test'])); @@ -846,6 +934,7 @@ public function test_statistics_are_generated(): void $options, $compression, StatisticsOptions::fromOptions($options), + new BufferValueStorage(), ); $builder->addColumn(new WriteFlatColumnValues($column, [0, 0, 0], [1, 1, 1], ['apple', 'zebra', 'banana'])); @@ -866,6 +955,7 @@ public function test_uncompressed_size_accumulates_multiple_pages(): void $options, $compression, StatisticsOptions::fromOptions($options), + new BufferValueStorage(), ); $sizes = []; @@ -891,6 +981,7 @@ public function test_uncompressed_size_increases_with_pages(): void $options, $compression, StatisticsOptions::fromOptions($options), + new BufferValueStorage(), ); $initialSize = $builder->uncompressedSize(); @@ -913,6 +1004,7 @@ public function test_uncompressed_size_initially_zero(): void $options, $compression, StatisticsOptions::fromOptions($options), + new BufferValueStorage(), ); static::assertSame(0, $builder->uncompressedSize()); @@ -928,6 +1020,7 @@ public function test_workflow_add_close_flush(): void $options, $compression, StatisticsOptions::fromOptions($options), + new BufferValueStorage(), ); $builder->addColumn(new WriteFlatColumnValues($column, [0], [1], ['test'])); @@ -951,6 +1044,7 @@ public function test_workflow_multiple_add_single_close_flush(): void $options, $compression, StatisticsOptions::fromOptions($options), + new BufferValueStorage(), ); for ($i = 0; $i < 5; $i++) { @@ -977,6 +1071,7 @@ public function test_workflow_multiple_cycles(): void $options, $compression, StatisticsOptions::fromOptions($options), + new BufferValueStorage(), ); for ($cycle = 0; $cycle < 3; $cycle++) { @@ -1000,6 +1095,7 @@ public function test_each_page_gets_a_column_index_entry_and_an_offset_index_loc new Options(), Compressions::UNCOMPRESSED, StatisticsOptions::fromOptions(new Options()), + new BufferValueStorage(), ); $builder->addColumn(new WriteFlatColumnValues($column, [0, 0], [1, 1], [1, 2])); $builder->closePage(); @@ -1018,4 +1114,33 @@ public function test_each_page_gets_a_column_index_entry_and_an_offset_index_loc static::assertSame($container->columnChunk->dataPageOffset(), $container->offsetIndex->pages[0]->offset); static::assertSame(4 + strlen($container->binaryBuffer), $last->offset + $last->compressedPageSize); } + + #[DataProvider('overflow_provider')] + public function test_overflow_pages_use_the_fallback_storage_encoding( + ValueStorage $fallback, + ?string $firstPageValue, + bool $indexed, + ): void { + $column = FlatColumn::string('test_col'); + $options = (new Options())->set(Option::DICTIONARY_PAGE_SIZE, 1); + $builder = new RLEDictionaryChunkBuilder( + $column, + $options, + Compressions::UNCOMPRESSED, + StatisticsOptions::fromOptions($options), + $fallback, + ); + + $builder->addColumn( + $firstPageValue === null + ? new WriteFlatColumnValues($column, [0], [0], []) + : new WriteFlatColumnValues($column, [0], [1], [$firstPageValue]), + ); + $builder->closePage(); + $builder->addColumn(new WriteFlatColumnValues($column, [0], [1], ['b'])); + $encodings = $builder->flush(4)[0]->columnChunk->encodings(); + + static::assertContains($fallback->encoding(), $encodings); + static::assertSame($indexed, in_array(Encodings::RLE_DICTIONARY, $encodings, true)); + } } diff --git a/src/lib/parquet/tests/Flow/Parquet/Tests/Unit/Writer/ColumnChunkBuilderFactoryTest.php b/src/lib/parquet/tests/Flow/Parquet/Tests/Unit/Writer/ColumnChunkBuilderFactoryTest.php index 7c0de2e393..2b6517211b 100644 --- a/src/lib/parquet/tests/Flow/Parquet/Tests/Unit/Writer/ColumnChunkBuilderFactoryTest.php +++ b/src/lib/parquet/tests/Flow/Parquet/Tests/Unit/Writer/ColumnChunkBuilderFactoryTest.php @@ -4,6 +4,7 @@ namespace Flow\Parquet\Tests\Unit\Writer; +use Flow\Parquet\Dremel\ColumnData\WriteFlatColumnValues; use Flow\Parquet\Exception\InvalidArgumentException; use Flow\Parquet\Option; use Flow\Parquet\Options; @@ -12,13 +13,15 @@ use Flow\Parquet\ParquetFile\Schema\FlatColumn; use Flow\Parquet\ParquetFile\Schema\LogicalType; use Flow\Parquet\ParquetFile\Schema\PhysicalType; -use Flow\Parquet\Writer\ColumnChunkBuilder\DeltaBinaryPackedColumnChunkBuilder; -use Flow\Parquet\Writer\ColumnChunkBuilder\PlainFlatColumnChunkBuilder; +use Flow\Parquet\Writer\ColumnChunkBuilder\FlatColumnChunkBuilder; use Flow\Parquet\Writer\ColumnChunkBuilder\RLEDictionaryChunkBuilder; use Flow\Parquet\Writer\ColumnChunkBuilderFactory; use Flow\Parquet\Writer\StatisticsOptions; +use PHPUnit\Framework\Attributes\DataProvider; use PHPUnit\Framework\TestCase; +use function array_values; + final class ColumnChunkBuilderFactoryTest extends TestCase { public function test_create_builder_custom_encoding_overrides_automatic_selection(): void @@ -36,10 +39,14 @@ public function test_create_builder_custom_encoding_overrides_automatic_selectio StatisticsOptions::fromOptions($options), ); - static::assertInstanceOf(PlainFlatColumnChunkBuilder::class, $builder); + static::assertInstanceOf(FlatColumnChunkBuilder::class, $builder); + + $builder->addColumn(new WriteFlatColumnValues($column, [0], [1], [7])); + + static::assertContains(Encodings::PLAIN, $builder->flush(0)[0]->columnChunk->encodings()); } - public function test_create_builder_for_int32_column_with_plain_encoding(): void + public function test_create_builder_for_int32_column_defaults_to_dictionary(): void { $column = new FlatColumn('user_id', PhysicalType::INT32); $options = Options::default(); @@ -52,13 +59,17 @@ public function test_create_builder_for_int32_column_with_plain_encoding(): void StatisticsOptions::fromOptions($options), ); - static::assertInstanceOf(PlainFlatColumnChunkBuilder::class, $builder); + static::assertInstanceOf(RLEDictionaryChunkBuilder::class, $builder); + + $builder->addColumn(new WriteFlatColumnValues($column, [0], [1], [7])); + + static::assertContains(Encodings::RLE_DICTIONARY, $builder->flush(0)[0]->columnChunk->encodings()); } public function test_create_builder_for_int32_column_with_writer_version_2(): void { $column = new FlatColumn('user_id', PhysicalType::INT32); - $options = Options::default()->set(Option::WRITER_VERSION, 2); + $options = Options::default()->set(Option::DICTIONARY_ENABLED, false)->set(Option::WRITER_VERSION, 2); $compressions = Compressions::UNCOMPRESSED; $builder = ColumnChunkBuilderFactory::createBuilder( @@ -68,13 +79,17 @@ public function test_create_builder_for_int32_column_with_writer_version_2(): vo StatisticsOptions::fromOptions($options), ); - static::assertInstanceOf(DeltaBinaryPackedColumnChunkBuilder::class, $builder); + static::assertInstanceOf(FlatColumnChunkBuilder::class, $builder); + + $builder->addColumn(new WriteFlatColumnValues($column, [0], [1], [7])); + + static::assertContains(Encodings::DELTA_BINARY_PACKED, $builder->flush(0)[0]->columnChunk->encodings()); } public function test_create_builder_for_int64_column_with_writer_version_2(): void { $column = new FlatColumn('timestamp', PhysicalType::INT64); - $options = Options::default()->set(Option::WRITER_VERSION, 2); + $options = Options::default()->set(Option::DICTIONARY_ENABLED, false)->set(Option::WRITER_VERSION, 2); $compressions = Compressions::UNCOMPRESSED; $builder = ColumnChunkBuilderFactory::createBuilder( @@ -84,10 +99,14 @@ public function test_create_builder_for_int64_column_with_writer_version_2(): vo StatisticsOptions::fromOptions($options), ); - static::assertInstanceOf(DeltaBinaryPackedColumnChunkBuilder::class, $builder); + static::assertInstanceOf(FlatColumnChunkBuilder::class, $builder); + + $builder->addColumn(new WriteFlatColumnValues($column, [0], [1], [7])); + + static::assertContains(Encodings::DELTA_BINARY_PACKED, $builder->flush(0)[0]->columnChunk->encodings()); } - public function test_create_builder_for_string_column(): void + public function test_create_builder_for_string_column_defaults_to_dictionary(): void { $column = new FlatColumn('name', PhysicalType::BYTE_ARRAY, logicalType: LogicalType::string()); $options = Options::default(); @@ -100,7 +119,11 @@ public function test_create_builder_for_string_column(): void StatisticsOptions::fromOptions($options), ); - static::assertInstanceOf(PlainFlatColumnChunkBuilder::class, $builder); + static::assertInstanceOf(RLEDictionaryChunkBuilder::class, $builder); + + $builder->addColumn(new WriteFlatColumnValues($column, [0], [1], ['a'])); + + static::assertContains(Encodings::RLE_DICTIONARY, $builder->flush(0)[0]->columnChunk->encodings()); } public function test_create_builder_with_custom_delta_binary_packed_encoding(): void @@ -118,7 +141,11 @@ public function test_create_builder_with_custom_delta_binary_packed_encoding(): StatisticsOptions::fromOptions($options), ); - static::assertInstanceOf(DeltaBinaryPackedColumnChunkBuilder::class, $builder); + static::assertInstanceOf(FlatColumnChunkBuilder::class, $builder); + + $builder->addColumn(new WriteFlatColumnValues($column, [0], [1], [7])); + + static::assertContains(Encodings::DELTA_BINARY_PACKED, $builder->flush(0)[0]->columnChunk->encodings()); } public function test_create_builder_with_custom_plain_encoding(): void @@ -136,7 +163,11 @@ public function test_create_builder_with_custom_plain_encoding(): void StatisticsOptions::fromOptions($options), ); - static::assertInstanceOf(PlainFlatColumnChunkBuilder::class, $builder); + static::assertInstanceOf(FlatColumnChunkBuilder::class, $builder); + + $builder->addColumn(new WriteFlatColumnValues($column, [0], [1], ['a'])); + + static::assertContains(Encodings::PLAIN, $builder->flush(0)[0]->columnChunk->encodings()); } public function test_create_builder_with_custom_rle_dictionary_encoding(): void @@ -173,7 +204,11 @@ public function test_create_builder_with_flat_path_encoding(): void StatisticsOptions::fromOptions($options), ); - static::assertInstanceOf(DeltaBinaryPackedColumnChunkBuilder::class, $builder); + static::assertInstanceOf(FlatColumnChunkBuilder::class, $builder); + + $builder->addColumn(new WriteFlatColumnValues($column, [0], [1], [7])); + + static::assertContains(Encodings::DELTA_BINARY_PACKED, $builder->flush(0)[0]->columnChunk->encodings()); } public function test_create_builder_with_rle_dictionary_encoding(): void @@ -207,13 +242,17 @@ public function test_empty_columns_encodings_option(): void StatisticsOptions::fromOptions($options), ); - static::assertInstanceOf(PlainFlatColumnChunkBuilder::class, $builder); + static::assertInstanceOf(RLEDictionaryChunkBuilder::class, $builder); + + $builder->addColumn(new WriteFlatColumnValues($column, [0], [1], [7])); + + static::assertContains(Encodings::RLE_DICTIONARY, $builder->flush(0)[0]->columnChunk->encodings()); } public function test_fallback_to_plain_when_no_custom_encoding_specified(): void { $column = new FlatColumn('description', PhysicalType::BYTE_ARRAY, logicalType: LogicalType::string()); - $options = Options::default()->set(Option::COLUMNS_ENCODINGS, [ + $options = Options::default()->set(Option::DICTIONARY_ENABLED, false)->set(Option::COLUMNS_ENCODINGS, [ 'other_column' => Encodings::RLE_DICTIONARY, ]); $compressions = Compressions::UNCOMPRESSED; @@ -225,7 +264,11 @@ public function test_fallback_to_plain_when_no_custom_encoding_specified(): void StatisticsOptions::fromOptions($options), ); - static::assertInstanceOf(PlainFlatColumnChunkBuilder::class, $builder); + static::assertInstanceOf(FlatColumnChunkBuilder::class, $builder); + + $builder->addColumn(new WriteFlatColumnValues($column, [0], [1], ['a'])); + + static::assertContains(Encodings::PLAIN, $builder->flush(0)[0]->columnChunk->encodings()); } public function test_invalid_delta_binary_packed_encoding_for_string_column_throws_exception(): void @@ -253,7 +296,7 @@ public function test_invalid_rle_dictionary_encoding_for_fixed_len_byte_array_th { $this->expectException(InvalidArgumentException::class); $this->expectExceptionMessage( - "RLE_DICTIONARY encoding is not supported for FIXED_LEN_BYTE_ARRAY columns. Column 'fixed_data' has type: FIXED_LEN_BYTE_ARRAY", + "RLE_DICTIONARY encoding is not supported for FIXED_LEN_BYTE_ARRAY columns under WRITER_VERSION 1. Column 'fixed_data' has type: FIXED_LEN_BYTE_ARRAY", ); $column = new FlatColumn('fixed_data', PhysicalType::FIXED_LEN_BYTE_ARRAY); @@ -286,6 +329,245 @@ public function test_non_enum_encoding_value_is_ignored(): void ); // Should fallback to default behavior since non-enum is ignored - static::assertInstanceOf(PlainFlatColumnChunkBuilder::class, $builder); + static::assertInstanceOf(RLEDictionaryChunkBuilder::class, $builder); + + $builder->addColumn(new WriteFlatColumnValues($column, [0], [1], ['a'])); + + static::assertContains(Encodings::RLE_DICTIONARY, $builder->flush(0)[0]->columnChunk->encodings()); + } + + /** + * @return array + */ + public static function default_encodings_provider(): array + { + return [ + 'v2 string' => [FlatColumn::string('c'), 2, 'a', Encodings::DELTA_BYTE_ARRAY], + 'v2 fixedSizeByteArray' => [FlatColumn::fixedSizeByteArray('c', 4), 2, 'abcd', Encodings::DELTA_BYTE_ARRAY], + 'v1 string' => [FlatColumn::string('c'), 1, 'a', Encodings::PLAIN], + 'v2 double' => [FlatColumn::double('c'), 2, 1.5, Encodings::PLAIN], + ]; + } + + /** + * @return array}> + */ + public static function boolean_encodings_provider(): array + { + return ['v1' => [1, [Encodings::RLE, Encodings::PLAIN]], 'v2' => [2, [Encodings::RLE]]]; + } + + #[DataProvider('boolean_encodings_provider')] + /** + * @param list $encodings + */ + public function test_a_boolean_column_gets_no_dictionary(int $writerVersion, array $encodings): void + { + $column = FlatColumn::boolean('flag'); + $options = Options::default()->set(Option::WRITER_VERSION, $writerVersion); + $builder = ColumnChunkBuilderFactory::createBuilder( + $column, + $options, + Compressions::UNCOMPRESSED, + StatisticsOptions::fromOptions($options), + ); + $builder->addColumn(new WriteFlatColumnValues($column, [0], [1], [true])); + + static::assertInstanceOf(FlatColumnChunkBuilder::class, $builder); + static::assertEqualsCanonicalizing($encodings, array_values($builder->flush(0)[0]->columnChunk->encodings())); + } + + /** + * @return array + */ + public static function dictionary_overflow_provider(): array + { + return [ + 'v2 string' => [FlatColumn::string('c'), 2, 'a', Encodings::DELTA_BYTE_ARRAY], + 'v2 fixedSizeByteArray' => [FlatColumn::fixedSizeByteArray('c', 4), 2, 'abcd', Encodings::DELTA_BYTE_ARRAY], + 'v2 int32' => [FlatColumn::int32('c'), 2, 7, Encodings::DELTA_BINARY_PACKED], + 'v2 int64' => [FlatColumn::int64('c'), 2, 7, Encodings::DELTA_BINARY_PACKED], + 'v2 double' => [FlatColumn::double('c'), 2, 1.5, Encodings::PLAIN], + 'v1 string' => [FlatColumn::string('c'), 1, 'a', Encodings::PLAIN], + ]; + } + + #[DataProvider('dictionary_overflow_provider')] + public function test_a_dictionary_overflow_falls_back_to_the_writer_version_fallback( + FlatColumn $column, + int $writerVersion, + bool|float|int|string $value, + Encodings $fallback, + ): void { + $options = Options::default()->set(Option::WRITER_VERSION, $writerVersion)->set( + Option::DICTIONARY_PAGE_SIZE, + 1, + ); + $builder = ColumnChunkBuilderFactory::createBuilder( + $column, + $options, + Compressions::UNCOMPRESSED, + StatisticsOptions::fromOptions($options), + ); + $builder->addColumn(new WriteFlatColumnValues($column, [0], [1], [$value])); + $encodings = $builder->flush(0)[0]->columnChunk->encodings(); + + static::assertInstanceOf(RLEDictionaryChunkBuilder::class, $builder); + static::assertContains($fallback, $encodings); + static::assertNotContains(Encodings::RLE_DICTIONARY, $encodings); + } + + #[DataProvider('default_encodings_provider')] + public function test_a_column_without_columns_encodings_writes_the_writer_version_fallback( + FlatColumn $column, + int $writerVersion, + bool|float|int|string $value, + Encodings $encoding, + ): void { + $options = Options::default()->set(Option::DICTIONARY_ENABLED, false)->set( + Option::WRITER_VERSION, + $writerVersion, + ); + $builder = ColumnChunkBuilderFactory::createBuilder( + $column, + $options, + Compressions::UNCOMPRESSED, + StatisticsOptions::fromOptions($options), + ); + $builder->addColumn(new WriteFlatColumnValues($column, [0], [1], [$value])); + + static::assertInstanceOf(FlatColumnChunkBuilder::class, $builder); + static::assertContains($encoding, $builder->flush(0)[0]->columnChunk->encodings()); + } + + public function test_create_builder_with_custom_delta_length_byte_array_encoding(): void + { + $column = FlatColumn::string('description'); + $options = Options::default()->set(Option::COLUMNS_ENCODINGS, [ + 'description' => Encodings::DELTA_LENGTH_BYTE_ARRAY, + ]); + $builder = ColumnChunkBuilderFactory::createBuilder( + $column, + $options, + Compressions::UNCOMPRESSED, + StatisticsOptions::fromOptions($options), + ); + $builder->addColumn(new WriteFlatColumnValues($column, [0], [1], ['a'])); + + static::assertContains(Encodings::DELTA_LENGTH_BYTE_ARRAY, $builder->flush(0)[0]->columnChunk->encodings()); + } + + public function test_invalid_delta_byte_array_encoding_for_int64_column_throws_exception(): void + { + $this->expectException(InvalidArgumentException::class); + $this->expectExceptionMessage( + 'DELTA_BYTE_ARRAY encoding is only supported for BYTE_ARRAY and FIXED_LEN_BYTE_ARRAY columns. ' + . "Column 'id' has type: INT64", + ); + + $options = Options::default()->set(Option::COLUMNS_ENCODINGS, ['id' => Encodings::DELTA_BYTE_ARRAY]); + + ColumnChunkBuilderFactory::createBuilder( + FlatColumn::int64('id'), + $options, + Compressions::UNCOMPRESSED, + StatisticsOptions::fromOptions($options), + ); + } + + public function test_invalid_delta_length_byte_array_encoding_for_fixed_len_byte_array_throws_exception(): void + { + $this->expectException(InvalidArgumentException::class); + $this->expectExceptionMessage( + "DELTA_LENGTH_BYTE_ARRAY encoding is only supported for BYTE_ARRAY columns. Column 'f' has type: FIXED_LEN_BYTE_ARRAY", + ); + + $options = Options::default()->set(Option::COLUMNS_ENCODINGS, ['f' => Encodings::DELTA_LENGTH_BYTE_ARRAY]); + + ColumnChunkBuilderFactory::createBuilder( + FlatColumn::fixedSizeByteArray('f', 4), + $options, + Compressions::UNCOMPRESSED, + StatisticsOptions::fromOptions($options), + ); + } + + /** + * @return array + */ + public static function dictionary_defaults_provider(): array + { + return [ + 'v1 int64' => [FlatColumn::int64('c'), 1, 7, Encodings::RLE_DICTIONARY], + 'v2 int64' => [FlatColumn::int64('c'), 2, 7, Encodings::RLE_DICTIONARY], + 'v1 double' => [FlatColumn::double('c'), 1, 1.5, Encodings::RLE_DICTIONARY], + 'v2 string' => [FlatColumn::string('c'), 2, 'a', Encodings::RLE_DICTIONARY], + 'v1 fixedSizeByteArray' => [FlatColumn::fixedSizeByteArray('c', 4), 1, 'abcd', Encodings::PLAIN], + 'v2 fixedSizeByteArray' => [FlatColumn::fixedSizeByteArray('c', 4), 2, 'abcd', Encodings::RLE_DICTIONARY], + ]; + } + + #[DataProvider('dictionary_defaults_provider')] + public function test_a_column_without_columns_encodings_is_dictionary_encoded_where_arrow_rs_does( + FlatColumn $column, + int $writerVersion, + bool|float|int|string $value, + Encodings $encoding, + ): void { + $options = Options::default()->set(Option::WRITER_VERSION, $writerVersion); + $builder = ColumnChunkBuilderFactory::createBuilder( + $column, + $options, + Compressions::UNCOMPRESSED, + StatisticsOptions::fromOptions($options), + ); + $builder->addColumn(new WriteFlatColumnValues($column, [0], [1], [$value])); + + static::assertContains($encoding, $builder->flush(0)[0]->columnChunk->encodings()); + } + + public function test_a_columns_encodings_dictionary_entry_wins_over_a_disabled_dictionary(): void + { + $options = Options::default()->set(Option::DICTIONARY_ENABLED, false)->set(Option::COLUMNS_ENCODINGS, [ + 'status' => Encodings::RLE_DICTIONARY, + ]); + + static::assertInstanceOf(RLEDictionaryChunkBuilder::class, ColumnChunkBuilderFactory::createBuilder( + FlatColumn::string('status'), + $options, + Compressions::UNCOMPRESSED, + StatisticsOptions::fromOptions($options), + )); + } + + public function test_create_builder_with_rle_dictionary_encoding_for_fixed_len_byte_array_under_writer_version_2(): void + { + $options = Options::default()->set(Option::WRITER_VERSION, 2)->set(Option::COLUMNS_ENCODINGS, [ + 'f' => Encodings::RLE_DICTIONARY, + ]); + + static::assertInstanceOf(RLEDictionaryChunkBuilder::class, ColumnChunkBuilderFactory::createBuilder( + FlatColumn::fixedSizeByteArray('f', 4), + $options, + Compressions::UNCOMPRESSED, + StatisticsOptions::fromOptions($options), + )); + } + + public function test_invalid_rle_encoding_for_int32_column_throws_exception(): void + { + $this->expectException(InvalidArgumentException::class); + $this->expectExceptionMessage( + "RLE encoding is only supported for BOOLEAN columns. Column 'id' has type: INT32", + ); + + $options = Options::default()->set(Option::COLUMNS_ENCODINGS, ['id' => Encodings::RLE]); + + ColumnChunkBuilderFactory::createBuilder( + FlatColumn::int32('id'), + $options, + Compressions::UNCOMPRESSED, + StatisticsOptions::fromOptions($options), + ); } } diff --git a/src/lib/parquet/tests/Flow/Parquet/Tests/Unit/Writer/ColumnChunkBuildersTest.php b/src/lib/parquet/tests/Flow/Parquet/Tests/Unit/Writer/ColumnChunkBuildersTest.php index 3f6b8e165f..5bf7b79376 100644 --- a/src/lib/parquet/tests/Flow/Parquet/Tests/Unit/Writer/ColumnChunkBuildersTest.php +++ b/src/lib/parquet/tests/Flow/Parquet/Tests/Unit/Writer/ColumnChunkBuildersTest.php @@ -22,8 +22,7 @@ use Flow\Parquet\ParquetFile\Schema\NestedColumn; use Flow\Parquet\ParquetFile\Schema\PhysicalType; use Flow\Parquet\Writer\ColumnChunkBuilder; -use Flow\Parquet\Writer\ColumnChunkBuilder\DeltaBinaryPackedColumnChunkBuilder; -use Flow\Parquet\Writer\ColumnChunkBuilder\PlainFlatColumnChunkBuilder; +use Flow\Parquet\Writer\ColumnChunkBuilder\FlatColumnChunkBuilder; use Flow\Parquet\Writer\ColumnChunkBuilder\RLEDictionaryChunkBuilder; use Flow\Parquet\Writer\ColumnChunkBuilders; use Flow\Parquet\Writer\ColumnChunkContainer; @@ -323,11 +322,25 @@ public function test_custom_encoding_mixed_with_default_selection(): void $buildersArray = $buildersProperty->getValue($builders); // Verify custom encodings - static::assertInstanceOf(DeltaBinaryPackedColumnChunkBuilder::class, $buildersArray['user_id']); + static::assertInstanceOf(FlatColumnChunkBuilder::class, $buildersArray['user_id']); + $buildersArray['user_id']->addColumn( + new WriteFlatColumnValues($userIdColumn, [0], [$userIdColumn->maxDefinitionsLevel()], [7]), + ); + static::assertContains( + Encodings::DELTA_BINARY_PACKED, + $buildersArray['user_id']->flush(0)[0]->columnChunk->encodings(), + ); static::assertInstanceOf(RLEDictionaryChunkBuilder::class, $buildersArray['status']); // Verify default encoding is used for description (not specified in custom encodings) - static::assertInstanceOf(PlainFlatColumnChunkBuilder::class, $buildersArray['description']); + static::assertInstanceOf(RLEDictionaryChunkBuilder::class, $buildersArray['description']); + $buildersArray['description']->addColumn( + new WriteFlatColumnValues($descriptionColumn, [0], [$descriptionColumn->maxDefinitionsLevel()], ['a']), + ); + static::assertContains( + Encodings::RLE_DICTIONARY, + $buildersArray['description']->flush(0)[0]->columnChunk->encodings(), + ); } public function test_custom_encoding_overrides_automatic_delta_selection(): void @@ -350,7 +363,11 @@ public function test_custom_encoding_overrides_automatic_delta_selection(): void $buildersArray = $buildersProperty->getValue($builders); static::assertIsArray($buildersArray); - static::assertInstanceOf(PlainFlatColumnChunkBuilder::class, $buildersArray['user_id']); + static::assertInstanceOf(FlatColumnChunkBuilder::class, $buildersArray['user_id']); + $buildersArray['user_id']->addColumn( + new WriteFlatColumnValues($flatColumn, [0], [$flatColumn->maxDefinitionsLevel()], [7]), + ); + static::assertContains(Encodings::PLAIN, $buildersArray['user_id']->flush(0)[0]->columnChunk->encodings()); } public function test_custom_encoding_selection_in_options(): void @@ -376,9 +393,16 @@ public function test_custom_encoding_selection_in_options(): void $buildersArray = $buildersProperty->getValue($builders); static::assertIsArray($buildersArray); - static::assertInstanceOf(PlainFlatColumnChunkBuilder::class, $buildersArray['col1']); + static::assertInstanceOf(FlatColumnChunkBuilder::class, $buildersArray['col1']); + $buildersArray['col1']->addColumn(new WriteFlatColumnValues($col1, [0], [$col1->maxDefinitionsLevel()], ['a'])); + static::assertContains(Encodings::PLAIN, $buildersArray['col1']->flush(0)[0]->columnChunk->encodings()); static::assertInstanceOf(RLEDictionaryChunkBuilder::class, $buildersArray['col2']); - static::assertInstanceOf(DeltaBinaryPackedColumnChunkBuilder::class, $buildersArray['col3']); + static::assertInstanceOf(FlatColumnChunkBuilder::class, $buildersArray['col3']); + $buildersArray['col3']->addColumn(new WriteFlatColumnValues($col3, [0], [$col3->maxDefinitionsLevel()], [7])); + static::assertContains( + Encodings::DELTA_BINARY_PACKED, + $buildersArray['col3']->flush(0)[0]->columnChunk->encodings(), + ); } // Custom encoding selection tests @@ -402,7 +426,14 @@ public function test_custom_encoding_selection_with_delta_binary_packed(): void $buildersArray = $buildersProperty->getValue($builders); static::assertIsArray($buildersArray); - static::assertInstanceOf(DeltaBinaryPackedColumnChunkBuilder::class, $buildersArray['user_id']); + static::assertInstanceOf(FlatColumnChunkBuilder::class, $buildersArray['user_id']); + $buildersArray['user_id']->addColumn( + new WriteFlatColumnValues($flatColumn, [0], [$flatColumn->maxDefinitionsLevel()], [7]), + ); + static::assertContains( + Encodings::DELTA_BINARY_PACKED, + $buildersArray['user_id']->flush(0)[0]->columnChunk->encodings(), + ); } public function test_custom_encoding_selection_with_nested_columns(): void @@ -442,7 +473,14 @@ public function test_custom_encoding_selection_with_nested_columns(): void // Verify the correct encoding builders were created // Order: [0] = user.id (DELTA_BINARY_PACKED), [1] = user.name (RLE_DICTIONARY) - static::assertInstanceOf(DeltaBinaryPackedColumnChunkBuilder::class, $childBuilderValues[0]); + static::assertInstanceOf(FlatColumnChunkBuilder::class, $childBuilderValues[0]); + $childBuilderValues[0]->addColumn( + new WriteFlatColumnValues($userIdColumn, [0], [$userIdColumn->maxDefinitionsLevel()], [7]), + ); + static::assertContains( + Encodings::DELTA_BINARY_PACKED, + $childBuilderValues[0]->flush(0)[0]->columnChunk->encodings(), + ); static::assertInstanceOf(RLEDictionaryChunkBuilder::class, $childBuilderValues[1]); } @@ -465,7 +503,11 @@ public function test_custom_encoding_selection_with_plain(): void $buildersArray = $buildersProperty->getValue($builders); static::assertIsArray($buildersArray); - static::assertInstanceOf(PlainFlatColumnChunkBuilder::class, $buildersArray['description']); + static::assertInstanceOf(FlatColumnChunkBuilder::class, $buildersArray['description']); + $buildersArray['description']->addColumn( + new WriteFlatColumnValues($flatColumn, [0], [$flatColumn->maxDefinitionsLevel()], ['a']), + ); + static::assertContains(Encodings::PLAIN, $buildersArray['description']->flush(0)[0]->columnChunk->encodings()); } public function test_custom_encoding_selection_with_rle_dictionary(): void @@ -593,7 +635,14 @@ public function test_empty_columns_encodings_option(): void static::assertIsArray($buildersArray); // Should use default PLAIN encoding since no custom encoding specified - static::assertInstanceOf(PlainFlatColumnChunkBuilder::class, $buildersArray['user_id']); + static::assertInstanceOf(RLEDictionaryChunkBuilder::class, $buildersArray['user_id']); + $buildersArray['user_id']->addColumn( + new WriteFlatColumnValues($flatColumn, [0], [$flatColumn->maxDefinitionsLevel()], [7]), + ); + static::assertContains( + Encodings::RLE_DICTIONARY, + $buildersArray['user_id']->flush(0)[0]->columnChunk->encodings(), + ); } public function test_flush_calculates_offset_correctly(): void diff --git a/src/lib/parquet/tests/Flow/Parquet/Tests/Unit/Writer/ColumnEncodingPolicyTest.php b/src/lib/parquet/tests/Flow/Parquet/Tests/Unit/Writer/ColumnEncodingPolicyTest.php new file mode 100644 index 0000000000..f6a4247a5c --- /dev/null +++ b/src/lib/parquet/tests/Flow/Parquet/Tests/Unit/Writer/ColumnEncodingPolicyTest.php @@ -0,0 +1,156 @@ + + */ + public static function default_encodings_provider(): array + { + return [ + 'v1 boolean' => [FlatColumn::boolean('c'), 1, true, Encodings::PLAIN], + 'v2 boolean' => [FlatColumn::boolean('c'), 2, true, Encodings::RLE], + 'v1 int32' => [FlatColumn::int32('c'), 1, true, Encodings::RLE_DICTIONARY], + 'v2 int64 without dictionary' => [FlatColumn::int64('c'), 2, false, Encodings::DELTA_BINARY_PACKED], + 'v1 int64 without dictionary' => [FlatColumn::int64('c'), 1, false, Encodings::PLAIN], + 'v2 double without dictionary' => [FlatColumn::double('c'), 2, false, Encodings::PLAIN], + 'v2 string' => [FlatColumn::string('c'), 2, true, Encodings::RLE_DICTIONARY], + 'v2 string without dictionary' => [FlatColumn::string('c'), 2, false, Encodings::DELTA_BYTE_ARRAY], + 'v1 fixedSizeByteArray' => [FlatColumn::fixedSizeByteArray('c', 4), 1, true, Encodings::PLAIN], + 'v2 fixedSizeByteArray' => [FlatColumn::fixedSizeByteArray('c', 4), 2, true, Encodings::RLE_DICTIONARY], + 'v2 fixedSizeByteArray without dictionary' => [ + FlatColumn::fixedSizeByteArray('c', 4), + 2, + false, + Encodings::DELTA_BYTE_ARRAY, + ], + ]; + } + + /** + * @return array + */ + public static function refused_encodings_provider(): array + { + return [ + 'RLE on int32' => [ + FlatColumn::int32('c'), + 1, + Encodings::RLE, + "RLE encoding is only supported for BOOLEAN columns. Column 'c' has type: INT32", + ], + 'RLE_DICTIONARY on boolean' => [ + FlatColumn::boolean('c'), + 2, + Encodings::RLE_DICTIONARY, + "RLE_DICTIONARY encoding is not supported for BOOLEAN columns. Column 'c' has type: BOOLEAN", + ], + 'RLE_DICTIONARY on fixedSizeByteArray under v1' => [ + FlatColumn::fixedSizeByteArray('c', 4), + 1, + Encodings::RLE_DICTIONARY, + 'RLE_DICTIONARY encoding is not supported for FIXED_LEN_BYTE_ARRAY columns under WRITER_VERSION 1. ' + . "Column 'c' has type: FIXED_LEN_BYTE_ARRAY", + ], + 'DELTA_BINARY_PACKED on double' => [ + FlatColumn::double('c'), + 1, + Encodings::DELTA_BINARY_PACKED, + "DELTA_BINARY_PACKED encoding is only supported for INT32 and INT64 columns. Column 'c' has type: DOUBLE", + ], + 'DELTA_LENGTH_BYTE_ARRAY on fixedSizeByteArray' => [ + FlatColumn::fixedSizeByteArray('c', 4), + 2, + Encodings::DELTA_LENGTH_BYTE_ARRAY, + 'DELTA_LENGTH_BYTE_ARRAY encoding is only supported for BYTE_ARRAY columns. ' + . "Column 'c' has type: FIXED_LEN_BYTE_ARRAY", + ], + 'DELTA_BYTE_ARRAY on int64' => [ + FlatColumn::int64('c'), + 1, + Encodings::DELTA_BYTE_ARRAY, + 'DELTA_BYTE_ARRAY encoding is only supported for BYTE_ARRAY and FIXED_LEN_BYTE_ARRAY columns. ' + . "Column 'c' has type: INT64", + ], + 'BYTE_STREAM_SPLIT' => [ + FlatColumn::double('c'), + 1, + Encodings::BYTE_STREAM_SPLIT, + "Encoding 'BYTE_STREAM_SPLIT' is not implemented. Supported encodings: PLAIN, RLE, RLE_DICTIONARY, " + . 'DELTA_BINARY_PACKED, DELTA_LENGTH_BYTE_ARRAY, DELTA_BYTE_ARRAY', + ], + ]; + } + + #[DataProvider('default_encodings_provider')] + public function test_a_column_without_a_requested_encoding_gets_the_default( + FlatColumn $column, + int $writerVersion, + bool $dictionaryEnabled, + Encodings $encoding, + ): void { + static::assertSame( + $encoding, + (new ColumnEncodingPolicy($writerVersion, $dictionaryEnabled))->encodingFor($column), + ); + } + + public function test_a_requested_dictionary_wins_over_a_disabled_dictionary(): void + { + static::assertSame(Encodings::RLE_DICTIONARY, (new ColumnEncodingPolicy(1, false))->encodingFor( + FlatColumn::string('c'), + Encodings::RLE_DICTIONARY, + )); + } + + public function test_a_requested_encoding_the_column_takes_is_kept(): void + { + static::assertSame(Encodings::DELTA_LENGTH_BYTE_ARRAY, (new ColumnEncodingPolicy(1))->encodingFor( + FlatColumn::string('c'), + Encodings::DELTA_LENGTH_BYTE_ARRAY, + )); + } + + #[DataProvider('refused_encodings_provider')] + public function test_a_requested_encoding_the_column_cannot_take_is_refused( + FlatColumn $column, + int $writerVersion, + Encodings $encoding, + string $message, + ): void { + $this->expectException(InvalidArgumentException::class); + $this->expectExceptionMessage($message); + + (new ColumnEncodingPolicy($writerVersion))->encodingFor($column, $encoding); + } + + public function test_fallback_encodings_follow_arrow_rs(): void + { + $v1 = new ColumnEncodingPolicy(1); + $v2 = new ColumnEncodingPolicy(2); + + foreach (PhysicalType::cases() as $type) { + static::assertSame(Encodings::PLAIN, $v1->fallbackEncoding($type), $type->name); + } + + static::assertSame(Encodings::RLE, $v2->fallbackEncoding(PhysicalType::BOOLEAN)); + static::assertSame(Encodings::DELTA_BINARY_PACKED, $v2->fallbackEncoding(PhysicalType::INT32)); + static::assertSame(Encodings::DELTA_BINARY_PACKED, $v2->fallbackEncoding(PhysicalType::INT64)); + static::assertSame(Encodings::DELTA_BYTE_ARRAY, $v2->fallbackEncoding(PhysicalType::BYTE_ARRAY)); + static::assertSame(Encodings::DELTA_BYTE_ARRAY, $v2->fallbackEncoding(PhysicalType::FIXED_LEN_BYTE_ARRAY)); + static::assertSame(Encodings::PLAIN, $v2->fallbackEncoding(PhysicalType::DOUBLE)); + static::assertSame(Encodings::PLAIN, $v2->fallbackEncoding(PhysicalType::INT96)); + } +} diff --git a/src/lib/parquet/tests/Flow/Parquet/Tests/Unit/Writer/PageBuilder/DictionaryBuilderTest.php b/src/lib/parquet/tests/Flow/Parquet/Tests/Unit/Writer/PageBuilder/DictionaryBuilderTest.php index 9fe64e04f7..65aefab70d 100644 --- a/src/lib/parquet/tests/Flow/Parquet/Tests/Unit/Writer/PageBuilder/DictionaryBuilderTest.php +++ b/src/lib/parquet/tests/Flow/Parquet/Tests/Unit/Writer/PageBuilder/DictionaryBuilderTest.php @@ -318,15 +318,14 @@ public function test_single_unique_value_creates_single_entry_dictionary(): void static::assertSame([0, 0, 0, 0], $result->indices); } - public function test_unsupported_byte_array_logical_type_throws_exception(): void + public function test_byte_array_without_a_listed_logical_type_builds_a_scalar_dictionary(): void { - $column = new FlatColumn('test_column', PhysicalType::BYTE_ARRAY, logicalType: LogicalType::integer(32, true)); - $data = new WriteFlatColumnValues($column, values: [1, 2, 3]); + $column = new FlatColumn('test_column', PhysicalType::FIXED_LEN_BYTE_ARRAY, typeLength: 2); - $this->expectException(RuntimeException::class); - $this->expectExceptionMessage('Building dictionary for "INTEGER" is not supported'); + $dictionary = $this->builder->build($column, new WriteFlatColumnValues($column, values: ['ab', 'cd', 'ab'])); - $this->builder->build($column, $data); + static::assertSame(['ab', 'cd'], $dictionary->dictionary); + static::assertSame([0, 1, 0], $dictionary->indices); } public function test_unsupported_physical_type_throws_exception(): void diff --git a/src/lib/parquet/tests/Flow/Parquet/Tests/Unit/Writer/ValueStorage/BooleanValueStorageTest.php b/src/lib/parquet/tests/Flow/Parquet/Tests/Unit/Writer/ValueStorage/BooleanValueStorageTest.php new file mode 100644 index 0000000000..9ed06a63d7 --- /dev/null +++ b/src/lib/parquet/tests/Flow/Parquet/Tests/Unit/Writer/ValueStorage/BooleanValueStorageTest.php @@ -0,0 +1,36 @@ +encoding()); + } + + /** + * @param non-negative-int $values + */ + #[TestWith([0, 0])] + #[TestWith([1, 1])] + #[TestWith([8, 1])] + #[TestWith([9, 2])] + public function test_plain_size_is_the_bit_packed_plain_size(int $values, int $bytes): void + { + $storage = new BooleanValueStorage(); + $storage->addValues(FlatColumn::boolean('c'), array_fill(0, $values, true)); + + static::assertSame($bytes, $storage->plainSize()); + } +} diff --git a/src/lib/parquet/tests/Flow/Parquet/Tests/Unit/Writer/ValueStorage/BufferValueStorageTest.php b/src/lib/parquet/tests/Flow/Parquet/Tests/Unit/Writer/ValueStorage/BufferValueStorageTest.php new file mode 100644 index 0000000000..fc8e69ed6b --- /dev/null +++ b/src/lib/parquet/tests/Flow/Parquet/Tests/Unit/Writer/ValueStorage/BufferValueStorageTest.php @@ -0,0 +1,17 @@ +encoding()); + } +} diff --git a/src/lib/parquet/tests/Flow/Parquet/Tests/Unit/Writer/ValueStorage/DeltaBinaryPackedValueStorageTest.php b/src/lib/parquet/tests/Flow/Parquet/Tests/Unit/Writer/ValueStorage/DeltaBinaryPackedValueStorageTest.php new file mode 100644 index 0000000000..73dd725c85 --- /dev/null +++ b/src/lib/parquet/tests/Flow/Parquet/Tests/Unit/Writer/ValueStorage/DeltaBinaryPackedValueStorageTest.php @@ -0,0 +1,75 @@ +encoding()); + } + + public function test_get_buffer_encodes_non_null_values(): void + { + $storage = new DeltaBinaryPackedValueStorage(); + $storage->addValues(FlatColumn::int32('c'), [1, null, 2]); + + static::assertSame((new DeltaBinaryPackedEncoder())->encode([1, 2]), $storage->getBuffer()); + } + + public function test_get_buffer_without_values_is_an_empty_stream_header(): void + { + static::assertSame("\x80\x01\x04\x00\x00", (new DeltaBinaryPackedValueStorage())->getBuffer()); + } + + public function test_rejects_a_non_integer_value(): void + { + $this->expectException(InvalidArgumentException::class); + $this->expectExceptionMessage('Delta encoding requires integer values, got string'); + + (new DeltaBinaryPackedValueStorage())->addValues(FlatColumn::int32('c'), ['1']); + } + + public function test_rejects_a_non_integer_column(): void + { + $this->expectException(InvalidArgumentException::class); + $this->expectExceptionMessage('Delta encoding only supports INT32 and INT64 physical types'); + + (new DeltaBinaryPackedValueStorage())->addValues(FlatColumn::string('c'), ['a']); + } + + public function test_reset_clears_values_and_size(): void + { + $storage = new DeltaBinaryPackedValueStorage(); + $storage->addValues(FlatColumn::int32('c'), [1]); + $storage->reset(); + + static::assertTrue($storage->isEmpty()); + static::assertSame(0, $storage->plainSize()); + } + + public function test_plain_size_is_the_plain_size_of_int32_values(): void + { + $storage = new DeltaBinaryPackedValueStorage(); + $storage->addValues(FlatColumn::int32('c'), [1, null, 2]); + + static::assertSame(8, $storage->plainSize()); + } + + public function test_plain_size_is_the_plain_size_of_int64_values(): void + { + $storage = new DeltaBinaryPackedValueStorage(); + $storage->addValues(FlatColumn::int64('c'), [1, null, 2]); + + static::assertSame(16, $storage->plainSize()); + } +} diff --git a/src/lib/parquet/tests/Flow/Parquet/Tests/Unit/Writer/ValueStorage/DeltaByteArrayValueStorageTest.php b/src/lib/parquet/tests/Flow/Parquet/Tests/Unit/Writer/ValueStorage/DeltaByteArrayValueStorageTest.php new file mode 100644 index 0000000000..00ea0d867c --- /dev/null +++ b/src/lib/parquet/tests/Flow/Parquet/Tests/Unit/Writer/ValueStorage/DeltaByteArrayValueStorageTest.php @@ -0,0 +1,78 @@ +encoding()); + } + + public function test_get_buffer_encodes_non_null_values(): void + { + $storage = new DeltaByteArrayValueStorage(); + $storage->addValues(FlatColumn::string('c'), ['axis', null, 'axle']); + + static::assertSame((new DeltaByteArrayEncoder())->encode(['axis', 'axle']), $storage->getBuffer()); + } + + public function test_get_buffer_without_values_is_two_empty_length_streams(): void + { + static::assertSame("\x80\x01\x04\x00\x00\x80\x01\x04\x00\x00", (new DeltaByteArrayValueStorage())->getBuffer()); + } + + public function test_rejects_a_non_byte_array_column(): void + { + $this->expectException(InvalidArgumentException::class); + $this->expectExceptionMessage( + 'DELTA_BYTE_ARRAY encoding only supports BYTE_ARRAY and FIXED_LEN_BYTE_ARRAY physical types', + ); + + (new DeltaByteArrayValueStorage())->addValues(FlatColumn::int64('c'), [1]); + } + + public function test_rejects_a_non_string_value(): void + { + $this->expectException(InvalidArgumentException::class); + $this->expectExceptionMessage('DELTA_BYTE_ARRAY encoding requires string values, got integer'); + + (new DeltaByteArrayValueStorage())->addValues(FlatColumn::string('c'), [1]); + } + + public function test_reset_clears_values_and_size(): void + { + $storage = new DeltaByteArrayValueStorage(); + $storage->addValues(FlatColumn::string('c'), ['a']); + $storage->reset(); + + static::assertTrue($storage->isEmpty()); + static::assertSame(0, $storage->plainSize()); + } + + public function test_plain_size_is_the_plain_size_of_byte_array_values(): void + { + $storage = new DeltaByteArrayValueStorage(); + $storage->addValues(FlatColumn::string('c'), ['ab', null, '']); + + static::assertFalse($storage->isEmpty()); + static::assertSame(10, $storage->plainSize()); + } + + public function test_plain_size_is_the_plain_size_of_fixed_len_byte_array_values(): void + { + $storage = new DeltaByteArrayValueStorage(); + $storage->addValues(FlatColumn::fixedSizeByteArray('c', 4), ['abcd', null, 'abce']); + + static::assertSame(8, $storage->plainSize()); + } +} diff --git a/src/lib/parquet/tests/Flow/Parquet/Tests/Unit/Writer/ValueStorage/DeltaLengthByteArrayValueStorageTest.php b/src/lib/parquet/tests/Flow/Parquet/Tests/Unit/Writer/ValueStorage/DeltaLengthByteArrayValueStorageTest.php new file mode 100644 index 0000000000..f21e721cfa --- /dev/null +++ b/src/lib/parquet/tests/Flow/Parquet/Tests/Unit/Writer/ValueStorage/DeltaLengthByteArrayValueStorageTest.php @@ -0,0 +1,68 @@ +encoding()); + } + + public function test_get_buffer_encodes_non_null_values(): void + { + $storage = new DeltaLengthByteArrayValueStorage(); + $storage->addValues(FlatColumn::string('c'), ['ab', null, '']); + + static::assertSame((new DeltaLengthByteArrayEncoder())->encode(['ab', '']), $storage->getBuffer()); + } + + public function test_get_buffer_without_values_is_an_empty_length_stream(): void + { + static::assertSame("\x80\x01\x04\x00\x00", (new DeltaLengthByteArrayValueStorage())->getBuffer()); + } + + public function test_rejects_a_fixed_len_byte_array_column(): void + { + $this->expectException(InvalidArgumentException::class); + $this->expectExceptionMessage('DELTA_LENGTH_BYTE_ARRAY encoding only supports BYTE_ARRAY physical type'); + + (new DeltaLengthByteArrayValueStorage())->addValues(FlatColumn::fixedSizeByteArray('c', 4), ['abcd']); + } + + public function test_rejects_a_non_string_value(): void + { + $this->expectException(InvalidArgumentException::class); + $this->expectExceptionMessage('DELTA_LENGTH_BYTE_ARRAY encoding requires string values, got integer'); + + (new DeltaLengthByteArrayValueStorage())->addValues(FlatColumn::string('c'), [1]); + } + + public function test_reset_clears_values_and_size(): void + { + $storage = new DeltaLengthByteArrayValueStorage(); + $storage->addValues(FlatColumn::string('c'), ['a']); + $storage->reset(); + + static::assertTrue($storage->isEmpty()); + static::assertSame(0, $storage->plainSize()); + } + + public function test_plain_size_is_the_plain_size_of_the_values(): void + { + $storage = new DeltaLengthByteArrayValueStorage(); + $storage->addValues(FlatColumn::string('c'), ['ab', null, '']); + + static::assertFalse($storage->isEmpty()); + static::assertSame(10, $storage->plainSize()); + } +} diff --git a/src/lib/parquet/tests/Flow/Parquet/Tests/Unit/Writer/ValueStorage/RLEBooleanValueStorageTest.php b/src/lib/parquet/tests/Flow/Parquet/Tests/Unit/Writer/ValueStorage/RLEBooleanValueStorageTest.php new file mode 100644 index 0000000000..9cc2be5251 --- /dev/null +++ b/src/lib/parquet/tests/Flow/Parquet/Tests/Unit/Writer/ValueStorage/RLEBooleanValueStorageTest.php @@ -0,0 +1,65 @@ +encoding()); + } + + public function test_get_buffer_is_the_length_prefixed_hybrid_of_the_non_null_values(): void + { + $storage = new RLEBooleanValueStorage(); + $storage->addValues(FlatColumn::boolean('c'), [true, null, false, true]); + + static::assertSame( + (new RLEBitPackedPacker(new RLEBitPackedHybrid()))->packWithLength(1, [1, 0, 1]), + $storage->getBuffer(), + ); + } + + public function test_get_buffer_without_values_is_an_empty_run(): void + { + static::assertSame("\x00\x00\x00\x00", (new RLEBooleanValueStorage())->getBuffer()); + } + + public function test_reset_clears_the_values(): void + { + $storage = new RLEBooleanValueStorage(); + $storage->addValues(FlatColumn::boolean('c'), [true]); + $storage->reset(); + + static::assertTrue($storage->isEmpty()); + static::assertSame(0, $storage->plainSize()); + } + + /** + * @param non-negative-int $values + */ + #[TestWith([0, 0])] + #[TestWith([1, 1])] + #[TestWith([8, 1])] + #[TestWith([9, 2])] + public function test_plain_size_is_the_bit_packed_plain_size(int $values, int $bytes): void + { + $storage = new RLEBooleanValueStorage(); + $storage->addValues(FlatColumn::boolean('c'), array_fill(0, $values, true)); + + static::assertSame($bytes, $storage->plainSize()); + static::assertSame($values === 0, $storage->isEmpty()); + } +}