From 0870ff883f25f3cdb45aa8630d490c84c4756a17 Mon Sep 17 00:00:00 2001 From: Yuang Gao Date: Tue, 20 Oct 2026 20:23:43 -0700 Subject: [PATCH] fix: map write.parquet.page-row-limit to pyarrow's max_rows_per_page --- pyiceberg/io/pyarrow.py | 13 +++++++++---- tests/integration/test_writes/test_writes.py | 1 + 2 files changed, 10 insertions(+), 4 deletions(-) diff --git a/pyiceberg/io/pyarrow.py b/pyiceberg/io/pyarrow.py index 7fd0c85b77..874646b711 100644 --- a/pyiceberg/io/pyarrow.py +++ b/pyiceberg/io/pyarrow.py @@ -2986,7 +2986,7 @@ def _get_parquet_writer_kwargs(table_properties: Properties) -> dict[str, Any]: if compression_codec == ICEBERG_UNCOMPRESSED_CODEC: compression_codec = PYARROW_UNCOMPRESSED_CODEC - return { + kwargs: dict[str, Any] = { "compression": compression_codec, "compression_level": compression_level, "data_page_size": property_as_int( @@ -2999,12 +2999,17 @@ def _get_parquet_writer_kwargs(table_properties: Properties) -> dict[str, Any]: property_name=TableProperties.PARQUET_DICT_SIZE_BYTES, default=TableProperties.PARQUET_DICT_SIZE_BYTES_DEFAULT, ), - "write_batch_size": property_as_int( + } + # pyarrow added max_rows_per_page, the per-page row limit, in 23.0.0. + if tuple(map(int, pa.__version__.split(".")[:2])) >= (23, 0): + kwargs["max_rows_per_page"] = property_as_int( properties=table_properties, property_name=TableProperties.PARQUET_PAGE_ROW_LIMIT, default=TableProperties.PARQUET_PAGE_ROW_LIMIT_DEFAULT, - ), - } + ) + elif TableProperties.PARQUET_PAGE_ROW_LIMIT in table_properties: + warnings.warn(f"{TableProperties.PARQUET_PAGE_ROW_LIMIT} requires pyarrow >= 23.0.0", stacklevel=2) + return kwargs def _dataframe_to_data_files( diff --git a/tests/integration/test_writes/test_writes.py b/tests/integration/test_writes/test_writes.py index 30fdd76ab7..fe720440fb 100644 --- a/tests/integration/test_writes/test_writes.py +++ b/tests/integration/test_writes/test_writes.py @@ -669,6 +669,7 @@ def test_write_parquet_compression_properties( [ ({"write.parquet.page-size-bytes": "42"}, {"data_page_size": 42}), ({"write.parquet.dict-size-bytes": "42"}, {"dictionary_pagesize_limit": 42}), + ({"write.parquet.page-row-limit": "42"}, {"max_rows_per_page": 42, "write_batch_size": None}), ], ) def test_write_parquet_other_properties(