diff --git a/pyiceberg/io/pyarrow.py b/pyiceberg/io/pyarrow.py index 20333882b5..b4f34641d5 100644 --- a/pyiceberg/io/pyarrow.py +++ b/pyiceberg/io/pyarrow.py @@ -159,6 +159,7 @@ from pyiceberg.table.metadata import TableMetadata from pyiceberg.table.metadata_columns import LAST_UPDATED_SEQUENCE_NUMBER, ROW_ID from pyiceberg.table.name_mapping import NameMapping, apply_name_mapping +from pyiceberg.table.sorting import NullOrder, SortDirection from pyiceberg.transforms import IdentityTransform, TruncateTransform from pyiceberg.typedef import EMPTY_DICT, Properties, Record, TableVersion from pyiceberg.types import ( @@ -3056,10 +3057,7 @@ def write_data_file(task: WriteTask) -> DataFile: file_format=file_format, partition=task.partition_key.partition if task.partition_key else Record(), file_size_in_bytes=len(fo), - # After this has been fixed: - # https://github.com/apache/iceberg-python/issues/271 - # sort_order_id=task.sort_order_id, - sort_order_id=None, + sort_order_id=task.sort_order_id, # Just copy these from the table for now spec_id=table_metadata.default_spec_id, equality_ids=None, @@ -3304,6 +3302,11 @@ def _dataframe_to_data_files( "Materialise the reader as a pa.Table first, or follow " "https://github.com/apache/iceberg-python/issues/2152 for partitioned streaming support." ) + if not table_metadata.sort_order().is_unsorted: + warnings.warn( + "Sort order is not applied to streaming RecordBatchReader writes; data files are marked unsorted", + stacklevel=2, + ) yield from write_file( io=io, table_metadata=table_metadata, @@ -3315,31 +3318,82 @@ def _dataframe_to_data_files( return if table_metadata.spec().is_unpartitioned(): + sorted_df, sort_order_id = _sort_table_for_write(table_metadata, df) yield from write_file( io=io, table_metadata=table_metadata, tasks=( - WriteTask(write_uuid=write_uuid, task_id=next(counter), record_batches=batches, schema=task_schema) - for batches in bin_pack_arrow_table(df, target_file_size) + WriteTask( + write_uuid=write_uuid, + task_id=next(counter), + record_batches=batches, + schema=task_schema, + sort_order_id=sort_order_id, + ) + for batches in bin_pack_arrow_table(sorted_df, target_file_size) ), ) else: partitions = _determine_partitions(spec=table_metadata.spec(), schema=table_metadata.schema(), arrow_table=df) - yield from write_file( - io=io, - table_metadata=table_metadata, - tasks=( + tasks: list[WriteTask] = [] + for partition in partitions: + sorted_partition, sort_order_id = _sort_table_for_write(table_metadata, partition.arrow_table_partition) + tasks.extend( WriteTask( write_uuid=write_uuid, task_id=next(counter), record_batches=batches, partition_key=partition.partition_key, schema=task_schema, + sort_order_id=sort_order_id, ) - for partition in partitions - for batches in bin_pack_arrow_table(partition.arrow_table_partition, target_file_size) - ), - ) + for batches in bin_pack_arrow_table(sorted_partition, target_file_size) + ) + yield from write_file(io=io, table_metadata=table_metadata, tasks=iter(tasks)) + + +def _sort_table_for_write(table_metadata: TableMetadata, table: pa.Table) -> tuple[pa.Table, int | None]: + """Sort the table for the table's sort order and return its file sort-order id. + + Returns the table unchanged and ``None`` when the sort order cannot be applied, so the + written files are marked unsorted. + """ + sort_order = table_metadata.sort_order() + if sort_order.is_unsorted: + return table, None + + schema = table_metadata.schema() + sort_keys: list[tuple[str, str]] = [] + null_orders: set[NullOrder] = set() + for sort_field in sort_order.fields: + if not isinstance(sort_field.transform, IdentityTransform): + warnings.warn( + f"Sort transform {sort_field.transform} is not supported for writes; data files are marked unsorted", + stacklevel=2, + ) + return table, None + column_name = schema.find_column_name(sort_field.source_id) + if column_name is None or column_name not in table.column_names: + warnings.warn( + f"Sort field with source id {sort_field.source_id} refers to a nested or missing column; " + "data files are marked unsorted", + stacklevel=2, + ) + return table, None + sort_keys.append((column_name, "descending" if sort_field.direction == SortDirection.DESC else "ascending")) + null_orders.add(sort_field.null_order) + + if len(null_orders) != 1: + warnings.warn("Mixed null ordering is not supported for writes; data files are marked unsorted", stacklevel=2) + return table, None + + null_placement = "at_start" if NullOrder.NULLS_FIRST in null_orders else "at_end" + if int(pa.__version__.split(".")[0]) >= 25: + # Per-key null placement replaced the SortOptions-level argument in Arrow 25. + indices = pc.sort_indices(table, sort_keys=cast(Any, [(*key, null_placement) for key in sort_keys])) + else: + indices = pc.sort_indices(table, sort_keys=cast(Any, sort_keys), null_placement=cast(Any, null_placement)) + return table.take(indices), sort_order.order_id @dataclass(frozen=True) diff --git a/tests/io/test_pyarrow.py b/tests/io/test_pyarrow.py index 439405d121..c41a7e000c 100644 --- a/tests/io/test_pyarrow.py +++ b/tests/io/test_pyarrow.py @@ -86,10 +86,12 @@ _check_pyarrow_schema_compatible, _ConvertToArrowSchema, _create_arrow_read_format, + _dataframe_to_data_files, _determine_partitions, _primitive_to_physical, _read_all_delete_files, _read_deletes, + _sort_table_for_write, _task_to_record_batches, _to_requested_schema, bin_pack_arrow_table, @@ -111,10 +113,11 @@ PROPERTY_REFERENCED_DATA_FILE, deletion_vectors_from_puffin_file, ) -from pyiceberg.table.metadata import TableMetadataV2, TableMetadataV3 +from pyiceberg.table.metadata import TableMetadataV2, TableMetadataV3, new_table_metadata from pyiceberg.table.metadata_columns import LAST_UPDATED_SEQUENCE_NUMBER, ROW_ID from pyiceberg.table.name_mapping import create_mapping_from_schema from pyiceberg.table.puffin import MAGIC_BYTES, PuffinFile +from pyiceberg.table.sorting import NullOrder, SortDirection, SortField, SortOrder from pyiceberg.transforms import BucketTransform, HourTransform, IdentityTransform, Transform, TruncateTransform from pyiceberg.typedef import UTF8, Properties, Record, TableVersion from pyiceberg.types import ( @@ -3111,6 +3114,125 @@ def test_bin_pack_arrow_table_target_size_smaller_than_row(arrow_table_with_null assert sum(batch.num_rows for bin_ in bin_packed for batch in bin_) == arrow_table_with_null.num_rows +def test_sort_table_for_write_uses_identity_sort_order() -> None: + schema = Schema( + NestedField(1, "id", LongType(), required=False), + NestedField(2, "value", StringType(), required=False), + ) + cases = [ + (SortDirection.ASC, NullOrder.NULLS_LAST, [1, 2, None]), + (SortDirection.ASC, NullOrder.NULLS_FIRST, [None, 1, 2]), + (SortDirection.DESC, NullOrder.NULLS_LAST, [2, 1, None]), + (SortDirection.DESC, NullOrder.NULLS_FIRST, [None, 2, 1]), + ] + + for direction, null_order, expected in cases: + metadata = new_table_metadata( + schema=schema, + partition_spec=PartitionSpec(), + sort_order=SortOrder(SortField(1, IdentityTransform(), direction, null_order)), + location="file:///tmp/sorted", + ) + table = pa.table({"id": [2, None, 1], "value": ["b", "null", "a"]}) + + sorted_table, sort_order_id = _sort_table_for_write(metadata, table) + + assert sorted_table["id"].to_pylist() == expected + assert sort_order_id == metadata.default_sort_order_id + + +def test_sort_table_for_write_leaves_unsupported_sort_orders_unsorted() -> None: + schema = Schema( + NestedField(1, "id", LongType(), required=False), + NestedField(2, "value", StringType(), required=False), + ) + table = pa.table({"id": [2, 1], "value": ["b", "a"]}) + cases = [ + ( + SortOrder(SortField(1, TruncateTransform(10))), + "Sort transform truncate\\[10\\] is not supported", + ), + ( + SortOrder( + SortField(1, IdentityTransform(), null_order=NullOrder.NULLS_FIRST), + SortField(2, IdentityTransform(), null_order=NullOrder.NULLS_LAST), + ), + "Mixed null ordering", + ), + ] + + for sort_order, message in cases: + metadata = new_table_metadata( + schema=schema, + partition_spec=PartitionSpec(), + sort_order=sort_order, + location="file:///tmp/unsorted", + ) + + with pytest.warns(UserWarning, match=message): + sorted_table, sort_order_id = _sort_table_for_write(metadata, table) + + assert sorted_table is table + assert sort_order_id is None + + metadata = new_table_metadata( + schema=schema, + partition_spec=PartitionSpec(), + sort_order=SortOrder(SortField(2, IdentityTransform())), + location="file:///tmp/unsorted-missing-column", + ) + table_without_sort_column = pa.table({"id": [2, 1]}) + + with pytest.warns(UserWarning, match="nested or missing column"): + sorted_table, sort_order_id = _sort_table_for_write(metadata, table_without_sort_column) + + assert sorted_table is table_without_sort_column + assert sort_order_id is None + + +def test_write_sorted_data_files_per_partition(tmp_path: Path) -> None: + schema = Schema( + NestedField(1, "id", LongType(), required=False), + NestedField(2, "category", StringType(), required=False), + ) + metadata = new_table_metadata( + schema=schema, + partition_spec=PartitionSpec(PartitionField(source_id=2, field_id=1000, transform=IdentityTransform(), name="category")), + sort_order=SortOrder(SortField(1, IdentityTransform(), SortDirection.ASC, NullOrder.NULLS_LAST)), + location=f"file://{tmp_path}/sorted-partitioned", + ) + table = pa.table({"id": [3, None, 1, 2, None, 4], "category": ["a", "b", "b", "a", "a", "b"]}) + io = PyArrowFileIO() + + data_files = list(_dataframe_to_data_files(table_metadata=metadata, df=table, io=io)) + + assert len(data_files) == 2 + for data_file in data_files: + assert data_file.sort_order_id == metadata.default_sort_order_id + with io.new_input(data_file.file_path).open() as f: + read_table = pq.read_table(f) + assert len(read_table["category"].unique()) == 1 + ids = read_table["id"].to_pylist() + assert ids == sorted(ids, key=lambda value: (value is None, value)) + + +def test_streaming_write_ignores_sort_order(tmp_path: Path) -> None: + schema = Schema(NestedField(1, "id", LongType(), required=False)) + metadata = new_table_metadata( + schema=schema, + partition_spec=PartitionSpec(), + sort_order=SortOrder(SortField(1, IdentityTransform())), + location=f"file://{tmp_path}/sorted-streaming", + ) + table = pa.table({"id": [2, 1]}) + + with pytest.warns(UserWarning, match="streaming RecordBatchReader writes"): + data_files = list(_dataframe_to_data_files(table_metadata=metadata, df=table.to_reader(), io=PyArrowFileIO())) + + assert data_files + assert all(data_file.sort_order_id is None for data_file in data_files) + + def test_bin_pack_record_batches_single_bin(arrow_table_with_null: pa.Table) -> None: batches = arrow_table_with_null.to_batches() bins = list(bin_pack_record_batches(iter(batches), target_file_size=arrow_table_with_null.nbytes * 10))