Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
82 changes: 68 additions & 14 deletions pyiceberg/io/pyarrow.py
Original file line number Diff line number Diff line change
Expand Up @@ -159,6 +159,7 @@
from pyiceberg.table.metadata import TableMetadata
from pyiceberg.table.metadata_columns import LAST_UPDATED_SEQUENCE_NUMBER, ROW_ID
from pyiceberg.table.name_mapping import NameMapping, apply_name_mapping
from pyiceberg.table.sorting import NullOrder, SortDirection
from pyiceberg.transforms import IdentityTransform, TruncateTransform
from pyiceberg.typedef import EMPTY_DICT, Properties, Record, TableVersion
from pyiceberg.types import (
Expand Down Expand Up @@ -3056,10 +3057,7 @@ def write_data_file(task: WriteTask) -> DataFile:
file_format=file_format,
partition=task.partition_key.partition if task.partition_key else Record(),
file_size_in_bytes=len(fo),
# After this has been fixed:
# https://github.com/apache/iceberg-python/issues/271
# sort_order_id=task.sort_order_id,
sort_order_id=None,
sort_order_id=task.sort_order_id,
# Just copy these from the table for now
spec_id=table_metadata.default_spec_id,
equality_ids=None,
Expand Down Expand Up @@ -3304,6 +3302,11 @@ def _dataframe_to_data_files(
"Materialise the reader as a pa.Table first, or follow "
"https://github.com/apache/iceberg-python/issues/2152 for partitioned streaming support."
)
if not table_metadata.sort_order().is_unsorted:
warnings.warn(
"Sort order is not applied to streaming RecordBatchReader writes; data files are marked unsorted",
stacklevel=2,
)
yield from write_file(
io=io,
table_metadata=table_metadata,
Expand All @@ -3315,31 +3318,82 @@ def _dataframe_to_data_files(
return

if table_metadata.spec().is_unpartitioned():
sorted_df, sort_order_id = _sort_table_for_write(table_metadata, df)
yield from write_file(
io=io,
table_metadata=table_metadata,
tasks=(
WriteTask(write_uuid=write_uuid, task_id=next(counter), record_batches=batches, schema=task_schema)
for batches in bin_pack_arrow_table(df, target_file_size)
WriteTask(
write_uuid=write_uuid,
task_id=next(counter),
record_batches=batches,
schema=task_schema,
sort_order_id=sort_order_id,
)
for batches in bin_pack_arrow_table(sorted_df, target_file_size)
),
)
else:
partitions = _determine_partitions(spec=table_metadata.spec(), schema=table_metadata.schema(), arrow_table=df)
yield from write_file(
io=io,
table_metadata=table_metadata,
tasks=(
tasks: list[WriteTask] = []
for partition in partitions:
sorted_partition, sort_order_id = _sort_table_for_write(table_metadata, partition.arrow_table_partition)
tasks.extend(
WriteTask(
write_uuid=write_uuid,
task_id=next(counter),
record_batches=batches,
partition_key=partition.partition_key,
schema=task_schema,
sort_order_id=sort_order_id,
)
for partition in partitions
for batches in bin_pack_arrow_table(partition.arrow_table_partition, target_file_size)
),
)
for batches in bin_pack_arrow_table(sorted_partition, target_file_size)
)
yield from write_file(io=io, table_metadata=table_metadata, tasks=iter(tasks))


def _sort_table_for_write(table_metadata: TableMetadata, table: pa.Table) -> tuple[pa.Table, int | None]:
"""Sort the table for the table's sort order and return its file sort-order id.

Returns the table unchanged and ``None`` when the sort order cannot be applied, so the
written files are marked unsorted.
"""
sort_order = table_metadata.sort_order()
if sort_order.is_unsorted:
return table, None

schema = table_metadata.schema()
sort_keys: list[tuple[str, str]] = []
null_orders: set[NullOrder] = set()
for sort_field in sort_order.fields:
if not isinstance(sort_field.transform, IdentityTransform):
warnings.warn(
f"Sort transform {sort_field.transform} is not supported for writes; data files are marked unsorted",
stacklevel=2,
)
return table, None
column_name = schema.find_column_name(sort_field.source_id)
if column_name is None or column_name not in table.column_names:
warnings.warn(
f"Sort field with source id {sort_field.source_id} refers to a nested or missing column; "
"data files are marked unsorted",
stacklevel=2,
)
return table, None
sort_keys.append((column_name, "descending" if sort_field.direction == SortDirection.DESC else "ascending"))
null_orders.add(sort_field.null_order)

if len(null_orders) != 1:
warnings.warn("Mixed null ordering is not supported for writes; data files are marked unsorted", stacklevel=2)
return table, None

null_placement = "at_start" if NullOrder.NULLS_FIRST in null_orders else "at_end"
if int(pa.__version__.split(".")[0]) >= 25:
# Per-key null placement replaced the SortOptions-level argument in Arrow 25.
indices = pc.sort_indices(table, sort_keys=cast(Any, [(*key, null_placement) for key in sort_keys]))
else:
indices = pc.sort_indices(table, sort_keys=cast(Any, sort_keys), null_placement=cast(Any, null_placement))
return table.take(indices), sort_order.order_id


@dataclass(frozen=True)
Expand Down
124 changes: 123 additions & 1 deletion tests/io/test_pyarrow.py
Original file line number Diff line number Diff line change
Expand Up @@ -86,10 +86,12 @@
_check_pyarrow_schema_compatible,
_ConvertToArrowSchema,
_create_arrow_read_format,
_dataframe_to_data_files,
_determine_partitions,
_primitive_to_physical,
_read_all_delete_files,
_read_deletes,
_sort_table_for_write,
_task_to_record_batches,
_to_requested_schema,
bin_pack_arrow_table,
Expand All @@ -111,10 +113,11 @@
PROPERTY_REFERENCED_DATA_FILE,
deletion_vectors_from_puffin_file,
)
from pyiceberg.table.metadata import TableMetadataV2, TableMetadataV3
from pyiceberg.table.metadata import TableMetadataV2, TableMetadataV3, new_table_metadata
from pyiceberg.table.metadata_columns import LAST_UPDATED_SEQUENCE_NUMBER, ROW_ID
from pyiceberg.table.name_mapping import create_mapping_from_schema
from pyiceberg.table.puffin import MAGIC_BYTES, PuffinFile
from pyiceberg.table.sorting import NullOrder, SortDirection, SortField, SortOrder
from pyiceberg.transforms import BucketTransform, HourTransform, IdentityTransform, Transform, TruncateTransform
from pyiceberg.typedef import UTF8, Properties, Record, TableVersion
from pyiceberg.types import (
Expand Down Expand Up @@ -3111,6 +3114,125 @@ def test_bin_pack_arrow_table_target_size_smaller_than_row(arrow_table_with_null
assert sum(batch.num_rows for bin_ in bin_packed for batch in bin_) == arrow_table_with_null.num_rows


def test_sort_table_for_write_uses_identity_sort_order() -> None:
schema = Schema(
NestedField(1, "id", LongType(), required=False),
NestedField(2, "value", StringType(), required=False),
)
cases = [
(SortDirection.ASC, NullOrder.NULLS_LAST, [1, 2, None]),
(SortDirection.ASC, NullOrder.NULLS_FIRST, [None, 1, 2]),
(SortDirection.DESC, NullOrder.NULLS_LAST, [2, 1, None]),
(SortDirection.DESC, NullOrder.NULLS_FIRST, [None, 2, 1]),
]

for direction, null_order, expected in cases:
metadata = new_table_metadata(
schema=schema,
partition_spec=PartitionSpec(),
sort_order=SortOrder(SortField(1, IdentityTransform(), direction, null_order)),
location="file:///tmp/sorted",
)
table = pa.table({"id": [2, None, 1], "value": ["b", "null", "a"]})

sorted_table, sort_order_id = _sort_table_for_write(metadata, table)

assert sorted_table["id"].to_pylist() == expected
assert sort_order_id == metadata.default_sort_order_id


def test_sort_table_for_write_leaves_unsupported_sort_orders_unsorted() -> None:
schema = Schema(
NestedField(1, "id", LongType(), required=False),
NestedField(2, "value", StringType(),