diff --git a/mkdocs/docs/configuration.md b/mkdocs/docs/configuration.md index c2f3ef5870..d6c1dbad79 100644 --- a/mkdocs/docs/configuration.md +++ b/mkdocs/docs/configuration.md @@ -88,8 +88,9 @@ Iceberg tables support table properties to configure table behavior. | `write.target-file-size-bytes` | Size in bytes | 536870912 (512MB) | Target size of generated data files. Measured in uncompressed in-memory Arrow bytes, so files on disk are typically smaller after compression | | `write.format.default` | `{parquet}` | parquet | File format for data files written by PyIceberg. Only Parquet is supported | | `write.avro.compression-codec` | `{gzip,zstd,snappy}` | gzip | Compression codec for manifest files and manifest lists. For `snappy`, install with `pip install pyiceberg[snappy]` | -| `write.metadata.metrics.default` | `{none,counts,truncate(N),full}` | truncate(16) | Default column metrics for data files: `none` records only column sizes, `counts` adds value and null counts, `full` adds lower/upper bounds, and `truncate(N)` truncates string and binary bounds to N characters or bytes. Nested fields collect at most counts | +| `write.metadata.metrics.default` | `{none,counts,truncate(N),full}` | truncate(16) | Default column metrics for data files: `none` records only column sizes, `counts` adds value and null counts, `full` adds lower/upper bounds, and `truncate(N)` truncates string and binary bounds to N characters or bytes. Applies to nested fields as well | | `write.metadata.metrics.column.{column-name}` | `{none,counts,truncate(N),full}` | (value of `write.metadata.metrics.default`) | Overrides `write.metadata.metrics.default` for a single column. Use the dot-separated name for nested fields | +| `write.metadata.metrics.max-inferred-column-defaults` | Integer | 100 | Maximum number of columns that get `write.metadata.metrics.default` when it is not configured explicitly. Further columns collect no metrics unless they have a `write.metadata.metrics.column` override | | `write.summary.partition-limit` | Integer | 0 | Maximum number of changed partitions for which per-partition summaries are included in the snapshot summary. The default of 0 disables them | | `write.metadata.previous-versions-max` | Integer | 100 | The max number of previous version metadata files to keep before deleting after commit. | | `write.metadata.delete-after-commit.enabled` | Boolean | False | Whether to automatically delete old *tracked* metadata files after each table commit. It will retain a number of the most recent metadata files, which can be set using property `write.metadata.previous-versions-max`. | diff --git a/pyiceberg/io/pyarrow.py b/pyiceberg/io/pyarrow.py index 20333882b5..b62c19a6e2 100644 --- a/pyiceberg/io/pyarrow.py +++ b/pyiceberg/io/pyarrow.py @@ -2565,6 +2565,48 @@ def match_metrics_mode(mode: str) -> MetricsMode: raise ValueError(f"Unsupported metrics mode: {mode}") +def _limit_metrics_field_ids(schema: Schema, limit: int) -> set[int]: + """Return the primitive field ids that get the default metrics mode, in schema order. + + Mirrors Java's MetricsConfig.limitFieldIds: struct fields add their primitive children + before descending into nested fields, and list and map add their primitive element, key, + or value ids before descending. + """ + field_ids: set[int] = set() + + def add(field_id: int) -> None: + if len(field_ids) < limit: + field_ids.add(field_id) + + def visit(iceberg_type: IcebergType) -> None: + if isinstance(iceberg_type, StructType): + for field in iceberg_type.fields: + if isinstance(field.field_type, PrimitiveType): + add(field.field_id) + for field in iceberg_type.fields: + if len(field_ids) >= limit: + return + if not isinstance(field.field_type, PrimitiveType): + visit(field.field_type) + elif isinstance(iceberg_type, ListType): + if isinstance(iceberg_type.element_type, PrimitiveType): + add(iceberg_type.element_id) + else: + visit(iceberg_type.element_type) + elif isinstance(iceberg_type, MapType): + if isinstance(iceberg_type.key_type, PrimitiveType): + add(iceberg_type.key_id) + if isinstance(iceberg_type.value_type, PrimitiveType): + add(iceberg_type.value_id) + if not isinstance(iceberg_type.key_type, PrimitiveType): + visit(iceberg_type.key_type) + if not isinstance(iceberg_type.value_type, PrimitiveType): + visit(iceberg_type.value_type) + + visit(schema.as_struct()) + return field_ids + + @dataclass(frozen=True) class StatisticsCollector: field_id: int @@ -2578,15 +2620,34 @@ class PyArrowStatisticsCollector(PreOrderSchemaVisitor[list[StatisticsCollector] _schema: Schema _properties: dict[str, str] _default_mode: str + _configured_default: bool + _inferred_field_ids: set[int] def __init__(self, schema: Schema, properties: dict[str, str]): from pyiceberg.table import TableProperties self._schema = schema self._properties = properties + self._configured_default = TableProperties.DEFAULT_WRITE_METRICS_MODE in self._properties self._default_mode = self._properties.get( TableProperties.DEFAULT_WRITE_METRICS_MODE, TableProperties.DEFAULT_WRITE_METRICS_MODE_DEFAULT ) + max_inferred = property_as_int( + properties, + TableProperties.METRICS_MAX_INFERRED_COLUMN_DEFAULTS, + TableProperties.METRICS_MAX_INFERRED_COLUMN_DEFAULTS_DEFAULT, + ) + if max_inferred is None: + max_inferred = TableProperties.METRICS_MAX_INFERRED_COLUMN_DEFAULTS_DEFAULT + elif max_inferred < 0: + logger.warning( + "Invalid value for %s (negative): %s, falling back to %s", + TableProperties.METRICS_MAX_INFERRED_COLUMN_DEFAULTS, + max_inferred, + TableProperties.METRICS_MAX_INFERRED_COLUMN_DEFAULTS_DEFAULT, + ) + max_inferred = TableProperties.METRICS_MAX_INFERRED_COLUMN_DEFAULTS_DEFAULT + self._inferred_field_ids = _limit_metrics_field_ids(schema, max_inferred) def schema( self, schema: Schema, struct_result: Callable[[], builtins.list[StatisticsCollector]] @@ -2629,11 +2690,14 @@ def primitive(self, primitive: PrimitiveType) -> builtins.list[StatisticsCollect if column_name is None: return [] - metrics_mode = match_metrics_mode(self._default_mode) - col_mode = self._properties.get(f"{TableProperties.METRICS_MODE_COLUMN_CONF_PREFIX}.{column_name}") if col_mode: metrics_mode = match_metrics_mode(col_mode) + elif not self._configured_default and self._field_id not in self._inferred_field_ids: + # Only the first max-inferred-column-defaults primitive fields get the default mode. + return [] + else: + metrics_mode = match_metrics_mode(self._default_mode) if ( not (isinstance(primitive, StringType) or isinstance(primitive, BinaryType)) @@ -2641,11 +2705,6 @@ def primitive(self, primitive: PrimitiveType) -> builtins.list[StatisticsCollect ): metrics_mode = MetricsMode(MetricModeTypes.FULL) - is_nested = column_name.find(".") >= 0 - - if is_nested and metrics_mode.type in [MetricModeTypes.TRUNCATE, MetricModeTypes.FULL]: - metrics_mode = MetricsMode(MetricModeTypes.COUNTS) - return [StatisticsCollector(field_id=self._field_id, iceberg_type=primitive, mode=metrics_mode, column_name=column_name)] diff --git a/pyiceberg/table/__init__.py b/pyiceberg/table/__init__.py index 3ea41e3f22..b86d6c1603 100644 --- a/pyiceberg/table/__init__.py +++ b/pyiceberg/table/__init__.py @@ -188,6 +188,8 @@ class TableProperties: DEFAULT_WRITE_METRICS_MODE_DEFAULT = "truncate(16)" METRICS_MODE_COLUMN_CONF_PREFIX = "write.metadata.metrics.column" + METRICS_MAX_INFERRED_COLUMN_DEFAULTS = "write.metadata.metrics.max-inferred-column-defaults" + METRICS_MAX_INFERRED_COLUMN_DEFAULTS_DEFAULT = 100 WRITE_PARTITION_SUMMARY_LIMIT = "write.summary.partition-limit" WRITE_PARTITION_SUMMARY_LIMIT_DEFAULT = 0 diff --git a/tests/io/test_pyarrow_stats.py b/tests/io/test_pyarrow_stats.py index 1c5e802a26..62d61db636 100644 --- a/tests/io/test_pyarrow_stats.py +++ b/tests/io/test_pyarrow_stats.py @@ -58,6 +58,7 @@ ) from pyiceberg.manifest import DataFile, FileFormat from pyiceberg.schema import Schema, pre_order_visit +from pyiceberg.table import TableProperties from pyiceberg.table.metadata import ( TableMetadata, TableMetadataUtil, @@ -336,11 +337,11 @@ def test_bounds() -> None: ) datafile = DataFile.from_args(**statistics.to_serialized_dict()) - assert len(datafile.lower_bounds) == 2 + assert len(datafile.lower_bounds) == 7 assert datafile.lower_bounds[1].decode() == "aaaaaaaaaaaaaaaa" assert datafile.lower_bounds[2] == STRUCT_FLOAT.pack(1.69) - assert len(datafile.upper_bounds) == 2 + assert len(datafile.upper_bounds) == 7 assert datafile.upper_bounds[1].decode() == "zzzzzzzzzzzzzzz{" assert datafile.upper_bounds[2] == STRUCT_FLOAT.pack(100) @@ -422,11 +423,11 @@ def test_metrics_mode_full() -> None: assert len(datafile.null_value_counts) == 7 assert len(datafile.nan_value_counts) == 0 - assert len(datafile.lower_bounds) == 2 + assert len(datafile.lower_bounds) == 7 assert datafile.lower_bounds[1].decode() == "aaaaaaaaaaaaaaaaaaaa" assert datafile.lower_bounds[2] == STRUCT_FLOAT.pack(1.69) - assert len(datafile.upper_bounds) == 2 + assert len(datafile.upper_bounds) == 7 assert datafile.upper_bounds[1].decode() == "zzzzzzzzzzzzzzzzzzzz" assert datafile.upper_bounds[2] == STRUCT_FLOAT.pack(100) @@ -447,11 +448,11 @@ def test_metrics_mode_non_default_trunc() -> None: assert len(datafile.null_value_counts) == 7 assert len(datafile.nan_value_counts) == 0 - assert len(datafile.lower_bounds) == 2 + assert len(datafile.lower_bounds) == 7 assert datafile.lower_bounds[1].decode() == "aa" assert datafile.lower_bounds[2] == STRUCT_FLOAT.pack(1.69) - assert len(datafile.upper_bounds) == 2 + assert len(datafile.upper_bounds) == 7 assert datafile.upper_bounds[1].decode() == "z{" assert datafile.upper_bounds[2] == STRUCT_FLOAT.pack(100) @@ -473,15 +474,75 @@ def test_column_metrics_mode() -> None: assert len(datafile.null_value_counts) == 6 assert len(datafile.nan_value_counts) == 0 - assert len(datafile.lower_bounds) == 1 + assert len(datafile.lower_bounds) == 6 assert datafile.lower_bounds[2] == STRUCT_FLOAT.pack(1.69) assert 1 not in datafile.lower_bounds - assert len(datafile.upper_bounds) == 1 + assert len(datafile.upper_bounds) == 6 assert datafile.upper_bounds[2] == STRUCT_FLOAT.pack(100) assert 1 not in datafile.upper_bounds +def test_metrics_max_inferred_column_defaults_limits_stats() -> None: + schema = Schema( + NestedField(1, "a", IntegerType(), required=False), + NestedField(2, "b", IntegerType(), required=False), + NestedField(3, "c", IntegerType(), required=False), + ) + + plan = compute_statistics_plan(schema, {TableProperties.METRICS_MAX_INFERRED_COLUMN_DEFAULTS: "2"}) + + assert set(plan) == {1, 2} + + +def test_metrics_max_inferred_column_defaults_keeps_column_overrides() -> None: + schema = Schema( + NestedField(1, "a", IntegerType(), required=False), + NestedField(2, "b", IntegerType(), required=False), + NestedField(3, "c", IntegerType(), required=False), + ) + + plan = compute_statistics_plan( + schema, + { + TableProperties.METRICS_MAX_INFERRED_COLUMN_DEFAULTS: "1", + f"{TableProperties.METRICS_MODE_COLUMN_CONF_PREFIX}.c": "full", + }, + ) + + assert set(plan) == {1, 3} + + +def test_configured_metrics_default_is_not_limited() -> None: + schema = Schema( + NestedField(1, "a", IntegerType(), required=False), + NestedField(2, "b", IntegerType(), required=False), + NestedField(3, "c", IntegerType(), required=False), + ) + + plan = compute_statistics_plan( + schema, + { + TableProperties.DEFAULT_WRITE_METRICS_MODE: "counts", + TableProperties.METRICS_MAX_INFERRED_COLUMN_DEFAULTS: "1", + }, + ) + + assert set(plan) == {1, 2, 3} + + +def test_negative_metrics_max_inferred_column_defaults_falls_back() -> None: + schema = Schema( + NestedField(1, "a", IntegerType(), required=False), + NestedField(2, "b", IntegerType(), required=False), + NestedField(3, "c", IntegerType(), required=False), + ) + + plan = compute_statistics_plan(schema, {TableProperties.METRICS_MAX_INFERRED_COLUMN_DEFAULTS: "-1"}) + + assert set(plan) == {1, 2, 3} + + def construct_test_table_primitive_types() -> tuple[pq.FileMetaData, TableMetadataV1 | TableMetadataV2]: table_metadata = { "format-version": 2,