Skip to content

Commit c4e68ca

Browse files
committed
fix: restore Exact precision for single-row-group NDV and round-trip write_row_group_number_distinct_values
Two CI fixes: - parquet_writer.rs: read write_row_group_number_distinct_values back from WriterProperties in session_config_from_writer_props instead of using the default - metadata.rs: restore Precision::Exact for single-row-group NDV so the aggregate_statistics optimizer can fold COUNT(DISTINCT) to constants
1 parent f272083 commit c4e68ca

2 files changed

Lines changed: 5 additions & 2 deletions

File tree

‎datafusion/common/src/file_options/parquet_writer.rs‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -633,8 +633,8 @@ mod tests {
633633
schema_force_view_types: global_options_defaults.schema_force_view_types,
634634
binary_as_string: global_options_defaults.binary_as_string,
635635
skip_arrow_metadata: global_options_defaults.skip_arrow_metadata,
636-
write_row_group_number_distinct_values: global_options_defaults
637-
.write_row_group_number_distinct_values,
636+
write_row_group_number_distinct_values: props
637+
.write_row_group_number_distinct_values(),
638638
coerce_int96: None,
639639
coerce_int96_tz: None,
640640
content_defined_chunking: props.content_defined_chunking().into(),

‎datafusion/datasource-parquet/src/metadata.rs‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -966,6 +966,9 @@ fn summarize_distinct_counts(
966966
}
967967

968968
Ok(match max_distinct_count {
969+
Some(distinct_count) if num_row_groups == 1 => {
970+
Precision::Exact(distinct_count as usize)
971+
}
969972
Some(distinct_count) => Precision::Inexact(distinct_count as usize),
970973
None => Precision::Absent,
971974
})

0 commit comments

Comments
 (0)