kosiew commented on code in PR #20042:
URL: https://github.com/apache/datafusion/pull/20042#discussion_r2735827108
##########
datafusion/core/src/datasource/physical_plan/parquet.rs:
##########
@@ -2263,6 +2263,40 @@ mod tests {
Ok(())
}
+ /// Tests that constant dictionary columns (where min == max in statistics)
+ /// are correctly handled. This reproduced a bug where the constant value
+ /// from statistics had type Utf8 but the schema expected Dictionary.
+ #[tokio::test]
+ async fn test_constant_dictionary_column_parquet() -> Result<()> {
+ let tmp_dir = TempDir::new()?;
+ let path = tmp_dir.path().to_str().unwrap().to_string() +
"/test.parquet";
+
+ // Write parquet with dictionary column where all values are the same
+ let schema = Arc::new(Schema::new(vec![Field::new(
+ "status",
+ DataType::Dictionary(Box::new(DataType::UInt16),
Box::new(DataType::Utf8)),
+ false,
+ )]));
+ let status: DictionaryArray<UInt16Type> =
+ vec!["active", "active"].into_iter().collect();
+ let batch = RecordBatch::try_new(schema.clone(),
vec![Arc::new(status)])?;
+ let file = File::create(&path)?;
+ let props = WriterProperties::builder()
+
.set_statistics_enabled(parquet::file::properties::EnabledStatistics::Page)
+ .build();
+ let mut writer = ArrowWriter::try_new(file, schema, Some(props))?;
+ writer.write(&batch)?;
+ writer.close()?;
+
+ // Query the constant dictionary column
+ let ctx = SessionContext::new();
+ ctx.register_parquet("t", &path, ParquetReadOptions::default())
+ .await?;
+ let result = ctx.sql("SELECT status FROM t").await?.collect().await?;
+ assert_eq!(result[0].num_rows(), 2);
Review Comment:
It'll be good to validate the values match expectations too to ensure the
dictionary encoding is preserved through the cast.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
