mbutrovich commented on code in PR #22026:
URL: https://github.com/apache/datafusion/pull/22026#discussion_r3191253052
##########
datafusion/datasource/src/table_schema.rs:
##########
@@ -149,9 +172,38 @@ impl TableSchema {
);
table_partition_cols.extend(partition_cols);
}
- let mut builder = SchemaBuilder::from(self.file_schema.as_ref());
- builder.extend(self.table_partition_cols.iter().cloned());
- self.table_schema = Arc::new(builder.finish());
+ self.table_schema = build_table_schema(
+ &self.file_schema,
+ self.table_partition_cols.as_ref(),
+ self.virtual_columns.as_ref(),
+ );
+ self
+ }
+
+ /// Add virtual columns to an existing TableSchema, returning a new
instance.
+ ///
+ /// Virtual columns are produced by the file reader (e.g. a Parquet
+ /// `row_number` column) rather than being stored in the files or derived
+ /// from partition paths. Each field must carry an arrow virtual extension
+ /// type so the reader can recognize it; `ParquetOpener` forwards these
+ /// fields to
`parquet::arrow::arrow_reader::ArrowReaderOptions::with_virtual_columns`.
+ ///
+ /// Virtual columns are appended at the end of the table schema, after any
+ /// partition columns.
+ pub fn with_virtual_columns(mut self, virtual_columns: Vec<FieldRef>) ->
Self {
Review Comment:
Sounds good, will do!
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
