eldenmoon commented on code in PR #63192:
URL: https://github.com/apache/doris/pull/63192#discussion_r3233481209
##########
be/src/format/parquet/vparquet_reader.cpp:
##########
@@ -432,11 +435,111 @@ Status
ParquetReader::on_before_init_reader(ReaderInitContext* ctx) {
RETURN_IF_ERROR(get_file_metadata_schema(&field_desc));
RETURN_IF_ERROR(TableSchemaChangeHelper::BuildTableInfoUtil::by_parquet_name(
ctx->tuple_descriptor, *field_desc, ctx->table_info_node));
+ auto column_id_result = _create_column_ids_by_name(field_desc,
ctx->tuple_descriptor);
+ ctx->column_ids = std::move(column_id_result.column_ids);
+ ctx->filter_column_ids = std::move(column_id_result.filter_column_ids);
}
return Status::OK();
}
+ColumnIdResult ParquetReader::_create_column_ids_by_name(const
FieldDescriptor* field_desc,
+ const
TupleDescriptor* tuple_descriptor) {
+ auto* mutable_field_desc = const_cast<FieldDescriptor*>(field_desc);
+ mutable_field_desc->assign_ids();
+
+ std::unordered_map<std::string, const FieldSchema*>
table_col_name_to_field_schema_map;
+ for (int i = 0; i < field_desc->size(); ++i) {
+ auto field_schema = field_desc->get_column(i);
+ if (!field_schema) {
+ continue;
+ }
+ table_col_name_to_field_schema_map[field_schema->lower_case_name] =
field_schema;
+ }
+
+ std::set<uint64_t> column_ids;
+ std::set<uint64_t> filter_column_ids;
+
+ auto process_access_paths = [](const FieldSchema* parquet_field,
+ const std::vector<TColumnAccessPath>&
access_paths,
+ std::set<uint64_t>& out_ids) {
+ process_nested_access_paths(
+ parquet_field, access_paths, out_ids,
+ [](const FieldSchema* field) { return field->get_column_id();
},
+ [](const FieldSchema* field) { return
field->get_max_column_id(); },
+ HiveParquetNestedColumnUtils::extract_nested_column_ids);
+ };
+
+ for (const auto* slot : tuple_descriptor->slots()) {
+ auto it =
table_col_name_to_field_schema_map.find(slot->col_name_lower_case());
+ if (it == table_col_name_to_field_schema_map.end()) {
+ continue;
+ }
+ auto field_schema = it->second;
+
Review Comment:
Fixed in 918c87c5454a. The shared nested access-path helper now falls back
to the full field column-id range when a complex/VARIANT slot has no usable
DATA/META access paths, so selecting id plus full v keeps the VARIANT leaves
instead of pruning them all. This helper is used by local Parquet, Hive Parquet
including the top-level-column-index path, and Iceberg Parquet. Added BE
coverage for empty/ignored access paths and a regression query selecting id
with full root v. Local verification: ./build.sh --be; ./run-be-ut.sh --run
--filter=NestedColumnAccessHelperTest.*:ParquetVariantReaderTest.*;
./run-regression-test.sh --run --conf tmp/regression-conf.auto.groovy -d
external_table_p0/tvf -s test_local_tvf_iceberg_variant;
build-support/check-format.sh; git diff --check.
##########
be/src/format/table/hive_reader.cpp:
##########
@@ -328,7 +328,7 @@ ColumnIdResult HiveParquetReader::_create_column_ids(const
FieldDescriptor* fiel
// primitive (non-nested) types
if ((slot->col_type() != TYPE_STRUCT && slot->col_type() != TYPE_ARRAY
&&
Review Comment:
Fixed in 918c87c5454a. The shared nested access-path helper now falls back
to the full field column-id range when a complex/VARIANT slot has no usable
DATA/META access paths, so selecting id plus full v keeps the VARIANT leaves
instead of pruning them all. This helper is used by local Parquet, Hive Parquet
including the top-level-column-index path, and Iceberg Parquet. Added BE
coverage for empty/ignored access paths and a regression query selecting id
with full root v. Local verification: ./build.sh --be; ./run-be-ut.sh --run
--filter=NestedColumnAccessHelperTest.*:ParquetVariantReaderTest.*;
./run-regression-test.sh --run --conf tmp/regression-conf.auto.groovy -d
external_table_p0/tvf -s test_local_tvf_iceberg_variant;
build-support/check-format.sh; git diff --check.
##########
be/src/format/table/iceberg_reader.cpp:
##########
@@ -344,7 +344,7 @@ ColumnIdResult
IcebergParquetReader::_create_column_ids(const FieldDescriptor* f
auto field_schema = it->second;
if ((slot->col_type() != TYPE_STRUCT && slot->col_type() != TYPE_ARRAY
&&
- slot->col_type() != TYPE_MAP)) {
+ slot->col_type() != TYPE_MAP && slot->col_type() !=
TYPE_VARIANT)) {
Review Comment:
Fixed in 918c87c5454a. The shared nested access-path helper now falls back
to the full field column-id range when a complex/VARIANT slot has no usable
DATA/META access paths, so selecting id plus full v keeps the VARIANT leaves
instead of pruning them all. This helper is used by local Parquet, Hive Parquet
including the top-level-column-index path, and Iceberg Parquet. Added BE
coverage for empty/ignored access paths and a regression query selecting id
with full root v. Local verification: ./build.sh --be; ./run-be-ut.sh --run
--filter=NestedColumnAccessHelperTest.*:ParquetVariantReaderTest.*;
./run-regression-test.sh --run --conf tmp/regression-conf.auto.groovy -d
external_table_p0/tvf -s test_local_tvf_iceberg_variant;
build-support/check-format.sh; git diff --check.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
