Gabriel39 commented on code in PR #65784:
URL: https://github.com/apache/doris/pull/65784#discussion_r3610322167
##########
be/src/format_v2/column_mapper.cpp:
##########
@@ -52,11 +52,59 @@
#include "format_v2/schema_projection.h"
#include "format_v2/table_reader.h"
#include "gen_cpp/Exprs_types.h"
+#include "util/url_coding.h"
namespace doris::format {
namespace {
+Status parse_initial_default(const ColumnDefinition& column,
std::optional<Field>* value) {
+ DORIS_CHECK(value != nullptr);
+ value->reset();
+ if (!column.initial_default_value.has_value()) {
+ return Status::OK();
+ }
+ const auto nested_type = remove_nullable(column.type);
+ Field parsed;
+ if (column.initial_default_value_is_base64 ||
+ nested_type->get_primitive_type() == TYPE_VARBINARY) {
+ std::string decoded;
+ if (!base64_decode(*column.initial_default_value, &decoded)) {
+ return Status::InvalidArgument("Invalid Base64 Iceberg initial
default for field {}",
+ column.name);
+ }
+ parsed = nested_type->get_primitive_type() == TYPE_VARBINARY
+ ?
Field::create_field<TYPE_VARBINARY>(StringView(decoded))
Review Comment:
Fixed in 3050fa2a2ca. Decoded VARBINARY values are now materialized into
owning const columns before the temporary decode buffer leaves scope in V1/V2
mapping, Parquet/ORC missing-column helpers, and equality-delete defaults. The
ASAN regression coverage now uses a 16-byte UUID payload.
##########
be/src/format_v2/table_reader.cpp:
##########
@@ -488,6 +490,7 @@ Status TableReader::annotate_projected_column(const
TFileScanSlotInfo& slot_info
context->schema_column =
build_schema_column_from_external_field(*schema_field, column->type);
column->identifier = context->schema_column->identifier;
column->name_mapping = context->schema_column->name_mapping;
+ column->has_name_mapping = context->schema_column->has_name_mapping;
Review Comment:
Fixed in 3050fa2a2ca. Projected roots now retain the Iceberg initial-default
payload and Base64 marker, and that metadata takes precedence over the generic
FE default expression. V1 top-level fill follows the same rule; top-level
binary result tests verify the decoded bytes.
##########
be/src/format_v2/column_mapper.cpp:
##########
@@ -2084,7 +2145,19 @@ const ColumnDefinition*
TableColumnMapper::_find_file_field(
});
return field_it == file_schema.end() ? nullptr : &*field_it;
}
- return matcher_for_mode(_options.mode).find(table_column, file_schema);
+ const auto* matched = matcher_for_mode(_options.mode).find(table_column,
file_schema);
+ if (matched != nullptr || _options.mode !=
TableColumnMappingMode::BY_FIELD_ID ||
+ !_options.allow_idless_complex_wrapper_projection ||
table_column.children.empty()) {
+ return matched;
+ }
+ const auto* wrapper = find_column_by_name(table_column, file_schema);
+ if (wrapper == nullptr || wrapper->has_identifier_field_id() ||
wrapper->children.empty() ||
+ !has_shared_descendant_field_id(table_column, *wrapper)) {
+ return nullptr;
+ }
+ // Iceberg Parquet's PruneColumns retains an ID-less complex wrapper when
a nested field ID is
Review Comment:
Fixed in 3050fa2a2ca. The opt-in Parquet ID-less complex-wrapper fallback is
now applied during recursive child matching, not only at the root. A
reader-result test covers outer#10 -> inner(no id) -> leaf#30 and reads the
selected leaf value.
##########
fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalUtil.java:
##########
@@ -199,7 +213,8 @@ private static TField getExternalSchema(Type columnType,
Column dorisColumn,
TFieldPtr fieldPtr = new TFieldPtr();
Column subColumn = subNameToSubColumn.get(subField.getName());
fieldPtr.setFieldPtr(getExternalSchema(
Review Comment:
Fixed in 3050fa2a2ca. Iceberg schema conversion now recursively copies
serialized initial defaults for every child type, while preserving the binary
transport marker separately. FE serialization tests cover a nested INT default
of 7, and BE V1/V2 nested materialization tests cover the receiving paths.
##########
fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/source/IcebergScanNode.java:
##########
@@ -532,13 +534,14 @@ private List<String> getOrderedPathPartitionKeys() {
public void createScanRangeLocations() throws UserException {
super.createScanRangeLocations();
// Extract name mapping from Iceberg table properties
- Map<Integer, List<String>> nameMapping = extractNameMapping();
+ Optional<Map<Integer, List<String>>> nameMapping =
extractNameMapping();
// Equality-delete keys are hidden scan dependencies and need not
appear in the query
// projection. Both scanners need the complete current schema to
resolve field ids,
// historical names, types, and initial defaults when an old data file
lacks such a key.
ExternalUtil.initSchemaInfoForAllColumn(params, -1L,
source.getTargetTable().getColumns(),
- nameMapping, getBase64EncodedInitialDefaultsForScan());
+ nameMapping.orElse(Collections.emptyMap()),
nameMapping.isPresent(),
+ getBase64EncodedInitialDefaultsForScan());
Review Comment:
Fixed in 3050fa2a2ca. Ordinary scans now obtain default metadata from the
current table schema; only explicit snapshot/ref selections resolve metadata
from the selected snapshot schema. Dedicated tests cover both schema sources.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
