This is an automated email from the ASF dual-hosted git repository.
Gabriel39 pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/master by this push:
new e08bcd1e94c [fix](iceberg) Fix V2 reads across nested schema evolution
(#67574)
e08bcd1e94c is described below
commit e08bcd1e94c6f69342450a0c48b06b1873314111
Author: Gabriel <[email protected]>
AuthorDate: Wed Sep 9 09:25:20 2026 +0800
[fix](iceberg) Fix V2 reads across nested schema evolution (#67574)
### What problem does this PR solve?
Issue Number: N/A
Related PR: N/A
Problem Summary:
Iceberg V2 scans could retain a stale nested file projection after a
predicate on a missing child was demoted, shifting projected sibling
values by physical ordinal. Separately, ID-less migrated files fell back
to current-name mapping after the explicit default name mapping was
removed, exposing stale physical values instead of NULL.
This PR reapplies the finalized scan projection to column mappings and
requires authoritative name mapping for V2 ID-less file reads.
### Release note
Fix Iceberg V2 reads when a rejected nested predicate widens projection
and when migrated ID-less files lose their default name mapping.
### Check List (For Author)
- Test: Unit Test and Regression test
- All 82 `IcebergV2ReaderTest` tests and all 15 `ColumnMapperTest` tests
passed.
- The four equality-delete tests affected by the stricter ID-less
mapping invariant passed after explicitly mapping their projected result
carrier.
- Both new Iceberg external-table regressions passed: nested projection
alignment and migrated ID-less nested NULL semantics.
- The existing `test_gen_iceberg_by_api` suite passed while validating
the required-field error for an ID-less fixture without authoritative
name mapping.
- ASAN BE build and FE build passed.
- Build hygiene and clang-format 16 checks passed.
- Behavior changed: Yes. Iceberg V2 nested projections preserve field-ID
alignment, and migrated ID-less fields without default name mapping
materialize as NULL.
- Does this need documentation: No
---
be/src/format_v2/column_mapper.cpp | 52 ++++
be/src/format_v2/column_mapper.h | 4 +
be/src/format_v2/table/iceberg_reader.cpp | 59 ++++
be/src/format_v2/table/iceberg_reader.h | 8 +
be/src/format_v2/table/iceberg_schema_utils.h | 12 +
be/src/format_v2/table_reader.cpp | 2 +
be/src/format_v2/table_reader.h | 4 +-
be/test/format_v2/column_mapper_test.cpp | 36 +++
be/test/format_v2/table/iceberg_reader_test.cpp | 302 ++++++++++++++++++++-
.../iceberg_load/run01.sql | 10 +-
.../data/data.parquet | Bin 0 -> 1081 bytes
...aafe0-9de1-4b3f-942d-305dc33fb82f.metadata.json | 1 +
...09fba-61c6-4e4b-b0f5-6023f70d5033.metadata.json | 1 +
...3fffe-54f9-486a-b15e-7333fdb96f61.metadata.json | 1 +
...3e871-e421-48ed-a735-354f4e1c9502.metadata.json | 1 +
...544-1-94f693c5-6efb-4e9d-a40e-f02ac301fc14.avro | Bin 0 -> 4667 bytes
...ifest-d04009d1-83e2-4d2a-859e-79d57e8dd381.avro | Bin 0 -> 7501 bytes
.../connector/iceberg/IcebergSchemaUtils.java | 12 +-
.../connector/iceberg/IcebergSchemaUtilsTest.java | 19 +-
.../java/org/apache/doris/qe/SessionVariable.java | 9 +-
.../org/apache/doris/qe/SessionVariablesTest.java | 12 +
.../iceberg/test_gen_iceberg_by_api.out | 3 -
...ceberg_migrated_nested_without_name_mapping.out | 3 +
.../test_iceberg_struct_schema_evolution.out | 3 +
.../iceberg/test_gen_iceberg_by_api.groovy | 11 +-
...erg_migrated_nested_without_name_mapping.groovy | 55 ++++
.../test_iceberg_struct_schema_evolution.groovy | 9 +
27 files changed, 599 insertions(+), 30 deletions(-)
diff --git a/be/src/format_v2/column_mapper.cpp
b/be/src/format_v2/column_mapper.cpp
index 955f858fb23..3ab3c1e9119 100644
--- a/be/src/format_v2/column_mapper.cpp
+++ b/be/src/format_v2/column_mapper.cpp
@@ -2134,6 +2134,20 @@ static const LocalColumnIndex* find_scan_projection(
return projection_it == scan_columns.end() ? nullptr : &*projection_it;
}
+static bool same_projected_file_shape(const std::vector<ColumnDefinition>& lhs,
+ const std::vector<ColumnDefinition>&
rhs) {
+ if (lhs.size() != rhs.size()) {
+ return false;
+ }
+ for (size_t index = 0; index < lhs.size(); ++index) {
+ if (lhs[index].local_id != rhs[index].local_id ||
+ !same_projected_file_shape(lhs[index].children,
rhs[index].children)) {
+ return false;
+ }
+ }
+ return true;
+}
+
// Apply the final scan projection of one root file column back to its
ColumnMapping. This updates
// mapping.file_type/projected_file_children from the original file schema to
the exact shape that
// FileReader will return.
@@ -2577,6 +2591,36 @@ Status TableColumnMapper::create_scan_request(
return Status::OK();
}
+Status TableColumnMapper::reconcile_scan_request_after_customization(
+ FileScanRequest* file_request) {
+ DORIS_CHECK(file_request != nullptr);
+ bool output_shape_changed = false;
+ for (auto& mapping : _mappings) {
+ if (!mapping.file_local_id.has_value() ||
+
!file_request->local_positions.contains(LocalColumnId(*mapping.file_local_id)))
{
+ continue;
+ }
+ const auto previous_file_type = mapping.file_type;
+ const auto previous_file_children = mapping.projected_file_children;
+
RETURN_IF_ERROR(apply_scan_projection_to_mapping_file_type(*file_request,
&mapping));
+ output_shape_changed |=
+ previous_file_type == nullptr || mapping.file_type == nullptr
||
+ !previous_file_type->equals(*mapping.file_type) ||
+ !same_projected_file_shape(previous_file_children,
mapping.projected_file_children);
+ rebuild_projection(&mapping, file_request->non_predicate_position(
+
LocalColumnId(*mapping.file_local_id)));
+ }
+ if (output_shape_changed) {
+ // Localized conjuncts embed nested child ordinals from the pre-hook
projection. Scanner
+ // still evaluates the original table conjuncts, so discard stale
file-local copies rather
+ // than allowing a late equality-delete dependency to reinterpret
another child.
+ file_request->conjuncts.clear();
+ file_request->metadata_pruning_safe_conjunct_count = 0;
+ }
+ RETURN_IF_ERROR(_build_filter_entries(*file_request));
+ return Status::OK();
+}
+
ColumnMapping* TableColumnMapper::_find_mapping(GlobalIndex global_index) {
for (auto& mapping : _mappings) {
if (mapping.global_index == global_index) {
@@ -2811,6 +2855,14 @@ Status TableColumnMapper::localize_filters(const
std::vector<TableFilter>& table
FileScanRequestBuilder builder(file_request);
RETURN_IF_ERROR(builder.add_non_predicate_column(std::move(demoted_projection)));
}
+ // Predicate demotion can widen a nested projection after mappings were
localized. Reapply the
+ // final shape so TableReader interprets the same child ordinals that
FileReader returns.
+ for (auto& mapping : _mappings) {
+ if (mapping.file_local_id.has_value() &&
+
file_request->local_positions.contains(LocalColumnId(*mapping.file_local_id))) {
+
RETURN_IF_ERROR(apply_scan_projection_to_mapping_file_type(*file_request,
&mapping));
+ }
+ }
// Final readers allocate a dense file block, so every retained slot must
follow the same compaction.
compact_file_block_positions(file_request);
RETURN_IF_ERROR(_build_filter_entries(*file_request));
diff --git a/be/src/format_v2/column_mapper.h b/be/src/format_v2/column_mapper.h
index ad216775980..6b6e2d83932 100644
--- a/be/src/format_v2/column_mapper.h
+++ b/be/src/format_v2/column_mapper.h
@@ -218,6 +218,10 @@ public:
const std::map<LocalColumnId, LocalIndex>* fixed_local_positions =
nullptr,
const std::map<LocalColumnId, LocalIndex>*
fixed_non_predicate_positions = nullptr);
+ // Table-format hooks may append hidden physical dependencies after the
initial request is
+ // localized. Reconcile output mappings with that final layout before
opening expressions.
+ Status reconcile_scan_request_after_customization(FileScanRequest*
file_request);
+
// Localize table-level filters to the file schema.
// Trivial mappings can copy structured predicates directly. Type changes
may be localized with
// a safe cast. Expressions that cannot be pushed down safely should be
handled by the
diff --git a/be/src/format_v2/table/iceberg_reader.cpp
b/be/src/format_v2/table/iceberg_reader.cpp
index 3fba91a3a5b..413c5382090 100644
--- a/be/src/format_v2/table/iceberg_reader.cpp
+++ b/be/src/format_v2/table/iceberg_reader.cpp
@@ -53,6 +53,7 @@
#include "format_v2/orc/orc_reader.h"
#include "format_v2/parquet/parquet_reader.h"
#include "format_v2/parquet/reader/column_reader.h"
+#include "format_v2/table/schema_history_util.h"
#include "format_v2/table_reader.h"
#include "io/file_factory.h"
#include "util/debug_points.h"
@@ -66,6 +67,40 @@ static constexpr int32_t ROW_LINEAGE_ROW_ID_FIELD_ID =
2147483540;
namespace {
+bool external_field_has_authoritative_name_mapping(const
schema::external::TField& field) {
+ if (field.__isset.name_mapping_is_authoritative &&
field.name_mapping_is_authoritative) {
+ return true;
+ }
+ if (!field.__isset.nestedField) {
+ return false;
+ }
+ if (field.nestedField.__isset.struct_field &&
field.nestedField.struct_field.__isset.fields) {
+ return std::ranges::any_of(field.nestedField.struct_field.fields,
[](const auto& child) {
+ const auto* child_field = format::get_field_ptr(child);
+ return child_field != nullptr &&
+ external_field_has_authoritative_name_mapping(*child_field);
+ });
+ }
+ if (field.nestedField.__isset.array_field &&
field.nestedField.array_field.__isset.item_field) {
+ const auto* item =
format::get_field_ptr(field.nestedField.array_field.item_field);
+ return item != nullptr &&
external_field_has_authoritative_name_mapping(*item);
+ }
+ if (field.nestedField.__isset.map_field) {
+ const auto& map_field = field.nestedField.map_field;
+ if (map_field.__isset.key_field) {
+ const auto* key = format::get_field_ptr(map_field.key_field);
+ if (key != nullptr &&
external_field_has_authoritative_name_mapping(*key)) {
+ return true;
+ }
+ }
+ if (map_field.__isset.value_field) {
+ const auto* value = format::get_field_ptr(map_field.value_field);
+ return value != nullptr &&
external_field_has_authoritative_name_mapping(*value);
+ }
+ }
+ return false;
+}
+
bool contains_variant_type(const DataTypePtr& input) {
if (input == nullptr) {
return false;
@@ -129,6 +164,30 @@ const char* file_format_name(FileFormat format) {
} // namespace
+bool IcebergTableReader::_scan_has_any_authoritative_name_mapping() const {
+ if (schema_has_any_authoritative_name_mapping(_projected_columns)) {
+ return true;
+ }
+ if (_scan_params == nullptr || !_scan_params->__isset.history_schema_info)
{
+ return false;
+ }
+ // Metadata-only scans have no projected data column carrying aliases.
Consult the complete
+ // schema so hidden equality-delete keys use the same authoritative
mapping as visible fields.
+ for (const auto& schema : _scan_params->history_schema_info) {
+ if (!schema.__isset.root_field || !schema.root_field.__isset.fields) {
+ continue;
+ }
+ if (std::ranges::any_of(schema.root_field.fields, [](const auto&
field) {
+ const auto* schema_field = format::get_field_ptr(field);
+ return schema_field != nullptr &&
+
external_field_has_authoritative_name_mapping(*schema_field);
+ })) {
+ return true;
+ }
+ }
+ return false;
+}
+
Status IcebergTableReader::validate_variant_file_mappings(
FileFormat format, const std::vector<format::ColumnMapping>& mappings)
{
if (format == FileFormat::PARQUET || !std::ranges::any_of(mappings,
mapping_reads_variant)) {
diff --git a/be/src/format_v2/table/iceberg_reader.h
b/be/src/format_v2/table/iceberg_reader.h
index cc4f6058aef..20e96c6ef75 100644
--- a/be/src/format_v2/table/iceberg_reader.h
+++ b/be/src/format_v2/table/iceberg_reader.h
@@ -74,6 +74,12 @@ public:
if (!_data_reader.file_schema.empty() && has_field_ids) {
return format::TableColumnMappingMode::BY_FIELD_ID;
}
+ if (!_data_reader.file_schema.empty() &&
supports_iceberg_scan_semantics_v2(_scan_params) &&
+ !_scan_has_any_authoritative_name_mapping()) {
+ // ID-less migrated files are name-readable only while Iceberg's
explicit default name
+ // mapping exists; current names must not resurrect file fields
after it is removed.
+ return format::TableColumnMappingMode::BY_FIELD_ID;
+ }
return format::TableColumnMappingMode::BY_NAME;
}
@@ -116,6 +122,8 @@ private:
static constexpr size_t ICEBERG_FILE_PATH_BLOCK_POSITION = 0;
static constexpr size_t ICEBERG_ROW_POS_BLOCK_POSITION = 1;
+ bool _scan_has_any_authoritative_name_mapping() const;
+
class PositionDeleteRowsCollector final {
public:
using PositionDeleteFile = std::unordered_map<std::string,
format::DeleteRows>;
diff --git a/be/src/format_v2/table/iceberg_schema_utils.h
b/be/src/format_v2/table/iceberg_schema_utils.h
index 516c982194b..c37b907c018 100644
--- a/be/src/format_v2/table/iceberg_schema_utils.h
+++ b/be/src/format_v2/table/iceberg_schema_utils.h
@@ -50,4 +50,16 @@ inline bool schema_has_all_field_ids(const
std::vector<ColumnDefinition>& schema
return true;
}
+inline bool schema_has_any_authoritative_name_mapping(const
std::vector<ColumnDefinition>& schema) {
+ for (const auto& field : schema) {
+ if (field.column_type != ColumnType::DATA_COLUMN) {
+ continue;
+ }
+ if (field.has_name_mapping ||
schema_has_any_authoritative_name_mapping(field.children)) {
+ return true;
+ }
+ }
+ return false;
+}
+
} // namespace doris::format::iceberg
diff --git a/be/src/format_v2/table_reader.cpp
b/be/src/format_v2/table_reader.cpp
index 7161cf9adb5..76aec676893 100644
--- a/be/src/format_v2/table_reader.cpp
+++ b/be/src/format_v2/table_reader.cpp
@@ -1170,6 +1170,8 @@ Status TableReader::refresh_conjuncts(VExprContextSPtrs
conjuncts) {
}
}
RETURN_IF_ERROR(customize_file_scan_request(refreshed_request.get()));
+ RETURN_IF_ERROR(
+
refreshed_mapper->reconcile_scan_request_after_customization(refreshed_request.get()));
if (_file_scan_request == nullptr ||
!same_physical_scan_layout(*refreshed_request, *_file_scan_request)) {
// A reader cannot reinterpret columns already materialized with
another block layout.
diff --git a/be/src/format_v2/table_reader.h b/be/src/format_v2/table_reader.h
index ccee6f8e406..097cee93734 100644
--- a/be/src/format_v2/table_reader.h
+++ b/be/src/format_v2/table_reader.h
@@ -465,7 +465,6 @@ protected:
RETURN_IF_ERROR(close_current_reader());
return Status::OK();
}
- RETURN_IF_ERROR(validate_file_mapping(*_data_reader.column_mapper));
// COUNT(*) has no semantic column argument, but Nereids retains a
minimum-width scan slot
// so the scan node still has an output tuple. Record only the current
non-predicate file
// columns before table-format hooks add row-position or
equality-delete dependencies. This
@@ -484,6 +483,9 @@ protected:
}
}
RETURN_IF_ERROR(customize_file_scan_request(file_request.get()));
+
RETURN_IF_ERROR(_data_reader.column_mapper->reconcile_scan_request_after_customization(
+ file_request.get()));
+ RETURN_IF_ERROR(validate_file_mapping(*_data_reader.column_mapper));
RETURN_IF_ERROR(_open_local_filter_exprs(*file_request));
_data_reader.file_block_layout.clear();
_data_reader.block_template.clear();
diff --git a/be/test/format_v2/column_mapper_test.cpp
b/be/test/format_v2/column_mapper_test.cpp
index 68b6ba777ff..4ae8ddf605d 100644
--- a/be/test/format_v2/column_mapper_test.cpp
+++ b/be/test/format_v2/column_mapper_test.cpp
@@ -4402,6 +4402,42 @@ TEST(ColumnMapperTest,
PredicateAccessPathsCreateDeferredStructOutputProjection)
EXPECT_TRUE(request.is_predicate_only(LocalColumnId(0)));
}
+TEST(ColumnMapperTest,
RejectedMissingStructPredicateRestoresFullOutputMapping) {
+ auto table_renamed = field_id_col("renamed", 2, i64());
+ auto table_keep = field_id_col("keep", 3, i64());
+ auto table_added = field_id_col("added", 6, i64());
+ auto table_struct = struct_col("s", 1, {table_renamed, table_keep});
+ auto full_table_struct = struct_col("s", 1, {table_renamed, table_keep,
table_added});
+ table_struct.type = full_table_struct.type;
+ table_struct.has_predicate_access_paths = true;
+ table_struct.predicate_children = {table_added};
+
+ auto file_removed = field_id_col("removed", 7, i64(), 0);
+ auto file_renamed = field_id_col("rename_me", 2, i64(), 1);
+ auto file_keep = field_id_col("keep", 3, i64(), 2);
+ auto file_struct = struct_col("s", 1, {file_removed, file_renamed,
file_keep}, 0);
+
+ ParquetColumnMapper mapper({.mode = TableColumnMappingMode::BY_FIELD_ID});
+ ASSERT_TRUE(mapper.create_mapping({table_struct}, {}, {file_struct}).ok());
+
+ auto added = struct_element(table_slot(0, 0, table_struct.type, "s"),
i64(), "added");
+ TableFilter filter {.conjunct =
VExprContext::create_shared(null_predicate(added, true)),
+ .global_indices = {GlobalIndex(0)}};
+
+ FileScanRequest request;
+ ASSERT_TRUE(mapper.create_scan_request({filter}, {table_struct},
&request).ok());
+ EXPECT_TRUE(request.predicate_columns.empty());
+ ASSERT_EQ(request.non_predicate_columns.size(), 1) <<
request.debug_string();
+ EXPECT_TRUE(request.non_predicate_columns[0].project_all_children);
+
+ ASSERT_EQ(mapper.mappings().size(), 1);
+ const auto& mapping = mapper.mappings()[0];
+ ASSERT_EQ(mapping.projected_file_children.size(), 3);
+ EXPECT_EQ(mapping.projected_file_children[0].name, "removed");
+ EXPECT_EQ(mapping.projected_file_children[1].name, "rename_me");
+ EXPECT_EQ(mapping.projected_file_children[2].name, "keep");
+}
+
TEST(ColumnMapperTest,
PredicateAccessPathsCreateDeferredVariantRootProjection) {
auto table_variant = field_id_col("v", 10, variant_v2());
table_variant.has_predicate_access_paths = true;
diff --git a/be/test/format_v2/table/iceberg_reader_test.cpp
b/be/test/format_v2/table/iceberg_reader_test.cpp
index 65a905a9e18..4bbfe6dd992 100644
--- a/be/test/format_v2/table/iceberg_reader_test.cpp
+++ b/be/test/format_v2/table/iceberg_reader_test.cpp
@@ -240,10 +240,12 @@ private:
class IcebergTableReaderMappingModeTestHelper final
: public doris::format::iceberg::IcebergTableReader {
public:
- TableColumnMappingMode
mapping_mode_for_schema(std::vector<ColumnDefinition> file_schema,
- TFileScanRangeParams*
scan_params = nullptr) {
+ TableColumnMappingMode mapping_mode_for_schema(
+ std::vector<ColumnDefinition> file_schema, TFileScanRangeParams*
scan_params = nullptr,
+ std::vector<ColumnDefinition> projected_columns = {}) {
_scan_params = scan_params;
_data_reader.file_schema = std::move(file_schema);
+ _projected_columns = std::move(projected_columns);
return mapping_mode();
}
};
@@ -818,6 +820,69 @@ void write_nested_equality_orc_file(const std::string&
file_path, const std::vec
output.write(memory_stream.getData(),
static_cast<std::streamsize>(memory_stream.getLength()));
}
+void write_nested_key_value_parquet_file(const std::string& file_path,
+ const std::vector<int32_t>& keys,
+ const std::vector<int32_t>& values) {
+ ASSERT_EQ(keys.size(), values.size());
+ auto key_field = arrow::field("delete_key", arrow::int32(), false)
+
->WithMetadata(arrow::key_value_metadata({"PARQUET:field_id"}, {"2"}));
+ auto value_field =
+ arrow::field("visible_value", arrow::int32(), false)
+
->WithMetadata(arrow::key_value_metadata({"PARQUET:field_id"}, {"3"}));
+ auto payload_result = arrow::StructArray::Make(
+ {build_int32_array(keys), build_int32_array(values)}, {key_field,
value_field});
+ ASSERT_TRUE(payload_result.ok()) << payload_result.status();
+ auto payload_field =
+ arrow::field("payload", arrow::struct_({key_field, value_field}),
false)
+
->WithMetadata(arrow::key_value_metadata({"PARQUET:field_id"}, {"1"}));
+ auto table = arrow::Table::Make(arrow::schema({payload_field}),
{*payload_result});
+
+ auto file_result = arrow::io::FileOutputStream::Open(file_path);
+ ASSERT_TRUE(file_result.ok()) << file_result.status();
+ std::shared_ptr<arrow::io::FileOutputStream> out = *file_result;
+ ::parquet::WriterProperties::Builder builder;
+ builder.version(::parquet::ParquetVersion::PARQUET_2_6);
+ builder.data_page_version(::parquet::ParquetDataPageVersion::V2);
+ builder.compression(::parquet::Compression::UNCOMPRESSED);
+ PARQUET_THROW_NOT_OK(::parquet::arrow::WriteTable(*table,
arrow::default_memory_pool(), out,
+
static_cast<int64_t>(keys.size()),
+ builder.build()));
+}
+
+void write_nested_key_value_orc_file(const std::string& file_path, const
std::vector<int64_t>& keys,
+ const std::vector<int64_t>& values) {
+ ASSERT_EQ(keys.size(), values.size());
+ auto type = std::unique_ptr<::orc::Type>(::orc::Type::buildTypeFromString(
+ "struct<payload:struct<delete_key:int,visible_value:int>>"));
+ type->getSubtype(0)->setAttribute("iceberg.id", "1");
+ type->getSubtype(0)->getSubtype(0)->setAttribute("iceberg.id", "2");
+ type->getSubtype(0)->getSubtype(1)->setAttribute("iceberg.id", "3");
+
+ MemoryOutputStream memory_stream(1024 * 1024);
+ ::orc::WriterOptions options;
+ options.setCompression(::orc::CompressionKind_NONE);
+ options.setMemoryPool(::orc::getDefaultPool());
+ auto writer = ::orc::createWriter(*type, &memory_stream, options);
+ auto batch = writer->createRowBatch(keys.size());
+ auto& root_batch = dynamic_cast<::orc::StructVectorBatch&>(*batch);
+ auto& payload_batch =
dynamic_cast<::orc::StructVectorBatch&>(*root_batch.fields[0]);
+ const std::array value_sets = {&keys, &values};
+ for (size_t field_idx = 0; field_idx < value_sets.size(); ++field_idx) {
+ auto& value_batch =
dynamic_cast<::orc::LongVectorBatch&>(*payload_batch.fields[field_idx]);
+ for (size_t row = 0; row < keys.size(); ++row) {
+ value_batch.data[row] = (*value_sets[field_idx])[row];
+ }
+ value_batch.numElements = keys.size();
+ }
+ root_batch.numElements = keys.size();
+ payload_batch.numElements = keys.size();
+ writer->add(*batch);
+ writer->close();
+
+ std::ofstream output(file_path, std::ios::binary);
+ output.write(memory_stream.getData(),
static_cast<std::streamsize>(memory_stream.getLength()));
+}
+
void write_timestamp_int_parquet_file(const std::string& file_path,
const std::vector<int64_t>& timestamps,
const std::vector<int32_t>& ids) {
@@ -1317,6 +1382,16 @@ void
init_iceberg_reader(doris::format::iceberg::IcebergTableReader* reader,
.ok());
}
+ColumnDefinition make_authoritatively_name_mapped_table_column(int32_t id,
std::string name,
+ const
DataTypePtr& type) {
+ auto column = make_table_column(id, name, type);
+ // Equality-delete tests need a readable result carrier while
independently exercising an
+ // ID-less hidden key. Make that carrier explicitly name-readable under
the V2 mapping rules.
+ column.name_mapping = {std::move(name)};
+ column.has_name_mapping = true;
+ return column;
+}
+
void expect_idless_equality_key_uses_delete_file_name(FileFormat file_format,
bool
authoritative_name_mapping) {
const std::string format_name = file_format == FileFormat::PARQUET ?
"parquet" : "orc";
@@ -1339,7 +1414,8 @@ void
expect_idless_equality_key_uses_delete_file_name(FileFormat file_format,
}
std::vector<ColumnDefinition> projected_columns;
- projected_columns.push_back(make_table_column(0, "id",
std::make_shared<DataTypeInt32>()));
+ projected_columns.push_back(make_authoritatively_name_mapped_table_column(
+ 0, "id", std::make_shared<DataTypeInt32>()));
auto equality_field = external_schema_field("future_name", 1, {}, "7");
if (authoritative_name_mapping) {
@@ -2755,6 +2831,25 @@ TEST(IcebergV2ReaderTest,
IcebergLegacyPlanKeepsAllFieldIdsMappingRule) {
TableColumnMappingMode::BY_NAME);
}
+TEST(IcebergV2ReaderTest, IcebergV2IdlessFileRequiresAuthoritativeNameMapping)
{
+ IcebergTableReaderMappingModeTestHelper reader;
+ TFileScanRangeParams scan_params;
+
scan_params.__set_iceberg_scan_semantics_version(ICEBERG_SCAN_SEMANTICS_VERSION_2);
+
+ auto file_column = make_file_column(1, "legacy_name",
std::make_shared<DataTypeInt32>());
+ file_column.identifier = Field {};
+ auto table_column = make_table_column(1, "current_name",
std::make_shared<DataTypeInt32>());
+
+ EXPECT_EQ(reader.mapping_mode_for_schema({file_column}, &scan_params,
{table_column}),
+ TableColumnMappingMode::BY_FIELD_ID);
+
+ table_column.has_name_mapping = true;
+ table_column.name_mapping = {"legacy_name"};
+ EXPECT_EQ(reader.mapping_mode_for_schema({std::move(file_column)},
&scan_params,
+ {std::move(table_column)}),
+ TableColumnMappingMode::BY_NAME);
+}
+
TEST(IcebergV2ReaderTest, VariantFormatGateUsesPhysicalFileMappings) {
ColumnMapping missing_variant;
missing_variant.table_type =
make_nullable(std::make_shared<DataTypeVariantV2>());
@@ -3142,6 +3237,157 @@ TEST(IcebergV2ReaderTest,
IcebergNestedEqualityDeleteFiltersCurrentAndDroppedFie
run_case(FileFormat::PARQUET, false, false);
}
+// Keep the Parquet/ORC setup identical because both readers must preserve the
projected child
+// ordinal when an equality-delete key widens the same struct root after
mapper localization.
+//
NOLINTNEXTLINE(readability-function-cognitive-complexity,readability-function-size)
+TEST(IcebergV2ReaderTest,
IcebergNestedEqualityDeletePreservesVisibleSiblingProjection) {
+ const auto run_case = [](FileFormat file_format) {
+ const bool is_parquet = file_format == FileFormat::PARQUET;
+ const std::string format_name = is_parquet ? "parquet" : "orc";
+ const auto test_dir = std::filesystem::temp_directory_path() /
+ ("doris_v2_nested_equality_visible_sibling_" +
format_name);
+ std::filesystem::remove_all(test_dir);
+ std::filesystem::create_directories(test_dir);
+ const auto file_path = (test_dir / ("split." + format_name)).string();
+ const auto delete_file_path = (test_dir / ("equality-delete." +
format_name)).string();
+ if (is_parquet) {
+ write_nested_key_value_parquet_file(file_path, {100, 200, 300},
{10, 20, 30});
+ write_nested_equality_parquet_file(delete_file_path, {}, {200},
{false}, true,
+ "delete_key", 2);
+ } else {
+ write_nested_key_value_orc_file(file_path, {100, 200, 300}, {10,
20, 30});
+ write_nested_equality_orc_file(delete_file_path, {}, {200},
{false}, "delete_key", 2);
+ }
+
+ auto schema_payload = external_struct_schema_field(
+ "payload", 1,
+ {external_schema_field("delete_key", 2, {}, std::nullopt,
+
external_primitive_type(TPrimitiveType::INT)),
+ external_schema_field("visible_value", 3, {}, std::nullopt,
+
external_primitive_type(TPrimitiveType::INT))});
+ auto scan_params = make_local_scan_params(file_format);
+
scan_params.__set_iceberg_scan_semantics_version(ICEBERG_SCAN_SEMANTICS_VERSION_2);
+ scan_params.__set_current_schema_id(100);
+ scan_params.__set_history_schema_info({external_schema(100,
{schema_payload})});
+
+ const auto int_type = std::make_shared<DataTypeInt32>();
+ auto visible_value = make_table_column(3, "visible_value", int_type);
+ auto payload_type =
+ std::make_shared<DataTypeStruct>(DataTypes {int_type}, Strings
{"visible_value"});
+ auto payload = make_table_column(1, "payload", payload_type);
+ payload.children = {visible_value};
+ std::vector<ColumnDefinition> projected_columns = {payload};
+
+ RuntimeProfile profile("test_profile");
+ RuntimeState state {TQueryOptions(), TQueryGlobals()};
+ io::FileReaderStats file_reader_stats;
+ io::FileCacheStatistics file_cache_stats;
+ auto io_ctx = make_io_context(&file_reader_stats, &file_cache_stats);
+ ShardedKVCache cache(1);
+ doris::format::iceberg::IcebergTableReader reader;
+ init_iceberg_reader(&reader, projected_columns, &scan_params, io_ctx,
&state, &profile,
+ file_format);
+
+ auto split_options = build_split_options(file_path);
+ split_options.cache = &cache;
+ split_options.current_split_format = file_format;
+ const auto thrift_file_format =
+ is_parquet ? TFileFormatType::FORMAT_PARQUET :
TFileFormatType::FORMAT_ORC;
+
split_options.current_range.__set_table_format_params(make_iceberg_table_format_desc(
+ file_path,
+ {make_iceberg_equality_delete_file(delete_file_path, {2},
thrift_file_format)}, 3));
+ ASSERT_TRUE(reader.prepare_split(split_options).ok());
+
+ Block block = build_table_block(projected_columns);
+ bool eos = false;
+ ASSERT_TRUE(reader.get_block(&block, &eos).ok());
+ ASSERT_FALSE(eos);
+ const auto& payload_result =
+ assert_cast<const
ColumnStruct&>(expect_not_null_table_column(block, 0));
+ expect_int32_column_values(payload_result.get_column(0), {10, 30});
+
+ ASSERT_TRUE(reader.close().ok());
+ std::filesystem::remove_all(test_dir);
+ };
+
+ for (const auto file_format : {FileFormat::PARQUET, FileFormat::ORC}) {
+ run_case(file_format);
+ }
+}
+
+// Metadata-only projections cannot carry schema aliases themselves. The
complete Iceberg schema
+// must still select name mapping for hidden equality-delete keys in both
physical readers.
+//
NOLINTNEXTLINE(readability-function-cognitive-complexity,readability-function-size)
+TEST(IcebergV2ReaderTest,
IcebergMetadataOnlyEqualityDeleteUsesHistoryNameMapping) {
+ const auto run_case = [](FileFormat file_format) {
+ const bool is_parquet = file_format == FileFormat::PARQUET;
+ const std::string format_name = is_parquet ? "parquet" : "orc";
+ const auto test_dir = std::filesystem::temp_directory_path() /
+ ("doris_v2_metadata_only_equality_delete_" +
format_name);
+ std::filesystem::remove_all(test_dir);
+ std::filesystem::create_directories(test_dir);
+ const auto file_path = (test_dir / ("split." + format_name)).string();
+ const auto delete_file_path = (test_dir / ("equality-delete." +
format_name)).string();
+ if (is_parquet) {
+ write_single_int_parquet_file(file_path, "legacy_key", {5, 7, 9},
std::nullopt);
+ write_iceberg_equality_delete_parquet_file(delete_file_path, 1, 7,
"legacy_key");
+ } else {
+ write_single_int_orc_file(file_path, "legacy_key", {5, 7, 9},
std::nullopt);
+ write_single_int_orc_file(delete_file_path, "legacy_key", {7}, 1);
+ }
+
+ auto key_field =
+ external_schema_field("current_key", 1, {"legacy_key"},
std::nullopt,
+
external_primitive_type(TPrimitiveType::INT), false, true);
+ key_field.field_ptr->__set_name_mapping_is_authoritative(true);
+ auto scan_params = make_local_scan_params(file_format);
+
scan_params.__set_iceberg_scan_semantics_version(ICEBERG_SCAN_SEMANTICS_VERSION_2);
+ scan_params.__set_current_schema_id(100);
+ scan_params.__set_history_schema_info({external_schema(100,
{key_field})});
+
+ std::vector<ColumnDefinition> projected_columns;
+ projected_columns.push_back(make_synthesized_table_column(
+ BeConsts::ICEBERG_FILE_PATH_COL,
std::make_shared<DataTypeString>()));
+ projected_columns.push_back(make_synthesized_table_column(
+ BeConsts::ICEBERG_ROW_POSITION_COL,
std::make_shared<DataTypeInt64>()));
+
+ RuntimeProfile profile("test_profile");
+ RuntimeState state {TQueryOptions(), TQueryGlobals()};
+ io::FileReaderStats file_reader_stats;
+ io::FileCacheStatistics file_cache_stats;
+ auto io_ctx = make_io_context(&file_reader_stats, &file_cache_stats);
+ ShardedKVCache cache(1);
+ doris::format::iceberg::IcebergTableReader reader;
+ init_iceberg_reader(&reader, projected_columns, &scan_params, io_ctx,
&state, &profile,
+ file_format);
+
+ auto split_options = build_split_options(file_path);
+ split_options.cache = &cache;
+ split_options.current_split_format = file_format;
+ const auto thrift_file_format =
+ is_parquet ? TFileFormatType::FORMAT_PARQUET :
TFileFormatType::FORMAT_ORC;
+
split_options.current_range.__set_table_format_params(make_iceberg_table_format_desc(
+ file_path,
+ {make_iceberg_equality_delete_file(delete_file_path, {1},
thrift_file_format)}, 3));
+ ASSERT_TRUE(reader.prepare_split(split_options).ok());
+
+ Block block = build_table_block(projected_columns);
+ bool eos = false;
+ ASSERT_TRUE(reader.get_block(&block, &eos).ok());
+ ASSERT_FALSE(eos);
+ ASSERT_EQ(block.rows(), 2);
+ expect_string_column_values(*block.get_by_position(0).column,
{file_path, file_path});
+ expect_int64_column_values(*block.get_by_position(1).column, {0, 2});
+
+ ASSERT_TRUE(reader.close().ok());
+ std::filesystem::remove_all(test_dir);
+ };
+
+ for (const auto file_format : {FileFormat::PARQUET, FileFormat::ORC}) {
+ run_case(file_format);
+ }
+}
+
// Keep the shared Parquet/ORC reader setup together so both V2 paths exercise
identical ID-less
// nested-name resolution.
//
NOLINTNEXTLINE(readability-function-cognitive-complexity,readability-function-size)
@@ -3188,7 +3434,8 @@ TEST(IcebergV2ReaderTest,
IcebergIdlessNestedEqualityKeyUsesAliasPathAndDeleteLe
}
std::vector<ColumnDefinition> projected_columns;
- projected_columns.push_back(make_table_column(0, "id",
std::make_shared<DataTypeInt32>()));
+
projected_columns.push_back(make_authoritatively_name_mapped_table_column(
+ 0, "id", std::make_shared<DataTypeInt32>()));
auto scan_params = make_local_scan_params(file_format);
scan_params.__set_iceberg_scan_semantics_version(ICEBERG_SCAN_SEMANTICS_VERSION_2);
scan_params.__set_current_schema_id(100);
@@ -3576,7 +3823,8 @@ TEST(IcebergV2ReaderTest,
IcebergEqualityDeleteMissingKeyDoesNotReadUnsupportedU
write_iceberg_equality_delete_parquet_file(delete_file_path, 1, 7,
"added_column");
std::vector<ColumnDefinition> projected_columns;
- projected_columns.push_back(make_table_column(0, "id",
std::make_shared<DataTypeInt32>()));
+ projected_columns.push_back(make_authoritatively_name_mapped_table_column(
+ 0, "id", std::make_shared<DataTypeInt32>()));
RuntimeProfile profile("test_profile");
RuntimeState state {TQueryOptions(), TQueryGlobals()};
@@ -4106,6 +4354,50 @@ TEST(IcebergV2ReaderTest,
ParquetReadsIdlessWrapperWithAuthoritativeEmptyMapping
std::filesystem::remove_all(test_dir);
}
+TEST(IcebergV2ReaderTest,
ParquetTreatsIdlessNestedFileAsMissingWithoutNameMapping) {
+ const auto test_dir = std::filesystem::temp_directory_path() /
+
"doris_iceberg_idless_nested_without_name_mapping_test";
+ std::filesystem::remove_all(test_dir);
+ std::filesystem::create_directories(test_dir);
+ const auto file_path = (test_dir / "split.parquet").string();
+ write_nested_equality_parquet_file(file_path, {}, {42}, {false}, true,
"leaf", 30, "outer",
+ false, false);
+
+ const auto int_type = std::make_shared<DataTypeInt32>();
+ auto leaf = make_table_column(30, "leaf", int_type);
+ auto outer_type = std::make_shared<DataTypeStruct>(DataTypes {int_type},
Strings {"leaf"});
+ auto outer = make_table_column(10, "outer", outer_type);
+ outer.children = {leaf};
+ std::vector<ColumnDefinition> projected_columns = {outer};
+
+ RuntimeProfile profile("test_profile");
+ RuntimeState state {TQueryOptions(), TQueryGlobals()};
+ auto scan_params = make_local_parquet_scan_params();
+
scan_params.__set_iceberg_scan_semantics_version(ICEBERG_SCAN_SEMANTICS_VERSION_2);
+ io::FileReaderStats file_reader_stats;
+ io::FileCacheStatistics file_cache_stats;
+ auto io_ctx = make_io_context(&file_reader_stats, &file_cache_stats);
+ ShardedKVCache cache(1);
+ doris::format::iceberg::IcebergTableReader reader;
+ init_iceberg_reader(&reader, projected_columns, &scan_params, io_ctx,
&state, &profile);
+ auto split_options = build_split_options(file_path);
+ split_options.cache = &cache;
+ split_options.current_range.__set_table_format_params(
+ make_iceberg_table_format_desc(file_path, {}));
+ ASSERT_TRUE(reader.prepare_split(split_options).ok());
+
+ Block block = build_table_block(projected_columns);
+ bool eos = false;
+ ASSERT_TRUE(reader.get_block(&block, &eos).ok());
+ ASSERT_FALSE(eos);
+ const auto& result = assert_cast<const
ColumnNullable&>(*block.get_by_position(0).column);
+ ASSERT_EQ(result.size(), 1);
+ EXPECT_EQ(result.get_null_map_data()[0], 1);
+
+ ASSERT_TRUE(reader.close().ok());
+ std::filesystem::remove_all(test_dir);
+}
+
TEST(IcebergV2ReaderTest,
ParquetUsesUnprojectedSiblingIdToRetainNullableWrapper) {
const auto test_dir = std::filesystem::temp_directory_path() /
"doris_iceberg_unprojected_sibling_id_wrapper_test";
diff --git
a/docker/thirdparties/docker-compose/iceberg/scripts/create_preinstalled_scripts/iceberg_load/run01.sql
b/docker/thirdparties/docker-compose/iceberg/scripts/create_preinstalled_scripts/iceberg_load/run01.sql
index bfe0a838131..4075719c954 100644
---
a/docker/thirdparties/docker-compose/iceberg/scripts/create_preinstalled_scripts/iceberg_load/run01.sql
+++
b/docker/thirdparties/docker-compose/iceberg/scripts/create_preinstalled_scripts/iceberg_load/run01.sql
@@ -9,6 +9,7 @@ drop table if exists multi_catalog.equality_delete_par_3;
drop table if exists multi_catalog.equality_delete_orc_1;
drop table if exists multi_catalog.equality_delete_orc_2;
drop table if exists multi_catalog.equality_delete_orc_3;
+drop table if exists multi_catalog.migrated_nested_without_name_mapping;
CALL system.register_table(
@@ -42,6 +43,13 @@ CALL system.register_table(
metadata_file =>
's3a://warehouse/wh/multi_catalog/equality_delete_orc_3/metadata/00010-f6ba4ee7-256f-41f3-8932-25ec703d8c8b.metadata.json'
);
+-- The imported Parquet file has no Iceberg field IDs, and the latest table
metadata intentionally
+-- omits schema.name-mapping.default to verify missing-field semantics after
migration.
+CALL system.register_table(
+ table => 'multi_catalog.migrated_nested_without_name_mapping',
+ metadata_file =>
's3a://warehouse/wh/multi_catalog/migrated_nested_without_name_mapping/metadata/00003-97f3e871-e421-48ed-a735-354f4e1c9502.metadata.json'
+);
+
-- flink
-- CREATE CATALOG iceberg_rest
-- WITH (
@@ -199,4 +207,4 @@ CALL system.register_table(
-- ALTER TABLE equality_delete_orc_3 MODIFY (
-- `new_id` INT , `name` string ,`data` STRING ,
-- PRIMARY KEY (new_id, `data`) NOT ENFORCED);
--- insert into equality_delete_orc_3 values(1, 'smith4', 'aaa');
\ No newline at end of file
+-- insert into equality_delete_orc_3 values(1, 'smith4', 'aaa');
diff --git
a/docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/iceberg/migrated_nested_without_name_mapping/data/data.parquet
b/docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/iceberg/migrated_nested_without_name_mapping/data/data.parquet
new file mode 100644
index 00000000000..0311c6f2595
Binary files /dev/null and
b/docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/iceberg/migrated_nested_without_name_mapping/data/data.parquet
differ
diff --git
a/docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/iceberg/migrated_nested_without_name_mapping/metadata/00000-4c8aafe0-9de1-4b3f-942d-305dc33fb82f.metadata.json
b/docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/iceberg/migrated_nested_without_name_mapping/metadata/00000-4c8aafe0-9de1-4b3f-942d-305dc33fb82f.metadata.json
new file mode 100644
index 00000000000..57b95fe8f77
--- /dev/null
+++
b/docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/iceberg/migrated_nested_without_name_mapping/metadata/00000-4c8aafe0-9de1-4b3f-942d-305dc33fb82f.metadata.json
@@ -0,0 +1 @@
+{"format-version":2,"table-uuid":"bd7ffeda-37f2-4199-92e0-1b97e7ddf643","location":"s3a://warehouse/wh/multi_catalog/migrated_nested_without_name_mapping","last-sequence-number":0,"last-updated-ms":1788760308700,"last-column-id":4,"current-schema-id":0,"schemas":[{"type":"struct","schema-id":0,"fields":[{"id":1,"name":"id","required":false,"type":"int"},{"id":2,"name":"nested_struct","required":false,"type":{"type":"struct","fields":[{"id":3,"name":"value_a","required":false,"type":"int"
[...]
\ No newline at end of file
diff --git
a/docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/iceberg/migrated_nested_without_name_mapping/metadata/00001-94d09fba-61c6-4e4b-b0f5-6023f70d5033.metadata.json
b/docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/iceberg/migrated_nested_without_name_mapping/metadata/00001-94d09fba-61c6-4e4b-b0f5-6023f70d5033.metadata.json
new file mode 100644
index 00000000000..9f50c4f5d4e
--- /dev/null
+++
b/docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/iceberg/migrated_nested_without_name_mapping/metadata/00001-94d09fba-61c6-4e4b-b0f5-6023f70d5033.metadata.json
@@ -0,0 +1 @@
+{"format-version":2,"table-uuid":"bd7ffeda-37f2-4199-92e0-1b97e7ddf643","location":"s3a://warehouse/wh/multi_catalog/migrated_nested_without_name_mapping","last-sequence-number":0,"last-updated-ms":1788760308887,"last-column-id":4,"current-schema-id":0,"schemas":[{"type":"struct","schema-id":0,"fields":[{"id":1,"name":"id","required":false,"type":"int"},{"id":2,"name":"nested_struct","required":false,"type":{"type":"struct","fields":[{"id":3,"name":"value_a","required":false,"type":"int"
[...]
\ No newline at end of file
diff --git
a/docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/iceberg/migrated_nested_without_name_mapping/metadata/00002-7493fffe-54f9-486a-b15e-7333fdb96f61.metadata.json
b/docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/iceberg/migrated_nested_without_name_mapping/metadata/00002-7493fffe-54f9-486a-b15e-7333fdb96f61.metadata.json
new file mode 100644
index 00000000000..b3fca919fff
--- /dev/null
+++
b/docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/iceberg/migrated_nested_without_name_mapping/metadata/00002-7493fffe-54f9-486a-b15e-7333fdb96f61.metadata.json
@@ -0,0 +1 @@
+{"format-version":2,"table-uuid":"bd7ffeda-37f2-4199-92e0-1b97e7ddf643","location":"s3a://warehouse/wh/multi_catalog/migrated_nested_without_name_mapping","last-sequence-number":1,"last-updated-ms":1788760313465,"last-column-id":4,"current-schema-id":0,"schemas":[{"type":"struct","schema-id":0,"fields":[{"id":1,"name":"id","required":false,"type":"int"},{"id":2,"name":"nested_struct","required":false,"type":{"type":"struct","fields":[{"id":3,"name":"value_a","required":false,"type":"int"
[...]
\ No newline at end of file
diff --git
a/docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/iceberg/migrated_nested_without_name_mapping/metadata/00003-97f3e871-e421-48ed-a735-354f4e1c9502.metadata.json
b/docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/iceberg/migrated_nested_without_name_mapping/metadata/00003-97f3e871-e421-48ed-a735-354f4e1c9502.metadata.json
new file mode 100644
index 00000000000..5c706e40195
--- /dev/null
+++
b/docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/iceberg/migrated_nested_without_name_mapping/metadata/00003-97f3e871-e421-48ed-a735-354f4e1c9502.metadata.json
@@ -0,0 +1 @@
+{"format-version":2,"table-uuid":"bd7ffeda-37f2-4199-92e0-1b97e7ddf643","location":"s3a://warehouse/wh/multi_catalog/migrated_nested_without_name_mapping","last-sequence-number":1,"last-updated-ms":1788760313716,"last-column-id":4,"current-schema-id":0,"schemas":[{"type":"struct","schema-id":0,"fields":[{"id":1,"name":"id","required":false,"type":"int"},{"id":2,"name":"nested_struct","required":false,"type":{"type":"struct","fields":[{"id":3,"name":"value_a","required":false,"type":"int"
[...]
\ No newline at end of file
diff --git
a/docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/iceberg/migrated_nested_without_name_mapping/metadata/snap-2748466134835934544-1-94f693c5-6efb-4e9d-a40e-f02ac301fc14.avro
b/docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/iceberg/migrated_nested_without_name_mapping/metadata/snap-2748466134835934544-1-94f693c5-6efb-4e9d-a40e-f02ac301fc14.avro
new file mode 100644
index 00000000000..b24d91edf04
Binary files /dev/null and
b/docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/iceberg/migrated_nested_without_name_mapping/metadata/snap-2748466134835934544-1-94f693c5-6efb-4e9d-a40e-f02ac301fc14.avro
differ
diff --git
a/docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/iceberg/migrated_nested_without_name_mapping/metadata/stage-7-task-401-manifest-d04009d1-83e2-4d2a-859e-79d57e8dd381.avro
b/docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/iceberg/migrated_nested_without_name_mapping/metadata/stage-7-task-401-manifest-d04009d1-83e2-4d2a-859e-79d57e8dd381.avro
new file mode 100644
index 00000000000..d087d3f68c0
Binary files /dev/null and
b/docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/iceberg/migrated_nested_without_name_mapping/metadata/stage-7-task-401-manifest-d04009d1-83e2-4d2a-859e-79d57e8dd381.avro
differ
diff --git
a/fe/fe-connector/fe-connector-iceberg/src/main/java/org/apache/doris/connector/iceberg/IcebergSchemaUtils.java
b/fe/fe-connector/fe-connector-iceberg/src/main/java/org/apache/doris/connector/iceberg/IcebergSchemaUtils.java
index 093b8b503d0..364d2d7e443 100644
---
a/fe/fe-connector/fe-connector-iceberg/src/main/java/org/apache/doris/connector/iceberg/IcebergSchemaUtils.java
+++
b/fe/fe-connector/fe-connector-iceberg/src/main/java/org/apache/doris/connector/iceberg/IcebergSchemaUtils.java
@@ -34,6 +34,7 @@ import org.apache.iceberg.Table;
import org.apache.iceberg.TableProperties;
import org.apache.iceberg.mapping.MappedField;
import org.apache.iceberg.mapping.MappedFields;
+import org.apache.iceberg.mapping.MappingUtil;
import org.apache.iceberg.mapping.NameMapping;
import org.apache.iceberg.mapping.NameMappingParser;
import org.apache.iceberg.transforms.Transforms;
@@ -213,8 +214,8 @@ public final class IcebergSchemaUtils {
* name-mapping property, and a present (possibly empty) map when it does
— the distinction #65784 relies on
* to make a table-level mapping AUTHORITATIVE (an unmapped field then
materializes its default/NULL instead
* of silently matching a physical column by its current name; see {@link
#buildField}). Port of legacy
- * {@code IcebergScanNode.extractNameMapping} + {@code
IcebergUtils.getNameMapping} (#65784); fail-soft (a
- * parse error logs + yields {@code Optional.empty()}, so a malformed
property never breaks the scan).
+ * {@code IcebergScanNode.extractNameMapping} + {@code
IcebergUtils.getNameMapping} (#65784). A malformed
+ * property fails soft to a current-name mapping instead of becoming
indistinguishable from no property.
*/
static Optional<Map<Integer, List<String>>> extractNameMapping(Table
table) {
String nameMappingJson =
table.properties().get(TableProperties.DEFAULT_NAME_MAPPING);
@@ -230,9 +231,12 @@ public final class IcebergSchemaUtils {
collectNameMappings(mapping.asMappedFields(), result);
return Optional.of(result);
} catch (Exception e) {
- // If name mapping parsing fails, continue without it (legacy
parity).
+ // Preserve legacy current-name readability for ID-less files when
a malformed table property
+ // cannot provide authoritative aliases; Optional.empty() now
means the property is truly absent.
LOG.warn("Failed to parse name mapping from Iceberg table
properties", e);
- return Optional.empty();
+ Map<Integer, List<String>> fallback = new HashMap<>();
+
collectNameMappings(MappingUtil.create(table.schema()).asMappedFields(),
fallback);
+ return Optional.of(fallback);
}
}
diff --git
a/fe/fe-connector/fe-connector-iceberg/src/test/java/org/apache/doris/connector/iceberg/IcebergSchemaUtilsTest.java
b/fe/fe-connector/fe-connector-iceberg/src/test/java/org/apache/doris/connector/iceberg/IcebergSchemaUtilsTest.java
index a36b1deb65d..aaa597580e1 100644
---
a/fe/fe-connector/fe-connector-iceberg/src/test/java/org/apache/doris/connector/iceberg/IcebergSchemaUtilsTest.java
+++
b/fe/fe-connector/fe-connector-iceberg/src/test/java/org/apache/doris/connector/iceberg/IcebergSchemaUtilsTest.java
@@ -715,14 +715,21 @@ public class IcebergSchemaUtilsTest {
}
@Test
- public void extractNameMappingFailsSoftOnMalformedProperty() {
- // A malformed name-mapping property must not break the scan (legacy
catches + warns). MUTATION: let the
- // parse exception propagate -> the whole scan fails on a benign
metadata quirk -> red.
+ public void malformedNameMappingKeepsIdlessCurrentNameFallback() {
+ // A malformed name-mapping property must not break the scan or look
identical to a genuinely absent
+ // mapping. Required and optional fields both need current-name
aliases for ID-less legacy files.
Table table = createTable("t1", SCHEMA,
Collections.singletonMap(TableProperties.DEFAULT_NAME_MAPPING,
"{not valid json"));
- // A malformed property yields Optional.empty() (absent, not a
present-empty mapping) -> hasNameMapping
- // false -> the scan proceeds on the legacy name fallback instead of
breaking.
-
Assertions.assertFalse(IcebergSchemaUtils.extractNameMapping(table).isPresent());
+
+ Map<Integer, List<String>> fallback =
IcebergSchemaUtils.extractNameMapping(table).orElseThrow();
+ Assertions.assertEquals(Collections.singletonList("id"),
fallback.get(1));
+ Assertions.assertEquals(Collections.singletonList("name"),
fallback.get(2));
+
+ Map<String, TField> fields = topFields(dict(table, "id", "name"));
+ Assertions.assertTrue(fields.get("id").isNameMappingIsAuthoritative());
+ Assertions.assertEquals(Collections.singletonList("id"),
fields.get("id").getNameMapping());
+
Assertions.assertTrue(fields.get("name").isNameMappingIsAuthoritative());
+ Assertions.assertEquals(Collections.singletonList("name"),
fields.get("name").getNameMapping());
}
// --- round-trip through the prop transport (what the generic node does)
---
diff --git a/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java
b/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java
index 69941c097ce..db150390d97 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java
@@ -1169,7 +1169,7 @@ public class SessionVariable implements Serializable,
Writable {
+ "read data of FileScanNode, default 16")
public int maxFileScannersConcurrency = 16;
- @VarAttrDef.VarAttr(name = ENABLE_FILE_SCANNER_V2, needForward = true,
fuzzy = true, description = "When enabled, "
+ @VarAttrDef.VarAttr(name = ENABLE_FILE_SCANNER_V2, needForward = true,
description = "When enabled, "
+ "FileScanNode uses FileScannerV2 for supported query scans.
Enabled by default.")
public boolean enableFileScannerV2 = true;
@@ -3636,10 +3636,9 @@ public class SessionVariable implements Serializable,
Writable {
this.enableLocalExchange = random.nextBoolean();
this.enableSharedExchangeSinkBuffer = random.nextBoolean();
this.useSerialExchange = random.nextBoolean();
- // Randomize the external file scanner engine (FileScannerV2 vs the
legacy V1 path). Kept
- // here rather than in setFuzzyForCatalog() so it also runs in the
external regression
- // pipeline, which enables fuzzy sessions with fuzzy_test_type=p1 (not
"external").
- this.enableFileScannerV2 = random.nextBoolean();
+ // Fuzzy sessions must exercise the production-default V2 path
consistently. Dedicated
+ // compatibility cases can still select the legacy scanner explicitly
after initialization.
+ this.enableFileScannerV2 = true;
this.disableStreamPreaggregations = random.nextBoolean();
this.enableStreamingAggHashJoinForcePassthrough = random.nextBoolean();
this.enableLocalExchangeBeforeAgg = random.nextBoolean();
diff --git
a/fe/fe-core/src/test/java/org/apache/doris/qe/SessionVariablesTest.java
b/fe/fe-core/src/test/java/org/apache/doris/qe/SessionVariablesTest.java
index b7f7b8807c4..5302ec645e2 100644
--- a/fe/fe-core/src/test/java/org/apache/doris/qe/SessionVariablesTest.java
+++ b/fe/fe-core/src/test/java/org/apache/doris/qe/SessionVariablesTest.java
@@ -212,6 +212,18 @@ public class SessionVariablesTest extends
TestWithFeService {
Assertions.assertTrue(varAttr.fuzzy());
}
+ @Test
+ public void testFileScannerV2RemainsEnabledInFuzzyMode() throws Exception {
+ SessionVariable sessionVar = new SessionVariable();
+ sessionVar.enableFileScannerV2 = false;
+ sessionVar.initFuzzyModeVariables();
+ Assertions.assertTrue(sessionVar.enableFileScannerV2);
+
+ Field field =
SessionVariable.class.getDeclaredField("enableFileScannerV2");
+ VarAttrDef.VarAttr varAttr =
field.getAnnotation(VarAttrDef.VarAttr.class);
+ Assertions.assertFalse(varAttr.fuzzy());
+ }
+
@Test
public void testForceEagerAggHintParseWhenSetSessionVariable() throws
Exception {
SessionVariable sessionVar = new SessionVariable();
diff --git
a/regression-test/data/external_table_p0/iceberg/test_gen_iceberg_by_api.out
b/regression-test/data/external_table_p0/iceberg/test_gen_iceberg_by_api.out
index b3d42dfcd19..21218a4e51c 100644
--- a/regression-test/data/external_table_p0/iceberg/test_gen_iceberg_by_api.out
+++ b/regression-test/data/external_table_p0/iceberg/test_gen_iceberg_by_api.out
@@ -5,6 +5,3 @@
2 1970-01-03T09:02:04.000001 c
2 1970-01-03T09:02:04.000001 d
--- !q02 --
-463870
-
diff --git
a/regression-test/data/external_table_p0/iceberg/test_iceberg_migrated_nested_without_name_mapping.out
b/regression-test/data/external_table_p0/iceberg/test_iceberg_migrated_nested_without_name_mapping.out
new file mode 100644
index 00000000000..9aea1c252d0
--- /dev/null
+++
b/regression-test/data/external_table_p0/iceberg/test_iceberg_migrated_nested_without_name_mapping.out
@@ -0,0 +1,3 @@
+-- This file is automatically generated. You should know what you did if you
want to edit this
+-- !idless_nested_fields_are_missing --
+\N \N \N
diff --git
a/regression-test/data/external_table_p0/iceberg/test_iceberg_struct_schema_evolution.out
b/regression-test/data/external_table_p0/iceberg/test_iceberg_struct_schema_evolution.out
index 8cfc10b8204..6925745a8cb 100644
---
a/regression-test/data/external_table_p0/iceberg/test_iceberg_struct_schema_evolution.out
+++
b/regression-test/data/external_table_p0/iceberg/test_iceberg_struct_schema_evolution.out
@@ -39,6 +39,9 @@ a_struct
struct<renamed:bigint,keep:bigint,drop_and_add:bigint,added:bigint> Yes
-- !struct_predicate_4 --
2
+-- !struct_projected_siblings_with_missing_predicate --
+11 12
+
-- !struct_multi --
11 12 \N \N
21 22 23 24
diff --git
a/regression-test/suites/external_table_p0/iceberg/test_gen_iceberg_by_api.groovy
b/regression-test/suites/external_table_p0/iceberg/test_gen_iceberg_by_api.groovy
index 29cfbeae772..bca69ad831a 100644
---
a/regression-test/suites/external_table_p0/iceberg/test_gen_iceberg_by_api.groovy
+++
b/regression-test/suites/external_table_p0/iceberg/test_gen_iceberg_by_api.groovy
@@ -45,10 +45,11 @@ suite("test_gen_iceberg_by_api", "p0,external") {
def q01 = {
qt_q01 """ select * from multi_partition2 order by val """
- try {
- qt_q02 """ select count(*) from table_with_append_file where
MAN_ID is not null """
- } catch (Exception e) {
- assertTrue(e.getMessage().contains("name_mapping must be set when
read missing field id data file."), e.getMessage());
+ test {
+ sql """ select count(*) from table_with_append_file where MAN_ID
is not null """
+ // This fixture has no field IDs or authoritative name mapping, so
its required
+ // columns must be treated as missing instead of being matched by
their current names.
+ exception "Missing required field: MAN_ID"
}
}
@@ -194,4 +195,4 @@ public class CreateTable {
}
-*/
\ No newline at end of file
+*/
diff --git
a/regression-test/suites/external_table_p0/iceberg/test_iceberg_migrated_nested_without_name_mapping.groovy
b/regression-test/suites/external_table_p0/iceberg/test_iceberg_migrated_nested_without_name_mapping.groovy
new file mode 100644
index 00000000000..8f1ceec0906
--- /dev/null
+++
b/regression-test/suites/external_table_p0/iceberg/test_iceberg_migrated_nested_without_name_mapping.groovy
@@ -0,0 +1,55 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+suite("test_iceberg_migrated_nested_without_name_mapping", "p0,external") {
+ String enabled = context.config.otherConfigs.get("enableIcebergTest")
+ if (enabled == null || !enabled.equalsIgnoreCase("true")) {
+ logger.info("disable iceberg test.")
+ return
+ }
+
+ String rest_port = context.config.otherConfigs.get("iceberg_rest_uri_port")
+ String minio_port = context.config.otherConfigs.get("iceberg_minio_port")
+ String externalEnvIp = context.config.otherConfigs.get("externalEnvIp")
+ String catalog_name = "test_iceberg_migrated_nested_without_name_mapping"
+
+ sql """drop catalog if exists ${catalog_name}"""
+ sql """
+ CREATE CATALOG ${catalog_name} PROPERTIES (
+ 'type'='iceberg',
+ 'iceberg.catalog.type'='rest',
+ 'uri' = 'http://${externalEnvIp}:${rest_port}',
+ 's3.access_key' = 'admin',
+ 's3.secret_key' = 'password',
+ 's3.endpoint' = 'http://${externalEnvIp}:${minio_port}',
+ 's3.region' = 'us-east-1'
+ )
+ """
+ sql """switch ${catalog_name}"""
+ sql """use multi_catalog"""
+ sql """set enable_fallback_to_original_planner=false"""
+
+ // The physical Parquet fields predate migration and have no Iceberg IDs.
Once the explicit
+ // default name mapping is removed, neither the root nor nested values may
match by current name.
+ qt_idless_nested_fields_are_missing """
+ SELECT id, nested_struct.value_a, nested_struct.value_b
+ FROM migrated_nested_without_name_mapping
+ ORDER BY id
+ """
+
+ sql """drop catalog if exists ${catalog_name}"""
+}
diff --git
a/regression-test/suites/external_table_p0/iceberg/test_iceberg_struct_schema_evolution.groovy
b/regression-test/suites/external_table_p0/iceberg/test_iceberg_struct_schema_evolution.groovy
index d8a6a427d82..e05ceff5b99 100644
---
a/regression-test/suites/external_table_p0/iceberg/test_iceberg_struct_schema_evolution.groovy
+++
b/regression-test/suites/external_table_p0/iceberg/test_iceberg_struct_schema_evolution.groovy
@@ -89,6 +89,15 @@ suite("test_iceberg_struct_schema_evolution", "p0,external")
{
qt_struct_predicate_3 """SELECT id FROM ${table_name} WHERE
element_at(a_struct, 'added') IS NULL ORDER BY id"""
qt_struct_predicate_4 """SELECT id FROM ${table_name} WHERE
element_at(a_struct, 'added') IS NOT NULL ORDER BY id"""
+ // A missing predicate child widens the physical scan back to the full
struct. The projected
+ // sibling fields must keep their Iceberg field-id mapping instead of
shifting by file ordinal.
+ qt_struct_projected_siblings_with_missing_predicate """
+ SELECT element_at(a_struct, 'renamed'), element_at(a_struct, 'keep')
+ FROM ${table_name}
+ WHERE element_at(a_struct, 'added') IS NULL
+ ORDER BY id
+ """
+
// Test 7: Multiple struct fields in one query
qt_struct_multi """SELECT element_at(a_struct, 'renamed'),
element_at(a_struct, 'keep'), element_at(a_struct, 'drop_and_add'),
element_at(a_struct, 'added') FROM ${table_name} ORDER BY id"""
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
