This is an automated email from the ASF dual-hosted git repository.
morningman pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/master by this push:
new 596383efa1d [fix](be) Parse complex Iceberg initial defaults at full
precision (#69369)
596383efa1d is described below
commit 596383efa1d789a809d902541512a9793a74fb93
Author: Mingyu Chen (Rayner) <[email protected]>
AuthorDate: Fri Oct 23 10:10:46 2026 +0100
[fix](be) Parse complex Iceberg initial defaults at full precision (#69369)
### What problem does this PR solve?
Issue Number: None
Related PR: #66538
Problem Summary:
**In short.** This ports a fix from branch-4.1 #66538 (Iceberg V3
default values): both BE Iceberg readers parse the JSON form of a
complex initial default with `rapidjson::kParseFullPrecisionFlag`, so a
DOUBLE inside it reads back bit-identical to Spark.
**Background.** Iceberg stores the initial default of a struct, list or
map field as single-value JSON, for example `[0.18172760479972437302]`
for a `list<double>` column added after some data files were written.
When Doris reads such an older file, it materializes the missing column
from that JSON: FileScanner V1 in `create_initial_default_column`
(`be/src/format/table/iceberg_default_value.h`) and FileScanner V2 when
it builds the default expression
(`be/src/format_v2/table/iceberg_reader.cpp`). Master added both in
#65851.
**The problem, and what it cost.** Both readers call rapidjson's default
`Parse`. Its fast path rounds a long decimal significand to a double and
then scales it by a power of ten, so a number with a long significand
can be rounded twice and land on the neighbouring double.
| | Master today | After this PR |
|---|---|---|
| `[0.18172760479972437302]` read by V1 / V2 | `0x3fc742d9a3b296db` |
`0x3fc742d9a3b296dc` (correctly rounded, same as Spark and Python) |
| Rows from files written before the column existed | can differ from
Spark in the last bit | identical to Spark |
**How this PR fixes it.** The two `Parse` calls become
`Parse<rapidjson::kParseFullPrecisionFlag>`, as on branch-4.1 and
branch-4.2. Top-level scalar defaults do not go through this JSON parse
and are unchanged.
**Results.** The two unit tests ported from #66538,
`IcebergReaderTest.materializes_complex_double_initial_default_at_full_precision`
(V1) and
`IcebergV2ReaderTest.AnnotateBuildsComplexDoubleDefaultAtFullPrecision`
(V2), check the exact bits of the decoded double. Without the fix both
fail with `0x3fc742d9a3b296db`; with it both pass, and all 144 tests of
the two reader suites pass.
Co-authored-by: daidai <[email protected]>
---
be/src/format/table/iceberg_default_value.h | 3 +-
be/src/format_v2/table/iceberg_reader.cpp | 3 +-
.../format/table/iceberg/iceberg_reader_test.cpp | 39 ++++++++++++++++++++++
be/test/format_v2/table/iceberg_reader_test.cpp | 39 ++++++++++++++++++++++
4 files changed, 82 insertions(+), 2 deletions(-)
diff --git a/be/src/format/table/iceberg_default_value.h
b/be/src/format/table/iceberg_default_value.h
index 5fe1834e3bc..9095ef6b9c5 100644
--- a/be/src/format/table/iceberg_default_value.h
+++ b/be/src/format/table/iceberg_default_value.h
@@ -418,7 +418,8 @@ inline Status build_initial_default_field(const
schema::external::TField& field,
const auto primitive_type = value_type->get_primitive_type();
if (is_complex_type(primitive_type)) {
rapidjson::Document document;
- document.Parse(field.initial_default_value.data(),
field.initial_default_value.size());
+
document.Parse<rapidjson::kParseFullPrecisionFlag>(field.initial_default_value.data(),
+
field.initial_default_value.size());
if (document.HasParseError()) {
return Status::InvalidArgument("Invalid Iceberg JSON initial
default for field '{}'",
field.name);
diff --git a/be/src/format_v2/table/iceberg_reader.cpp
b/be/src/format_v2/table/iceberg_reader.cpp
index b1e1e704457..151744480ee 100644
--- a/be/src/format_v2/table/iceberg_reader.cpp
+++ b/be/src/format_v2/table/iceberg_reader.cpp
@@ -616,7 +616,8 @@ static Status build_v2_initial_default_field(const
format::ColumnDefinition& fie
}
if (is_complex_type(primitive_type)) {
rapidjson::Document document;
- document.Parse(field.initial_default_value->data(),
field.initial_default_value->size());
+
document.Parse<rapidjson::kParseFullPrecisionFlag>(field.initial_default_value->data(),
+
field.initial_default_value->size());
if (document.HasParseError()) {
return Status::InvalidArgument("Invalid Iceberg JSON initial
default for field '{}'",
field.name);
diff --git a/be/test/format/table/iceberg/iceberg_reader_test.cpp
b/be/test/format/table/iceberg/iceberg_reader_test.cpp
index 904f6f88589..528ed13f0f4 100644
--- a/be/test/format/table/iceberg/iceberg_reader_test.cpp
+++ b/be/test/format/table/iceberg/iceberg_reader_test.cpp
@@ -28,6 +28,7 @@
#include <parquet/arrow/writer.h>
#include <array>
+#include <bit>
#include <cmath>
#include <filesystem>
#include <fstream>
@@ -1403,6 +1404,44 @@ protected:
{"pos", 1}};
};
+TEST_F(IcebergReaderTest,
materializes_complex_double_initial_default_at_full_precision) {
+ auto element = std::make_shared<schema::external::TField>();
+ element->__set_name("element");
+ element->__set_id(2);
+ element->__set_is_optional(false);
+ TColumnType element_thrift_type;
+ element_thrift_type.__set_type(TPrimitiveType::DOUBLE);
+ element->__set_type(element_thrift_type);
+ schema::external::TFieldPtr element_ptr;
+ element_ptr.__set_field_ptr(std::move(element));
+
+ schema::external::TField values;
+ values.__set_name("values");
+ values.__set_id(1);
+ values.__set_is_optional(true);
+ values.__set_initial_default_value("[0.18172760479972437302]");
+ TColumnType values_thrift_type;
+ values_thrift_type.__set_type(TPrimitiveType::ARRAY);
+ values.__set_type(values_thrift_type);
+ schema::external::TArrayField array_metadata;
+ array_metadata.__set_item_field(std::move(element_ptr));
+ values.nestedField.__set_array_field(std::move(array_metadata));
+ values.__isset.nestedField = true;
+
+ const auto double_type = std::make_shared<DataTypeFloat64>();
+ const auto values_type =
make_nullable(std::make_shared<DataTypeArray>(double_type));
+ ColumnPtr default_column;
+ ASSERT_TRUE(iceberg::create_initial_default_column(values, values_type,
&default_column).ok());
+ const auto& nullable = assert_cast<const ColumnNullable&>(*default_column);
+ ASSERT_FALSE(nullable.is_null_at(0));
+ const auto& array = assert_cast<const
ColumnArray&>(nullable.get_nested_column());
+ const auto& nullable_doubles = assert_cast<const
ColumnNullable&>(array.get_data());
+ ASSERT_FALSE(nullable_doubles.is_null_at(0));
+ const auto& doubles = assert_cast<const
ColumnFloat64&>(nullable_doubles.get_nested_column());
+ ASSERT_EQ(doubles.size(), 1);
+ EXPECT_EQ(std::bit_cast<uint64_t>(doubles.get_data()[0]),
0x3fc742d9a3b296dcULL);
+}
+
TEST_F(IcebergReaderTest, detects_fully_dictionary_encoded_parquet_column) {
tparquet::ColumnMetaData column_metadata;
column_metadata.type = tparquet::Type::BYTE_ARRAY;
diff --git a/be/test/format_v2/table/iceberg_reader_test.cpp
b/be/test/format_v2/table/iceberg_reader_test.cpp
index 9a7af2a574d..adcecd1c3ad 100644
--- a/be/test/format_v2/table/iceberg_reader_test.cpp
+++ b/be/test/format_v2/table/iceberg_reader_test.cpp
@@ -26,6 +26,7 @@
#include <algorithm>
#include <array>
+#include <bit>
#include <chrono>
#include <cmath>
#include <cstring>
@@ -2130,6 +2131,44 @@ TEST(IcebergV2ReaderTest,
AnnotateBuildsComplexInitialDefaults) {
EXPECT_EQ(values[0].get<TYPE_INT>(), 9);
}
+TEST(IcebergV2ReaderTest, AnnotateBuildsComplexDoubleDefaultAtFullPrecision) {
+ const auto double_type = std::make_shared<DataTypeFloat64>();
+ const auto list_type =
make_nullable(std::make_shared<DataTypeArray>(double_type));
+ auto list_element =
+ external_schema_field("element", 2, {}, std::nullopt,
+
external_primitive_type(TPrimitiveType::DOUBLE), false, false);
+ auto list_field = external_schema_field("values", 1, {},
"[0.18172760479972437302]",
+ std::nullopt, false, true);
+ schema::external::TArrayField array_metadata;
+ array_metadata.__set_item_field(std::move(list_element));
+
list_field.field_ptr->nestedField.__set_array_field(std::move(array_metadata));
+ list_field.field_ptr->__isset.nestedField = true;
+ TFileScanRangeParams scan_params;
+
scan_params.__set_iceberg_scan_semantics_version(ICEBERG_SCAN_SEMANTICS_VERSION_2);
+ scan_params.__set_current_schema_id(100);
+ scan_params.__set_history_schema_info({external_schema(100,
{std::move(list_field)})});
+
+ ColumnDefinition column;
+ column.name = "values";
+ column.type = list_type;
+ ProjectedColumnBuildContext context {.scan_params = &scan_params};
+ doris::format::iceberg::IcebergTableReader reader;
+ const auto status = reader.annotate_projected_column(TFileScanSlotInfo(),
&context, &column);
+ ASSERT_TRUE(status.ok()) << status;
+ ASSERT_NE(column.default_expr, nullptr);
+ const auto* literal = dynamic_cast<const
VLiteral*>(column.default_expr->root().get());
+ ASSERT_NE(literal, nullptr);
+ const auto literal_column =
literal->get_column_ptr()->convert_to_full_column_if_const();
+ const auto& nullable = assert_cast<const ColumnNullable&>(*literal_column);
+ ASSERT_FALSE(nullable.is_null_at(0));
+ const auto& array = assert_cast<const
ColumnArray&>(nullable.get_nested_column());
+ const auto& nullable_doubles = assert_cast<const
ColumnNullable&>(array.get_data());
+ ASSERT_FALSE(nullable_doubles.is_null_at(0));
+ const auto& doubles = assert_cast<const
ColumnFloat64&>(nullable_doubles.get_nested_column());
+ ASSERT_EQ(doubles.size(), 1);
+ EXPECT_EQ(std::bit_cast<uint64_t>(doubles.get_data()[0]),
0x3fc742d9a3b296dcULL);
+}
+
TEST(IcebergV2ReaderTest, ComplexInitialDefaultPrefersExactChildNameOverAlias)
{
const auto int_type = std::make_shared<DataTypeInt32>();
const auto struct_type =
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
