This is an automated email from the ASF dual-hosted git repository.
yiguolei pushed a commit to branch branch-4.1
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/branch-4.1 by this push:
new 20dab94092a branch-4.1:[feature](iceberg) Enforce NULL defaults for
VARIANT (#67323)
20dab94092a is described below
commit 20dab94092a91dde79b454db010f55ca5c031131
Author: daidai <[email protected]>
AuthorDate: Sun Sep 6 23:48:17 2026 +0800
branch-4.1:[feature](iceberg) Enforce NULL defaults for VARIANT (#67323)
### What problem does this PR solve?
Related PR: #66538
Problem Summary:
Iceberg V3 requires both `initial-default` and `write-default` of a
VARIANT field to be NULL. After adding general Iceberg V3 default-value
support, Doris still needs explicit VARIANT handling to avoid accepting
unsupported non-NULL defaults or reconstructing a compute-V2 VARIANT as
the legacy type.
This PR:
- Rejects non-NULL VARIANT `initial-default` and `write-default` values
in FE.
- Adds the same validation to the BE read path for malformed or
externally forged Iceberg metadata.
- Preserves the compute-V2 VARIANT type when reconstructing the current
schema from Thrift metadata.
- Supports typed SQL NULL defaults for optional VARIANT fields,
including VARIANT fields nested inside structs.
- Adds FE and BE unit coverage.
- Adds an external Iceberg regression covering historical Parquet/ORC
rows, INSERT, UPDATE, MERGE, explicit `DEFAULT`, SQL NULL versus JSON
null, invalid DDL, and Spark interoperability.
### Release note
None
### Check List (For Author)
- Test <!-- At least one of them must be included. -->
- [ ] Regression test
- [ ] Unit Test
- [ ] Manual test (add detailed scripts or steps below)
- [ ] No need to test or manual test. Explain why:
- [ ] This is a refactor/code format and no logic has been changed.
- [ ] Previous test can cover this change.
- [ ] No code files have been changed.
- [ ] Other reason <!-- Add your reason? -->
- Behavior changed:
- [ ] No.
- [ ] Yes. <!-- Explain the behavior change -->
- Does this need documentation?
- [ ] No.
- [ ] Yes. <!-- Add document PR link here. eg:
https://github.com/apache/doris-website/pull/1214 -->
### Check List (For Reviewer who merge this PR)
- [ ] Confirm the release note
- [ ] Confirm test cases
- [ ] Confirm document
- [ ] Add branch pick label <!-- Add branch pick label that this PR
should merge into -->
---
be/src/format_v2/table/iceberg_reader.cpp | 8 ++
be/src/format_v2/table_reader.cpp | 5 +
be/test/format_v2/table/iceberg_reader_test.cpp | 103 +++++++++++++
.../doris/datasource/iceberg/IcebergUtils.java | 5 +
.../iceberg/IcebergWriteSchemaContext.java | 2 +
.../doris/datasource/iceberg/IcebergUtilsTest.java | 21 +++
.../iceberg/IcebergWriteSchemaContextTest.java | 42 ++++++
.../iceberg/test_iceberg_variant_defaults.out | 22 +++
.../iceberg/test_iceberg_variant_defaults.groovy | 160 +++++++++++++++++++++
9 files changed, 368 insertions(+)
diff --git a/be/src/format_v2/table/iceberg_reader.cpp
b/be/src/format_v2/table/iceberg_reader.cpp
index 24c2a5862ad..078b413a059 100644
--- a/be/src/format_v2/table/iceberg_reader.cpp
+++ b/be/src/format_v2/table/iceberg_reader.cpp
@@ -311,6 +311,10 @@ static Status build_v2_json_scalar_default(const
format::ColumnDefinition& field
std::deque<std::string>*
binary_storage,
const cctz::time_zone* timezone,
Field* result) {
const auto primitive_type = value_type->get_primitive_type();
+ if (primitive_type == TYPE_VARIANT) {
+ return Status::InvalidArgument(
+ "Iceberg VARIANT initial-default for field '{}' must be NULL",
field.name);
+ }
std::string serialized_value = iceberg_json_scalar_text(json_value);
const bool binary_like =
field.initial_default_value_is_base64 || primitive_type ==
TYPE_VARBINARY;
@@ -395,6 +399,10 @@ static Status build_v2_initial_default_field(const
format::ColumnDefinition& fie
const auto value_type = remove_nullable(data_type);
const auto primitive_type = value_type->get_primitive_type();
+ if (primitive_type == TYPE_VARIANT) {
+ return Status::InvalidArgument(
+ "Iceberg VARIANT initial-default for field '{}' must be NULL",
field.name);
+ }
if (is_complex_type(primitive_type)) {
rapidjson::Document document;
document.Parse<rapidjson::kParseFullPrecisionFlag>(field.initial_default_value->data(),
diff --git a/be/src/format_v2/table_reader.cpp
b/be/src/format_v2/table_reader.cpp
index 12756e7c319..78089087c76 100644
--- a/be/src/format_v2/table_reader.cpp
+++ b/be/src/format_v2/table_reader.cpp
@@ -511,6 +511,11 @@ DataTypePtr restore_current_primitive_type(const
schema::external::TField& field
}
DORIS_CHECK(fallback_type != nullptr);
const auto primitive_type = thrift_to_type(field.type.type);
+ if (primitive_type == TYPE_VARIANT) {
+ // TColumnType predates the execution-only variant_is_v2 marker.
Reconstructing from its
+ // primitive enum would silently replace an Iceberg compute-V2 carrier
with legacy VARIANT.
+ return fallback_type;
+ }
if (is_complex_type(primitive_type)) {
return fallback_type;
}
diff --git a/be/test/format_v2/table/iceberg_reader_test.cpp
b/be/test/format_v2/table/iceberg_reader_test.cpp
index cc2b2096e14..b66655832d7 100644
--- a/be/test/format_v2/table/iceberg_reader_test.cpp
+++ b/be/test/format_v2/table/iceberg_reader_test.cpp
@@ -60,6 +60,7 @@
#include "core/data_type/data_type_struct.h"
#include "core/data_type/data_type_timestamptz.h"
#include "core/data_type/data_type_varbinary.h"
+#include "core/data_type/data_type_variant_v2.h"
#include "exec/common/endian.h"
#include "exec/scan/access_path_parser.h"
#include "exprs/runtime_filter_expr.h"
@@ -1505,6 +1506,108 @@ TEST(IcebergV2ReaderTest,
AnnotateConvertsTimestamptzDefaultToSessionTimezone) {
"2025-01-18 09:02:03.654321");
}
+TEST(IcebergV2ReaderTest, AnnotateVariantInitialDefaultMustBeNull) {
+ const auto variant_type =
make_nullable(std::make_shared<DataTypeVariantV2>());
+
+ auto optional_variant =
+ external_schema_field("payload", 1, {}, std::nullopt,
+
external_primitive_type(TPrimitiveType::VARIANT), false, true);
+ TFileScanRangeParams valid_scan_params;
+
valid_scan_params.__set_iceberg_scan_semantics_version(ICEBERG_SCAN_SEMANTICS_VERSION_2);
+ valid_scan_params.__set_current_schema_id(100);
+ valid_scan_params.__set_history_schema_info(
+ {external_schema(100, {std::move(optional_variant)})});
+
+ ColumnDefinition valid_column;
+ valid_column.name = "payload";
+ valid_column.type = variant_type;
+ ProjectedColumnBuildContext valid_context {.scan_params =
&valid_scan_params};
+ doris::format::iceberg::IcebergTableReader reader;
+ ASSERT_TRUE(reader.annotate_projected_column(TFileScanSlotInfo(),
&valid_context, &valid_column)
+ .ok());
+ ASSERT_TRUE(valid_column.is_optional.has_value());
+ EXPECT_TRUE(*valid_column.is_optional);
+ EXPECT_FALSE(valid_column.initial_default_value.has_value());
+ EXPECT_EQ(valid_column.default_expr, nullptr);
+ ASSERT_TRUE(valid_context.schema_column.has_value());
+ EXPECT_TRUE(valid_context.schema_column->type->equals(*variant_type));
+
+ auto malformed_variant =
+ external_schema_field("payload", 1, {},
"{\"source\":\"malformed\"}",
+
external_primitive_type(TPrimitiveType::VARIANT), false, true);
+ TFileScanRangeParams malformed_scan_params;
+
malformed_scan_params.__set_iceberg_scan_semantics_version(ICEBERG_SCAN_SEMANTICS_VERSION_2);
+ malformed_scan_params.__set_current_schema_id(101);
+ malformed_scan_params.__set_history_schema_info(
+ {external_schema(101, {std::move(malformed_variant)})});
+
+ ColumnDefinition malformed_column;
+ malformed_column.name = "payload";
+ malformed_column.type = variant_type;
+ ProjectedColumnBuildContext malformed_context {.scan_params =
&malformed_scan_params};
+ const auto status = reader.annotate_projected_column(TFileScanSlotInfo(),
&malformed_context,
+ &malformed_column);
+ EXPECT_TRUE(status.is<ErrorCode::INVALID_ARGUMENT>()) << status;
+ EXPECT_NE(status.to_string().find("VARIANT initial-default for field
'payload' must be NULL"),
+ std::string::npos)
+ << status;
+}
+
+TEST(IcebergV2ReaderTest, StructInitialDefaultKeepsVariantChildNull) {
+ const auto variant_type =
make_nullable(std::make_shared<DataTypeVariantV2>());
+ const auto struct_type = make_nullable(
+ std::make_shared<DataTypeStruct>(DataTypes {variant_type}, Strings
{"payload"}));
+
+ auto optional_variant =
+ external_schema_field("payload", 2, {}, std::nullopt,
+
external_primitive_type(TPrimitiveType::VARIANT), false, true);
+ auto valid_struct =
+ external_struct_schema_field("event", 1,
{std::move(optional_variant)}, true, "{}");
+ TFileScanRangeParams valid_scan_params;
+
valid_scan_params.__set_iceberg_scan_semantics_version(ICEBERG_SCAN_SEMANTICS_VERSION_2);
+ valid_scan_params.__set_current_schema_id(100);
+ valid_scan_params.__set_history_schema_info({external_schema(100,
{std::move(valid_struct)})});
+
+ ColumnDefinition valid_column;
+ valid_column.name = "event";
+ valid_column.type = struct_type;
+ ProjectedColumnBuildContext valid_context {.scan_params =
&valid_scan_params};
+ doris::format::iceberg::IcebergTableReader reader;
+ ASSERT_TRUE(reader.annotate_projected_column(TFileScanSlotInfo(),
&valid_context, &valid_column)
+ .ok());
+ ASSERT_NE(valid_column.default_expr, nullptr);
+ const auto* literal = dynamic_cast<const
VLiteral*>(valid_column.default_expr->root().get());
+ ASSERT_NE(literal, nullptr);
+ EXPECT_TRUE(literal->get_data_type()->equals(*struct_type));
+ Field value;
+ literal->get_column_ptr()->get(0, value);
+ const auto& struct_fields = value.get<TYPE_STRUCT>();
+ ASSERT_EQ(struct_fields.size(), 1);
+ EXPECT_TRUE(struct_fields[0].is_null());
+
+ auto malformed_variant =
+ external_schema_field("payload", 2, {}, std::nullopt,
+
external_primitive_type(TPrimitiveType::VARIANT), false, true);
+ auto malformed_struct = external_struct_schema_field(
+ "event", 1, {std::move(malformed_variant)}, true,
"{\"2\":{\"source\":\"malformed\"}}");
+ TFileScanRangeParams malformed_scan_params;
+
malformed_scan_params.__set_iceberg_scan_semantics_version(ICEBERG_SCAN_SEMANTICS_VERSION_2);
+ malformed_scan_params.__set_current_schema_id(101);
+ malformed_scan_params.__set_history_schema_info(
+ {external_schema(101, {std::move(malformed_struct)})});
+
+ ColumnDefinition malformed_column;
+ malformed_column.name = "event";
+ malformed_column.type = struct_type;
+ ProjectedColumnBuildContext malformed_context {.scan_params =
&malformed_scan_params};
+ const auto status = reader.annotate_projected_column(TFileScanSlotInfo(),
&malformed_context,
+ &malformed_column);
+ EXPECT_TRUE(status.is<ErrorCode::INVALID_ARGUMENT>()) << status;
+ EXPECT_NE(status.to_string().find("VARIANT initial-default for field
'payload' must be NULL"),
+ std::string::npos)
+ << status;
+}
+
TEST(IcebergV2ReaderTest, AnnotateBuildsComplexInitialDefaults) {
const auto required_int_type = std::make_shared<DataTypeInt32>();
const auto optional_string_type =
make_nullable(std::make_shared<DataTypeString>());
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergUtils.java
b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergUtils.java
index 7c56314c569..c60f64c6f9c 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergUtils.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergUtils.java
@@ -1224,6 +1224,8 @@ public class IcebergUtils {
} catch (NumberFormatException e) {
throw new IllegalArgumentException("Invalid Decimal
string: " + value, e);
}
+ case VARIANT:
+ throw new IllegalArgumentException("Iceberg VARIANT default
values must be NULL");
default:
throw new IllegalArgumentException("Cannot parse unknown type:
" + type);
}
@@ -1427,6 +1429,9 @@ public class IcebergUtils {
private static String
serializeInitialDefault(org.apache.iceberg.types.Type type, Object value,
boolean enableMappingTimestampTz) {
+ if (type.typeId() == TypeID.VARIANT) {
+ throw new IllegalArgumentException("Iceberg VARIANT
initial-default must be NULL");
+ }
if (type.isNestedType()) {
// Keep Iceberg's type-directed JSON representation for
struct/list/map values. In
// particular, struct members are keyed by field id and an empty
object is the V3
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergWriteSchemaContext.java
b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergWriteSchemaContext.java
index 4de141ad1e5..9b2e8cfc859 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergWriteSchemaContext.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergWriteSchemaContext.java
@@ -641,6 +641,8 @@ public final class IcebergWriteSchemaContext {
case STRUCT:
return structExpression((Types.StructType) icebergType, value,
targetType,
enableMappingVarbinary, enableMappingTimestampTz);
+ case VARIANT:
+ throw new AnalysisException("Iceberg VARIANT write-default
must be NULL");
default:
throw new AnalysisException("Unsupported Iceberg write-default
type: " + icebergType);
}
diff --git
a/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/IcebergUtilsTest.java
b/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/IcebergUtilsTest.java
index 4c88db354b0..5ba783d1576 100644
---
a/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/IcebergUtilsTest.java
+++
b/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/IcebergUtilsTest.java
@@ -394,6 +394,27 @@ public class IcebergUtilsTest {
Assert.assertEquals(Types.VariantType.get(),
IcebergUtils.dorisTypeToIcebergType(type));
}
+ @Test
+ public void testIcebergVariantDefaultsMustBeNull() {
+ Types.VariantType variantType = Types.VariantType.get();
+ Schema schema = new Schema(Types.NestedField.optional(1, "payload",
variantType));
+
+ Assert.assertNull(IcebergUtils.parseIcebergLiteral(null, variantType));
+ Assert.assertTrue(IcebergUtils.getSerializedInitialDefaults(schema,
false).isEmpty());
+
+ IllegalArgumentException ddlException =
Assert.assertThrows(IllegalArgumentException.class,
+ () -> IcebergUtils.parseIcebergLiteral("{\"source\":\"ddl\"}",
variantType));
+ Assert.assertTrue(ddlException.getMessage().contains("VARIANT default
values must be NULL"));
+
+ Types.NestedField malformedField =
Mockito.mock(Types.NestedField.class);
+ Mockito.when(malformedField.fieldId()).thenReturn(1);
+ Mockito.when(malformedField.type()).thenReturn(variantType);
+
Mockito.when(malformedField.initialDefault()).thenReturn("non-null-variant");
+ IllegalArgumentException readException =
Assert.assertThrows(IllegalArgumentException.class,
+ () -> IcebergUtils.getSerializedInitialDefault(malformedField,
false));
+ Assert.assertTrue(readException.getMessage().contains("VARIANT
initial-default must be NULL"));
+ }
+
@Test
public void testIcebergVariantWriteCapabilityMatrix() {
Type variant =
IcebergUtils.icebergTypeToDorisType(Types.VariantType.get(), false, false);
diff --git
a/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/IcebergWriteSchemaContextTest.java
b/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/IcebergWriteSchemaContextTest.java
index a93010db281..65fd64d2115 100644
---
a/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/IcebergWriteSchemaContextTest.java
+++
b/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/IcebergWriteSchemaContextTest.java
@@ -217,6 +217,48 @@ public class IcebergWriteSchemaContextTest {
assertUnhexBytes(uuid, "123E4567E89B12D3A456426614174000");
}
+ @Test
+ public void testVariantWriteDefaultMustBeNull() {
+ Types.VariantType variantType = Types.VariantType.get();
+ Schema schema = new Schema(18, Arrays.asList(
+ Types.NestedField.optional(1, "optional_payload", variantType),
+ Types.NestedField.required(2, "required_payload",
variantType)));
+ IcebergWriteSchemaContext context =
IcebergWriteSchemaContext.forSchema(
+ schema, 3, true, true);
+ Map<String, Column> columns = context.getColumns().stream()
+ .collect(Collectors.toMap(Column::getName, column -> column));
+
+ Expression optionalDefault =
context.resolveWriteDefault(columns.get("optional_payload"));
+ Assertions.assertTrue(optionalDefault instanceof NullLiteral);
+
Assertions.assertEquals(DataType.fromCatalogType(columns.get("optional_payload").getType()),
+ optionalDefault.getDataType());
+ Assertions.assertThrows(AnalysisException.class,
+ () ->
context.resolveWriteDefault(columns.get("required_payload")));
+
+ AnalysisException nonNullException =
Assertions.assertThrows(AnalysisException.class,
+ () -> IcebergWriteSchemaContext.toDorisExpression(
+ variantType, new Object(),
optionalDefault.getDataType(), true, true));
+ Assertions.assertTrue(nonNullException.getMessage().contains(
+ "VARIANT write-default must be NULL"));
+ }
+
+ @Test
+ public void testStructWriteDefaultKeepsVariantChildNull() {
+ Types.StructType structType = Types.StructType.of(
+ Types.NestedField.optional(101, "payload",
Types.VariantType.get()));
+ DataType targetType =
DataType.fromCatalogType(IcebergUtils.icebergTypeToDorisType(
+ structType, true, true));
+
+ Expression expression = IcebergWriteSchemaContext.toDorisExpression(
+ structType, new ArrayStructLike((Object) null), targetType,
true, true);
+
+ Assertions.assertTrue(expression instanceof StructLiteral);
+ List<?> fields = ((StructLiteral) expression).getValue();
+ Assertions.assertEquals(1, fields.size());
+ Assertions.assertTrue(fields.get(0) instanceof NullLiteral);
+ Assertions.assertEquals(targetType, expression.getDataType());
+ }
+
@Test
public void testWriteDefaultResolutionDoesNotFallBackToReusedName() {
Schema pinnedSchema = new Schema(18, ImmutableList.of(defaultField(
diff --git
a/regression-test/data/external_table_p0/iceberg/test_iceberg_variant_defaults.out
b/regression-test/data/external_table_p0/iceberg/test_iceberg_variant_defaults.out
new file mode 100644
index 00000000000..37954279b42
--- /dev/null
+++
b/regression-test/data/external_table_p0/iceberg/test_iceberg_variant_defaults.out
@@ -0,0 +1,22 @@
+-- This file is automatically generated. You should know what you did if you
want to edit this
+-- !variant_default_historical_row --
+1 true \N
+
+-- !variant_default_rows --
+1 true \N
+2 true \N
+3 true \N
+4 true \N
+5 true \N
+6 true \N
+7 false null
+8 true \N
+
+-- !variant_default_counts --
+8 1 7
+
+-- !variant_explicit_default_null_column --
+8 0 8
+
+-- !variant_orc_historical_null --
+10 true \N
diff --git
a/regression-test/suites/external_table_p0/iceberg/test_iceberg_variant_defaults.groovy
b/regression-test/suites/external_table_p0/iceberg/test_iceberg_variant_defaults.groovy
new file mode 100644
index 00000000000..9d23975c1b9
--- /dev/null
+++
b/regression-test/suites/external_table_p0/iceberg/test_iceberg_variant_defaults.groovy
@@ -0,0 +1,160 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+suite("test_iceberg_variant_defaults",
+
"p0,external,iceberg,external_docker,external_docker_iceberg,nonConcurrent") {
+ String enabled = context.config.otherConfigs.get("enableIcebergTest")
+ if (enabled == null || !enabled.equalsIgnoreCase("true")) {
+ logger.info("disable iceberg test")
+ return
+ }
+
+ String restPort = context.config.otherConfigs.get("iceberg_rest_uri_port")
+ String minioPort = context.config.otherConfigs.get("iceberg_minio_port")
+ String externalEnvIp = context.config.otherConfigs.get("externalEnvIp")
+ String restUri = context.config.otherConfigs.get("iceberg_rest_uri")
+ if (restUri == null) {
+ restUri = "http://${externalEnvIp}:${restPort}";
+ }
+ String catalogName = "test_iceberg_variant_defaults"
+ String dbName = "iceberg_variant_defaults_db"
+ String tableName = "variant_defaults"
+ String orcTableName = "variant_defaults_orc_history"
+
+ spark_iceberg_multi """
+ CREATE NAMESPACE IF NOT EXISTS demo.${dbName};
+ DROP TABLE IF EXISTS demo.${dbName}.${tableName};
+ CREATE TABLE demo.${dbName}.${tableName} (id INT) USING iceberg
+ TBLPROPERTIES (
+ 'format-version'='3',
+ 'write.format.default'='parquet',
+ 'write.parquet.shred-variants'='false',
+ 'write.delete.mode'='merge-on-read',
+ 'write.update.mode'='merge-on-read',
+ 'write.merge.mode'='merge-on-read'
+ );
+ INSERT INTO demo.${dbName}.${tableName} VALUES (1);
+ ALTER TABLE demo.${dbName}.${tableName} ADD COLUMN payload VARIANT;
+
+ DROP TABLE IF EXISTS demo.${dbName}.${orcTableName};
+ CREATE TABLE demo.${dbName}.${orcTableName} (id INT) USING iceberg
+ TBLPROPERTIES ('format-version'='3', 'write.format.default'='orc');
+ INSERT INTO demo.${dbName}.${orcTableName} VALUES (10);
+ ALTER TABLE demo.${dbName}.${orcTableName} ADD COLUMN payload VARIANT;
+ """
+
+ sql """DROP CATALOG IF EXISTS ${catalogName}"""
+ sql """
+ CREATE CATALOG ${catalogName} PROPERTIES (
+ 'type'='iceberg',
+ 'iceberg.catalog.type'='rest',
+ 'uri'='${restUri}',
+ 's3.access_key'='admin',
+ 's3.secret_key'='password',
+ 's3.endpoint'='http://${externalEnvIp}:${minioPort}',
+ 's3.region'='us-east-1',
+ 'meta.cache.iceberg.table.ttl-second'='0',
+ 'meta.cache.iceberg.schema.ttl-second'='0'
+ )
+ """
+ sql """SWITCH ${catalogName}"""
+ sql """USE ${dbName}"""
+ sql """SET enable_file_scanner_v2=true"""
+
+ setFeConfigTemporary([enable_variant_v2: true]) {
+ order_qt_variant_default_historical_row """
+ SELECT id, payload IS NULL, CAST(payload AS STRING)
+ FROM ${tableName}
+ ORDER BY id
+ """
+
+ sql """INSERT INTO ${tableName} (id) VALUES (2)"""
+ sql """INSERT INTO ${tableName} (id, payload) VALUES (3, DEFAULT)"""
+ sql """
+ INSERT INTO ${tableName} VALUES
+ (4, PARSE_TO_VARIANT('{"source":"update"}')),
+ (5, PARSE_TO_VARIANT('{"source":"merge"}'))
+ """
+ sql """UPDATE ${tableName} SET payload = DEFAULT(payload) WHERE id =
4"""
+ sql """
+ MERGE INTO ${tableName} t
+ USING (SELECT 5 AS id UNION ALL SELECT 6 AS id) s
+ ON t.id = s.id
+ WHEN MATCHED THEN UPDATE SET payload = DEFAULT(payload)
+ WHEN NOT MATCHED THEN INSERT (id, payload) VALUES (s.id,
DEFAULT(payload))
+ """
+ sql """INSERT INTO ${tableName} VALUES (7, PARSE_TO_VARIANT('null'))"""
+ sql """
+ INSERT INTO ${tableName} (id, payload)
+ SELECT 8, DEFAULT(payload) FROM ${tableName} WHERE id = 7
+ """
+
+ order_qt_variant_default_rows """
+ SELECT id, payload IS NULL, CAST(payload AS STRING)
+ FROM ${tableName}
+ ORDER BY id
+ """
+ qt_variant_default_counts """
+ SELECT COUNT(*), COUNT(payload), SUM(payload IS NULL)
+ FROM ${tableName}
+ """
+
+ sql """ALTER TABLE ${tableName} ADD COLUMN auxiliary VARIANT DEFAULT
NULL"""
+ qt_variant_explicit_default_null_column """
+ SELECT COUNT(*), COUNT(auxiliary), SUM(auxiliary IS NULL)
+ FROM ${tableName}
+ """
+
+ List<List<Object>> schemaBeforeInvalidDefault = sql """DESC
${tableName}"""
+ test {
+ sql """
+ ALTER TABLE ${tableName}
+ ADD COLUMN invalid_payload VARIANT DEFAULT '{"invalid":true}'
+ """
+ exception "support null"
+ }
+ assertEquals(schemaBeforeInvalidDefault, sql("DESC ${tableName}"),
+ "Rejected non-null VARIANT default changed the Iceberg schema")
+
+ test {
+ sql """
+ ALTER TABLE ${tableName}
+ ADD COLUMN required_payload VARIANT NOT NULL DEFAULT NULL
+ """
+ exception "null default value"
+ }
+ assertEquals(schemaBeforeInvalidDefault, sql("DESC ${tableName}"),
+ "Rejected required VARIANT default changed the Iceberg schema")
+
+ // Spark caches Iceberg snapshots, so refresh after Doris commits
before cross-engine reads.
+ spark_iceberg """REFRESH TABLE demo.${dbName}.${tableName}"""
+ List<List<Object>> sparkCounts = spark_iceberg """
+ SELECT COUNT(*), COUNT(payload), COUNT(auxiliary)
+ FROM demo.${dbName}.${tableName}
+ """
+ assertEquals(1, sparkCounts.size())
+ assertEquals("8", sparkCounts[0][0].toString())
+ assertEquals("1", sparkCounts[0][1].toString())
+ assertEquals("0", sparkCounts[0][2].toString())
+
+ order_qt_variant_orc_historical_null """
+ SELECT id, payload IS NULL, CAST(payload AS STRING)
+ FROM ${orcTableName}
+ ORDER BY id
+ """
+ }
+}
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
