This is an automated email from the ASF dual-hosted git repository.
yiguolei pushed a commit to branch branch-4.1
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/branch-4.1 by this push:
new c3cfb3805d4 [opt](paimon) report error when mapping variant type
disable variant_v2 (#66466)
c3cfb3805d4 is described below
commit c3cfb3805d4303175cabb1ca42277b1017c46dc3
Author: zhangstar333 <[email protected]>
AuthorDate: Fri Aug 7 10:50:23 2026 +0800
[opt](paimon) report error when mapping variant type disable variant_v2
(#66466)
### What problem does this PR solve?
Problem Summary:
variant type mapping in paimon, should set enable_file_scanner_v2 ,
experimental_enable_variant_v2 to true.
and format should be parquet files. others will report error.
### Release note
None
### Check List (For Author)
- Test <!-- At least one of them must be included. -->
- [x] Regression test
- [ ] Unit Test
- [ ] Manual test (add detailed scripts or steps below)
- [ ] No need to test or manual test. Explain why:
- [ ] This is a refactor/code format and no logic has been changed.
- [ ] Previous test can cover this change.
- [ ] No code files have been changed.
- [ ] Other reason <!-- Add your reason? -->
- Behavior changed:
- [ ] No.
- [ ] Yes. <!-- Explain the behavior change -->
- Does this need documentation?
- [ ] No.
- [ ] Yes. <!-- Add document PR link here. eg:
https://github.com/apache/doris-website/pull/1214 -->
### Check List (For Reviewer who merge this PR)
- [ ] Confirm the release note
- [ ] Confirm test cases
- [ ] Confirm document
- [ ] Add branch pick label <!-- Add branch pick label that this PR
should merge into -->
---
be/src/format_v2/table/iceberg_reader.cpp | 92 ----------------------
be/src/format_v2/table/iceberg_reader.h | 4 -
be/src/format_v2/table_reader.cpp | 60 ++++++++++++++
be/src/format_v2/table_reader.h | 4 +-
be/test/format_v2/table/iceberg_reader_test.cpp | 39 ---------
be/test/format_v2/table_reader_test.cpp | 40 ++++++++++
.../create_preinstalled_scripts/paimon/run13.sql | 26 ++++++
.../apache/doris/datasource/paimon/PaimonUtil.java | 18 +++++
.../datasource/paimon/source/PaimonScanNode.java | 12 +++
.../paimon/source/PaimonScanNodeTest.java | 15 ++++
.../paimon/test_paimon_catalog_variant.out | 16 ++++
.../paimon/test_paimon_catalog_variant.groovy | 60 ++++++++++++++
.../test_remote_doris_variant_select.groovy | 3 +-
13 files changed, 252 insertions(+), 137 deletions(-)
diff --git a/be/src/format_v2/table/iceberg_reader.cpp
b/be/src/format_v2/table/iceberg_reader.cpp
index 3291c9c79cc..5289ba18651 100644
--- a/be/src/format_v2/table/iceberg_reader.cpp
+++ b/be/src/format_v2/table/iceberg_reader.cpp
@@ -31,8 +31,6 @@
#include "core/column/column_string.h"
#include "core/column/column_struct.h"
#include "core/column/column_vector.h"
-#include "core/data_type/data_type_array.h"
-#include "core/data_type/data_type_map.h"
#include "core/data_type/data_type_number.h"
#include "core/data_type/data_type_struct.h"
#include "core/data_type/define_primitive_type.h"
@@ -55,96 +53,6 @@ namespace doris::format::iceberg {
static constexpr const char* ROW_LINEAGE_ROW_ID = "_row_id";
static constexpr int32_t ROW_LINEAGE_ROW_ID_FIELD_ID = 2147483540;
-namespace {
-
-bool contains_variant_type(const DataTypePtr& input) {
- if (input == nullptr) {
- return false;
- }
- const auto type = remove_nullable(input);
- switch (type->get_primitive_type()) {
- case TYPE_VARIANT:
- return true;
- case TYPE_ARRAY:
- return contains_variant_type(assert_cast<const
DataTypeArray&>(*type).get_nested_type());
- case TYPE_MAP: {
- const auto& map = assert_cast<const DataTypeMap&>(*type);
- return contains_variant_type(map.get_key_type()) ||
- contains_variant_type(map.get_value_type());
- }
- case TYPE_STRUCT:
- return std::ranges::any_of(assert_cast<const
DataTypeStruct&>(*type).get_elements(),
- contains_variant_type);
- default:
- return false;
- }
-}
-
-bool mapping_reads_variant(const format::ColumnMapping& mapping) {
- if (!mapping.file_local_id.has_value()) {
- return false;
- }
- if (contains_variant_type(mapping.original_file_type)) {
- return true;
- }
- if (mapping.table_type != nullptr &&
- remove_nullable(mapping.table_type)->get_primitive_type() ==
TYPE_VARIANT) {
- return true;
- }
- return std::ranges::any_of(mapping.child_mappings, mapping_reads_variant);
-}
-
-const char* file_format_name(FileFormat format) {
- switch (format) {
- case FileFormat::PARQUET:
- return "PARQUET";
- case FileFormat::ORC:
- return "ORC";
- case FileFormat::CSV:
- return "CSV";
- case FileFormat::JSON:
- return "JSON";
- case FileFormat::TEXT:
- return "TEXT";
- case FileFormat::JNI:
- return "JNI";
- case FileFormat::NATIVE:
- return "NATIVE";
- case FileFormat::ARROW:
- return "ARROW";
- case FileFormat::WAL:
- return "WAL";
- case FileFormat::LANCE:
- return "LANCE";
- }
- return "UNKNOWN";
-}
-
-} // namespace
-
-Status IcebergTableReader::validate_variant_file_mappings(
- FileFormat format, const std::vector<format::ColumnMapping>& mappings)
{
- if (format == FileFormat::PARQUET || !std::ranges::any_of(mappings,
mapping_reads_variant)) {
- return Status::OK();
- }
- // Gate on a physical mapping, not the table schema: an older ORC/Avro
file may legitimately
- // omit a Variant field added by schema evolution, in which case the
mapper synthesizes NULL.
- return Status::NotSupported(
- "Iceberg Variant is supported only for Parquet files in
FileScannerV2; file format {} "
- "(including ORC/Avro readers) is not supported",
- file_format_name(format));
-}
-
-Status IcebergTableReader::validate_file_mapping(const
format::TableColumnMapper& mapper) const {
- if (_push_down_agg_type == TPushAggOp::type::COUNT &&
_push_down_count_columns.has_value() &&
- _push_down_count_columns->empty()) {
- // COUNT(*) may retain an arbitrary minimum-width slot, but that
carrier is never a
- // semantic physical read and must not trigger the Variant file-format
capability gate.
- return Status::OK();
- }
- return validate_variant_file_mappings(_format, mapper.mappings());
-}
-
template <typename T>
static std::string join_values_for_debug(const std::vector<T>& values) {
std::ostringstream out;
diff --git a/be/src/format_v2/table/iceberg_reader.h
b/be/src/format_v2/table/iceberg_reader.h
index fd97f130408..d28be3d7f98 100644
--- a/be/src/format_v2/table/iceberg_reader.h
+++ b/be/src/format_v2/table/iceberg_reader.h
@@ -49,8 +49,6 @@ namespace doris::format::iceberg {
class IcebergTableReader : public format::TableReader {
public:
~IcebergTableReader() override = default;
- static Status validate_variant_file_mappings(
- FileFormat format, const std::vector<format::ColumnMapping>&
mappings);
Status init(format::TableReadOptions&& options) override {
RETURN_IF_ERROR(format::TableReader::init(std::move(options)));
_mapper_options.mode = format::TableColumnMappingMode::BY_FIELD_ID;
@@ -70,8 +68,6 @@ public:
}
protected:
- Status validate_file_mapping(const format::TableColumnMapper& mapper)
const override;
-
void configure_mapper_options(format::TableColumnMapperOptions* options)
const override {
options->enable_row_lineage_virtual_columns = true;
options->allow_idless_complex_wrapper_projection =
diff --git a/be/src/format_v2/table_reader.cpp
b/be/src/format_v2/table_reader.cpp
index 07bcae2ceb5..2ff72c1edb9 100644
--- a/be/src/format_v2/table_reader.cpp
+++ b/be/src/format_v2/table_reader.cpp
@@ -98,6 +98,43 @@ std::string file_format_to_string(FileFormat format) {
return "UNKNOWN";
}
+bool contains_variant_type(const DataTypePtr& input) {
+ if (input == nullptr) {
+ return false;
+ }
+ const auto type = remove_nullable(input);
+ switch (type->get_primitive_type()) {
+ case TYPE_VARIANT:
+ return true;
+ case TYPE_ARRAY:
+ return contains_variant_type(assert_cast<const
DataTypeArray&>(*type).get_nested_type());
+ case TYPE_MAP: {
+ const auto& map = assert_cast<const DataTypeMap&>(*type);
+ return contains_variant_type(map.get_key_type()) ||
+ contains_variant_type(map.get_value_type());
+ }
+ case TYPE_STRUCT:
+ return std::ranges::any_of(assert_cast<const
DataTypeStruct&>(*type).get_elements(),
+ contains_variant_type);
+ default:
+ return false;
+ }
+}
+
+bool mapping_reads_variant(const ColumnMapping& mapping) {
+ if (!mapping.file_local_id.has_value()) {
+ return false;
+ }
+ if (contains_variant_type(mapping.original_file_type)) {
+ return true;
+ }
+ if (mapping.table_type != nullptr &&
+ remove_nullable(mapping.table_type)->get_primitive_type() ==
TYPE_VARIANT) {
+ return true;
+ }
+ return std::ranges::any_of(mapping.child_mappings, mapping_reads_variant);
+}
+
std::string push_down_agg_to_string(TPushAggOp::type op) {
switch (op) {
case TPushAggOp::NONE:
@@ -746,6 +783,29 @@ Status TableReader::init(TableReadOptions&& options) {
return Status::OK();
}
+Status TableReader::validate_variant_file_mappings(FileFormat format,
+ const
std::vector<ColumnMapping>& mappings) {
+ if (format == FileFormat::PARQUET || !std::ranges::any_of(mappings,
mapping_reads_variant)) {
+ return Status::OK();
+ }
+ // Gate on a physical mapping, not the table schema: an older file may
legitimately omit a
+ // Variant field added by schema evolution, in which case the mapper
synthesizes NULL.
+ return Status::NotSupported(
+ "External Variant is supported only for Parquet files in
FileScannerV2; file format "
+ "{} is not supported",
+ file_format_to_string(format));
+}
+
+Status TableReader::validate_file_mapping(const TableColumnMapper& mapper)
const {
+ if (_push_down_agg_type == TPushAggOp::type::COUNT &&
_push_down_count_columns.has_value() &&
+ _push_down_count_columns->empty()) {
+ // COUNT(*) may retain an arbitrary minimum-width slot, but that
carrier is never a
+ // semantic physical read and must not trigger the Variant file-format
capability gate.
+ return Status::OK();
+ }
+ return validate_variant_file_mappings(_format, mapper.mappings());
+}
+
Status TableReader::_build_table_filters_from_conjuncts() {
_table_filters.clear();
_constant_pruning_safe_filter_count = 0;
diff --git a/be/src/format_v2/table_reader.h b/be/src/format_v2/table_reader.h
index f801f36e66b..e160ee2867e 100644
--- a/be/src/format_v2/table_reader.h
+++ b/be/src/format_v2/table_reader.h
@@ -411,7 +411,9 @@ protected:
DORIS_CHECK(file_schema != nullptr);
return Status::OK();
}
- virtual Status validate_file_mapping(const TableColumnMapper&) const {
return Status::OK(); }
+ static Status validate_variant_file_mappings(FileFormat format,
+ const
std::vector<ColumnMapping>& mappings);
+ virtual Status validate_file_mapping(const TableColumnMapper& mapper)
const;
// Open the concrete reader for the current split/task and build the
file-local scan request.
virtual Status open_reader() {
diff --git a/be/test/format_v2/table/iceberg_reader_test.cpp
b/be/test/format_v2/table/iceberg_reader_test.cpp
index 4884cb7eee0..782463e335a 100644
--- a/be/test/format_v2/table/iceberg_reader_test.cpp
+++ b/be/test/format_v2/table/iceberg_reader_test.cpp
@@ -58,7 +58,6 @@
#include "core/data_type/data_type_struct.h"
#include "core/data_type/data_type_timestamptz.h"
#include "core/data_type/data_type_varbinary.h"
-#include "core/data_type/data_type_variant_v2.h"
#include "exec/common/endian.h"
#include "exec/scan/access_path_parser.h"
#include "exprs/runtime_filter_expr.h"
@@ -2035,44 +2034,6 @@ TEST(IcebergV2ReaderTest,
IcebergLegacyPlanKeepsAllFieldIdsMappingRule) {
TableColumnMappingMode::BY_NAME);
}
-TEST(IcebergV2ReaderTest, VariantFormatGateUsesPhysicalFileMappings) {
- ColumnMapping missing_variant;
- missing_variant.table_type =
make_nullable(std::make_shared<DataTypeVariantV2>());
-
EXPECT_TRUE(doris::format::iceberg::IcebergTableReader::validate_variant_file_mappings(
- FileFormat::ORC, {missing_variant})
- .ok());
-
- ColumnMapping physical_variant = missing_variant;
- physical_variant.file_local_id = 0;
- const auto orc_status =
-
doris::format::iceberg::IcebergTableReader::validate_variant_file_mappings(
- FileFormat::ORC, {physical_variant});
- EXPECT_TRUE(orc_status.is<ErrorCode::NOT_IMPLEMENTED_ERROR>()) <<
orc_status;
-
EXPECT_TRUE(doris::format::iceberg::IcebergTableReader::validate_variant_file_mappings(
- FileFormat::PARQUET, {physical_variant})
- .ok());
-
- ColumnMapping projected_struct;
- projected_struct.table_type =
make_nullable(std::make_shared<DataTypeStruct>(
- DataTypes {make_nullable(std::make_shared<DataTypeString>()),
- make_nullable(std::make_shared<DataTypeVariantV2>())},
- Strings {"label", "payload"}));
- projected_struct.file_local_id = 0;
- ColumnMapping label;
- label.table_type = make_nullable(std::make_shared<DataTypeString>());
- label.file_local_id = 1;
- projected_struct.child_mappings = {label, missing_variant};
-
EXPECT_TRUE(doris::format::iceberg::IcebergTableReader::validate_variant_file_mappings(
- FileFormat::ORC, {projected_struct})
- .ok());
-
- projected_struct.child_mappings[1] = physical_variant;
- const auto nested_orc_status =
-
doris::format::iceberg::IcebergTableReader::validate_variant_file_mappings(
- FileFormat::ORC, {projected_struct});
- EXPECT_TRUE(nested_orc_status.is<ErrorCode::NOT_IMPLEMENTED_ERROR>()) <<
nested_orc_status;
-}
-
TEST(IcebergV2ReaderTest,
IcebergTableReaderDoesNotPushDownAggregateWithPositionDelete) {
const auto test_dir =
std::filesystem::temp_directory_path() /
"doris_iceberg_aggregate_position_delete_test";
diff --git a/be/test/format_v2/table_reader_test.cpp
b/be/test/format_v2/table_reader_test.cpp
index f4bc1911d17..22b62286842 100644
--- a/be/test/format_v2/table_reader_test.cpp
+++ b/be/test/format_v2/table_reader_test.cpp
@@ -50,6 +50,7 @@
#include "core/data_type/data_type_string.h"
#include "core/data_type/data_type_struct.h"
#include "core/data_type/data_type_varbinary.h"
+#include "core/data_type/data_type_variant_v2.h"
#include "exprs/runtime_filter_expr.h"
#include "exprs/vectorized_fn_call.h"
#include "exprs/vexpr.h"
@@ -76,6 +77,45 @@ std::vector<int32_t> projection_ids(const
std::vector<LocalColumnIndex>& project
return ids;
}
+class VariantValidationTableReader final : public TableReader {
+public:
+ static Status validate(FileFormat format, const
std::vector<ColumnMapping>& mappings) {
+ return validate_variant_file_mappings(format, mappings);
+ }
+};
+
+TEST(TableReaderTest, VariantFormatGateUsesPhysicalFileMappings) {
+ ColumnMapping missing_variant;
+ missing_variant.table_type =
make_nullable(std::make_shared<DataTypeVariantV2>());
+ EXPECT_TRUE(VariantValidationTableReader::validate(FileFormat::ORC,
{missing_variant}).ok());
+
+ ColumnMapping physical_variant = missing_variant;
+ physical_variant.file_local_id = 0;
+ const auto orc_status =
+ VariantValidationTableReader::validate(FileFormat::ORC,
{physical_variant});
+ EXPECT_TRUE(orc_status.is<ErrorCode::NOT_IMPLEMENTED_ERROR>()) <<
orc_status;
+ EXPECT_NE(orc_status.to_string().find("supported only for Parquet"),
std::string::npos);
+ EXPECT_TRUE(
+ VariantValidationTableReader::validate(FileFormat::PARQUET,
{physical_variant}).ok());
+
+ ColumnMapping projected_struct;
+ projected_struct.table_type =
make_nullable(std::make_shared<DataTypeStruct>(
+ DataTypes {make_nullable(std::make_shared<DataTypeString>()),
+ make_nullable(std::make_shared<DataTypeVariantV2>())},
+ Strings {"label", "payload"}));
+ projected_struct.file_local_id = 0;
+ ColumnMapping label;
+ label.table_type = make_nullable(std::make_shared<DataTypeString>());
+ label.file_local_id = 1;
+ projected_struct.child_mappings = {label, missing_variant};
+ EXPECT_TRUE(VariantValidationTableReader::validate(FileFormat::ORC,
{projected_struct}).ok());
+
+ projected_struct.child_mappings[1] = physical_variant;
+ const auto nested_orc_status =
+ VariantValidationTableReader::validate(FileFormat::ORC,
{projected_struct});
+ EXPECT_TRUE(nested_orc_status.is<ErrorCode::NOT_IMPLEMENTED_ERROR>()) <<
nested_orc_status;
+}
+
TEST(LocalColumnIndexTest,
MergeUnionsPartialChildrenAndFullProjectionDominates) {
LocalColumnIndex target {.index = 10, .project_all_children = false};
target.children.push_back({.index = 1});
diff --git
a/docker/thirdparties/docker-compose/iceberg/scripts/create_preinstalled_scripts/paimon/run13.sql
b/docker/thirdparties/docker-compose/iceberg/scripts/create_preinstalled_scripts/paimon/run13.sql
index 6e528dcdb9f..fc4506d0a1e 100644
---
a/docker/thirdparties/docker-compose/iceberg/scripts/create_preinstalled_scripts/paimon/run13.sql
+++
b/docker/thirdparties/docker-compose/iceberg/scripts/create_preinstalled_scripts/paimon/run13.sql
@@ -70,3 +70,29 @@ alter table variant_mixed_su unset tblproperties (
);
insert into variant_mixed_su values
(2, date '2026-07-01',
parse_json('{"name":"bob","age":30,"layout":"unshredded"}'));
+
+-- Keep a primary-key table separate from variant_smoke so the regression
covers both Paimon
+-- append-only reads and deduplicate merge reads with Variant payloads.
+drop table if exists variant_primary_key_smoke;
+create table variant_primary_key_smoke (
+ id BIGINT,
+ payload VARIANT
+) using paimon
+tblproperties (
+ 'primary-key' = 'id',
+ 'bucket' = '1',
+ 'merge-engine' = 'deduplicate',
+ 'file.format' = 'parquet'
+);
+
+insert into variant_primary_key_smoke values
+ (1,
parse_json('{"name":"alice","version":1,"active":false,"profile":{"city":"beijing"}}')),
+ (2, parse_json('{"name":"bob","version":1,"tags":["old"]}')),
+ (3, parse_json('[10,"original"]'));
+
+-- A second snapshot updates two existing keys and inserts a new key. Doris
should expose only the
+-- latest logical row for ids 1 and 2.
+insert into variant_primary_key_smoke values
+ (1,
parse_json('{"name":"alice-updated","version":2,"active":true,"profile":{"city":"hangzhou"}}')),
+ (2, parse_json('{"name":"bob","version":2,"tags":["new","primary-key"]}')),
+ (4, parse_json('{"name":"carol","version":1,"active":true}'));
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonUtil.java
b/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonUtil.java
index c3deea13c63..2bb908c6204 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonUtil.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonUtil.java
@@ -345,6 +345,9 @@ public class PaimonUtil {
}
return ScalarType.createDatetimeV2Type(tsScale);
case VARIANT:
+ // External-table schemas are cached and shared across
sessions, so this mapping
+ // must not depend on enable_variant_v2. PaimonScanNode checks
that
+ // session variable against the VARIANT slots projected by
each query instead.
return VariantType.COMPUTE_V2_INSTANCE;
case ARRAY:
ArrayType arrayType = (ArrayType) dataType;
@@ -377,6 +380,21 @@ public class PaimonUtil {
return paimonPrimitiveTypeToDorisType(type, enableVarbinaryMapping,
enableTimestampTzMapping);
}
+ public static boolean containsVariant(Type type) {
+ if (type.isVariantType()) {
+ return true;
+ } else if (type.isArrayType()) {
+ return containsVariant(((org.apache.doris.catalog.ArrayType)
type).getItemType());
+ } else if (type.isMapType()) {
+ org.apache.doris.catalog.MapType mapType =
(org.apache.doris.catalog.MapType) type;
+ return containsVariant(mapType.getKeyType()) ||
containsVariant(mapType.getValueType());
+ } else if (type.isStructType()) {
+ return ((org.apache.doris.catalog.StructType)
type).getFields().stream()
+ .anyMatch(field -> containsVariant(field.getType()));
+ }
+ return false;
+ }
+
public static void updatePaimonColumnUniqueId(Column column, DataType
dataType) {
List<Column> columns = column.getChildren();
if (columns == null) {
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/source/PaimonScanNode.java
b/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/source/PaimonScanNode.java
index ebd0dcbf4fb..bcf5be93649 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/source/PaimonScanNode.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/source/PaimonScanNode.java
@@ -198,6 +198,7 @@ public class PaimonScanNode extends FileQueryScanNode {
@Override
protected void doInitialize() throws UserException {
+ checkVariantV2Enabled(desc, sessionVariable.isEnableVariantV2());
Optional<MvccSnapshot> relationSnapshot = getRelationSnapshot();
if (desc.getTable() instanceof PaimonExternalTable
|| desc.getTable() instanceof PaimonSysExternalTable) {
@@ -252,6 +253,17 @@ public class PaimonScanNode extends FileQueryScanNode {
}
}
+ @VisibleForTesting
+ static void checkVariantV2Enabled(TupleDescriptor tuple, boolean enabled)
throws UserException {
+ // Enforce the session switch here instead of during Paimon schema
conversion. The cached
+ // external schema is shared by sessions, while this tuple contains
only the slots projected
+ // by the current query and therefore does not reject scans of
unrelated non-VARIANT columns.
+ if (!enabled && tuple.getSlots().stream().anyMatch(slot ->
PaimonUtil.containsVariant(slot.getType()))) {
+ throw new UserException(
+ "Paimon VARIANT columns require enable_variant_v2=true");
+ }
+ }
+
private void serializeProcessedTable() throws UserException {
// System-table splits are materialized by the BE JNI reader, so it
must receive the same
// option-bearing table copy that FE uses to plan the split.
diff --git
a/fe/fe-core/src/test/java/org/apache/doris/datasource/paimon/source/PaimonScanNodeTest.java
b/fe/fe-core/src/test/java/org/apache/doris/datasource/paimon/source/PaimonScanNodeTest.java
index 92f8da8d3b6..50c37928ac9 100644
---
a/fe/fe-core/src/test/java/org/apache/doris/datasource/paimon/source/PaimonScanNodeTest.java
+++
b/fe/fe-core/src/test/java/org/apache/doris/datasource/paimon/source/PaimonScanNodeTest.java
@@ -17,11 +17,13 @@
package org.apache.doris.datasource.paimon.source;
+import org.apache.doris.analysis.SlotDescriptor;
import org.apache.doris.analysis.SlotId;
import org.apache.doris.analysis.TableScanParams;
import org.apache.doris.analysis.TupleDescriptor;
import org.apache.doris.analysis.TupleId;
import org.apache.doris.catalog.Column;
+import org.apache.doris.catalog.VariantType;
import org.apache.doris.common.ExceptionChecker;
import org.apache.doris.common.UserException;
import org.apache.doris.datasource.CatalogProperty;
@@ -105,6 +107,19 @@ public class PaimonScanNodeTest {
@Mock
private PaimonFileExternalCatalog paimonFileExternalCatalog;
+ @Test
+ public void testVariantProjectionRequiresVariantV2() throws UserException {
+ TupleDescriptor desc = new TupleDescriptor(new TupleId(0));
+ SlotDescriptor slot = new SlotDescriptor(new SlotId(0), desc);
+ slot.setColumn(new Column("payload", VariantType.COMPUTE_V2_INSTANCE));
+ desc.addSlot(slot);
+
+ ExceptionChecker.expectThrowsWithMsg(UserException.class,
+ "Paimon VARIANT columns require enable_variant_v2=true",
+ () -> PaimonScanNode.checkVariantV2Enabled(desc, false));
+ PaimonScanNode.checkVariantV2Enabled(desc, true);
+ }
+
@Test
public void testSerializedTableCacheKeyIsStablePerScanNode() {
PaimonScanNode first = newTestNode(new PlanNodeId(0), new TupleId(0),
sv);
diff --git
a/regression-test/data/external_table_p0/paimon/test_paimon_catalog_variant.out
b/regression-test/data/external_table_p0/paimon/test_paimon_catalog_variant.out
index 830e466af48..b33530f97e0 100644
---
a/regression-test/data/external_table_p0/paimon/test_paimon_catalog_variant.out
+++
b/regression-test/data/external_table_p0/paimon/test_paimon_catalog_variant.out
@@ -24,6 +24,22 @@ payload variant<PROPERTIES
("variant_max_subcolumns_count" = "0","variant_enable
-- !subpath_predicate --
2 bob
+-- !primary_key_full_variant --
+1
{"active":true,"name":"alice-updated","profile":{"city":"hangzhou"},"version":2}
+2 {"name":"bob","tags":["new","primary-key"],"version":2}
+3 [10,"original"]
+4 {"active":true,"name":"carol","version":1}
+
+-- !primary_key_deduplicate --
+1 alice-updated 2 true
+2 bob 2 \N
+3 \N \N \N
+4 carol 1 true
+
+-- !primary_key_subpath_predicate --
+1 alice-updated
+2 bob
+
-- !native_full_variant --
1
{"active":true,"age":18,"missing":null,"name":"alice","profile":{"city":"beijing","zip":100000},"score":98.5,"tags":["flink","paimon"]}
2
{"active":false,"age":30,"extra":{"levels":[1,2,3]},"name":"bob","profile":{"city":"shanghai"},"tags":["doris"]}
diff --git
a/regression-test/suites/external_table_p0/paimon/test_paimon_catalog_variant.groovy
b/regression-test/suites/external_table_p0/paimon/test_paimon_catalog_variant.groovy
index 203d5788310..fdba0df3f8a 100644
---
a/regression-test/suites/external_table_p0/paimon/test_paimon_catalog_variant.groovy
+++
b/regression-test/suites/external_table_p0/paimon/test_paimon_catalog_variant.groovy
@@ -34,7 +34,9 @@ suite("test_paimon_catalog_variant",
"p0,external,doris,external_docker,external
"s3.path.style.access" = "true"
);"""
sql """use `${catalogName}`.`test_paimon_spark`"""
+ // JNI reader cases.
sql """set enable_variant_v2 = true"""
+ sql """set enable_file_scanner_v2 = true"""
sql """set force_jni_scanner = true"""
explain {
@@ -87,6 +89,34 @@ suite("test_paimon_catalog_variant",
"p0,external,doris,external_docker,external
order by id
"""
+ // variant_smoke is append-only. This table has duplicate primary-key
writes, so the scan
+ // must preserve Paimon's deduplicate semantics while materializing
Variant values.
+ order_qt_primary_key_full_variant """
+ select id, payload
+ from variant_primary_key_smoke
+ order by id
+ """
+
+ order_qt_primary_key_deduplicate """
+ select id,
+ cast(payload['name'] as string),
+ cast(payload['version'] as int),
+ cast(payload['active'] as boolean)
+ from variant_primary_key_smoke
+ order by id
+ """
+
+ order_qt_primary_key_subpath_predicate """
+ select id, cast(payload['name'] as string)
+ from variant_primary_key_smoke
+ where cast(payload['version'] as int) = 2
+ order by id
+ """
+
+ // Native reader cases. Reset every relevant switch explicitly so the
JNI cases above do
+ // not leak their session state into this block.
+ sql """set enable_variant_v2 = true"""
+ sql """set enable_file_scanner_v2 = true"""
sql """set force_jni_scanner = false"""
explain {
@@ -220,5 +250,35 @@ suite("test_paimon_catalog_variant",
"p0,external,doris,external_docker,external
} finally {
sql """drop materialized view if exists ${mvName}"""
}
+
+ // FileScannerV2 is required by both the native and JNI scan paths for
external VARIANT.
+ sql """set enable_variant_v2 = true"""
+ sql """set enable_file_scanner_v2 = false"""
+ sql """set force_jni_scanner = false"""
+ test {
+ sql """select * from
${catalogName}.test_paimon_spark.variant_smoke"""
+ exception "External VARIANT columns require FileScannerV2"
+ }
+
+ sql """set force_jni_scanner = true"""
+ test {
+ sql """select * from
${catalogName}.test_paimon_spark.variant_smoke"""
+ exception "External VARIANT columns require FileScannerV2"
+ }
+
+ // The Paimon VARIANT feature switch is checked for both JNI and
native planning.
+ sql """set enable_file_scanner_v2 = true"""
+ sql """set enable_variant_v2 = false"""
+ sql """set force_jni_scanner = true"""
+ test {
+ sql """select * from
${catalogName}.test_paimon_spark.variant_smoke"""
+ exception "Paimon VARIANT columns require enable_variant_v2=true"
+ }
+
+ sql """set force_jni_scanner = false"""
+ test {
+ sql """select * from
${catalogName}.test_paimon_spark.variant_smoke"""
+ exception "Paimon VARIANT columns require enable_variant_v2=true"
+ }
}
}
diff --git
a/regression-test/suites/external_table_p0/remote_doris/test_remote_doris_variant_select.groovy
b/regression-test/suites/external_table_p0/remote_doris/test_remote_doris_variant_select.groovy
index 6aa297aa98b..26f35391934 100644
---
a/regression-test/suites/external_table_p0/remote_doris/test_remote_doris_variant_select.groovy
+++
b/regression-test/suites/external_table_p0/remote_doris/test_remote_doris_variant_select.groovy
@@ -112,7 +112,8 @@ suite("test_remote_doris_variant_select",
"p0,external,doris,external_docker,ext
select * from
`${catalog_arrow_name}`.`${db_name}`.`test_remote_doris_variant_select_t` order
by id
"""
// check exception message contains
- exception "[NOT_IMPLEMENTED_ERROR]read_column_from_arrow with type
variant"
+ exception "External Variant is supported only for Parquet files in
FileScannerV2; "
+ + "file format ARROW is not supported"
}
qt_sql """
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
