This is an automated email from the ASF dual-hosted git repository.
eldenmoon pushed a commit to branch branch-4.1
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/branch-4.1 by this push:
new df8ed67a8e5 branch-4.1: [fix](be) Reject legacy Variant blocks before
deserialization (#69434) (#70492)
df8ed67a8e5 is described below
commit df8ed67a8e5dbdc91f1065b50dae23d834117800
Author: lihangyu <[email protected]>
AuthorDate: Thu Nov 12 10:59:49 2026 +0800
branch-4.1: [fix](be) Reject legacy Variant blocks before deserialization
(#69434) (#70492)
cherry-pick #69434
---
be/src/core/data_type/data_type_factory.cpp | 3 +-
be/src/core/data_type/data_type_variant.cpp | 15 ++-
be/src/core/data_type/data_type_variant.h | 4 +-
.../core/block/block_variant_wire_format_test.cpp | 147 +++++++++++++++++++++
4 files changed, 165 insertions(+), 4 deletions(-)
diff --git a/be/src/core/data_type/data_type_factory.cpp
b/be/src/core/data_type/data_type_factory.cpp
index bee6065acac..93697c9cf7e 100644
--- a/be/src/core/data_type/data_type_factory.cpp
+++ b/be/src/core/data_type/data_type_factory.cpp
@@ -250,7 +250,8 @@ DataTypePtr DataTypeFactory::create_data_type(const
PColumnMeta& pcolumn) {
pcolumn.variant_enable_doc_mode());
} else {
nested =
std::make_shared<DataTypeVariant>(pcolumn.variant_max_subcolumns_count(),
-
pcolumn.variant_enable_doc_mode());
+
pcolumn.variant_enable_doc_mode(),
+
pcolumn.has_variant_enable_doc_mode());
}
break;
case PGenericType::JSONB:
diff --git a/be/src/core/data_type/data_type_variant.cpp
b/be/src/core/data_type/data_type_variant.cpp
index c83efe4cc45..19acf664c13 100644
--- a/be/src/core/data_type/data_type_variant.cpp
+++ b/be/src/core/data_type/data_type_variant.cpp
@@ -49,8 +49,11 @@ namespace doris {
DataTypeVariant::DataTypeVariant(int32_t max_subcolumns_count)
: DataTypeVariant(max_subcolumns_count, false) {}
-DataTypeVariant::DataTypeVariant(int32_t max_subcolumns_count, bool
enable_doc_mode)
- : _max_subcolumns_count(max_subcolumns_count),
_enable_doc_mode(enable_doc_mode) {
+DataTypeVariant::DataTypeVariant(int32_t max_subcolumns_count, bool
enable_doc_mode,
+ bool has_doc_value_serialization)
+ : _max_subcolumns_count(max_subcolumns_count),
+ _enable_doc_mode(enable_doc_mode),
+ _has_doc_value_serialization(has_doc_value_serialization) {
name = fmt::format("Variant(max subcolumns count = {}, enable doc mode =
{})",
max_subcolumns_count, enable_doc_mode);
}
@@ -179,6 +182,14 @@ Field DataTypeVariant::get_field(const TExprNode& node)
const {
const char* DataTypeVariant::deserialize(const char* buf, MutableColumnPtr*
column,
int be_exec_version) const {
+ // Older V1 senders omit both this metadata field and the trailing
doc-value Map.
+ // Even doc_mode=false senders on this branch serialize that Map, so check
presence.
+ if (!_has_doc_value_serialization) {
+ throw Exception(Status::NotSupported(
+ "Unsupported VARIANT block format: Variant V1 blocks without
doc-value data "
+ "cannot be decoded by this backend; use compatible backend
versions for "
+ "Variant block transfer"));
+ }
auto column_variant = assert_cast<ColumnVariant*>(column->get());
// 1. deserialize num of subcolumns
diff --git a/be/src/core/data_type/data_type_variant.h
b/be/src/core/data_type/data_type_variant.h
index 3f4e08a0ada..98e028ecf36 100644
--- a/be/src/core/data_type/data_type_variant.h
+++ b/be/src/core/data_type/data_type_variant.h
@@ -49,6 +49,7 @@ class DataTypeVariant : public IDataType {
private:
int32_t _max_subcolumns_count = 0;
bool _enable_doc_mode = false;
+ bool _has_doc_value_serialization = true;
std::string name = "Variant";
public:
@@ -56,7 +57,8 @@ public:
PrimitiveType get_primitive_type() const override { return
PrimitiveType::TYPE_VARIANT; }
DataTypeVariant() = default;
DataTypeVariant(int32_t max_subcolumns_count);
- DataTypeVariant(int32_t max_subcolumns_count, bool enable_doc_mode);
+ DataTypeVariant(int32_t max_subcolumns_count, bool enable_doc_mode,
+ bool has_doc_value_serialization = true);
String do_get_name() const override { return name; }
const std::string get_family_name() const override { return "Variant"; }
diff --git a/be/test/core/block/block_variant_wire_format_test.cpp
b/be/test/core/block/block_variant_wire_format_test.cpp
new file mode 100644
index 00000000000..11958f759eb
--- /dev/null
+++ b/be/test/core/block/block_variant_wire_format_test.cpp
@@ -0,0 +1,147 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#include <gen_cpp/data.pb.h>
+#include <gtest/gtest.h>
+
+#include <cstring>
+#include <memory>
+#include <string>
+
+#include "agent/be_exec_version_manager.h"
+#include "core/block/block.h"
+#include "core/column/column_variant.h"
+#include "core/data_type/data_type_array.h"
+#include "core/data_type/data_type_map.h"
+#include "core/data_type/data_type_nullable.h"
+#include "core/data_type/data_type_string.h"
+#include "core/data_type/data_type_struct.h"
+#include "core/data_type/data_type_variant.h"
+#include "core/data_type/data_type_variant_v2.h"
+
+namespace doris {
+namespace {
+PBlock serialize_wire(const Block& block, segment_v2::CompressionTypePB
compression) {
+ PBlock wire;
+ size_t uncompressed = 0;
+ size_t compressed = 0;
+ int64_t time = 0;
+ EXPECT_TRUE(block.serialize(BeExecVersionManager::get_newest_version(),
&wire, &uncompressed,
+ &compressed, &time, compression)
+ .ok());
+ return wire;
+}
+Status decode_wire(const PBlock& wire, Block& result) {
+ size_t bytes = 0;
+ int64_t time = 0;
+ return result.deserialize(wire, &bytes, &time);
+}
+void expect_legacy_rejected(const PBlock& wire) {
+ Block result;
+ const Status status = decode_wire(wire, result);
+ EXPECT_TRUE(status.is<ErrorCode::NOT_IMPLEMENTED_ERROR>()) << status;
+ EXPECT_NE(status.to_string().find("Unsupported VARIANT block format"),
std::string::npos);
+}
+} // namespace
+
+TEST(BlockVariantWireFormatTest, RejectsMissingDocValueFormatMarker) {
+ for (bool doc_mode : {false, true}) {
+ DataTypePtr type = std::make_shared<DataTypeVariant>(0, doc_mode);
+ auto column = type->create_column();
+ column->insert_default();
+ Block source({{std::move(column), type, "v"}});
+ for (auto compression : {segment_v2::NO_COMPRESSION,
segment_v2::SNAPPY}) {
+ PBlock wire = serialize_wire(source, compression);
+ ASSERT_TRUE(wire.column_metas(0).has_variant_enable_doc_mode());
+ wire.mutable_column_metas(0)->clear_variant_enable_doc_mode();
+ expect_legacy_rejected(wire);
+ }
+ }
+}
+
+TEST(BlockVariantWireFormatTest, RejectsActualPreDocValuePayload) {
+ // The pre-doc-value V1 wire layout has only subcolumn count, row count
and sparse Map.
+ const auto& map_type = ColumnVariant::get_binary_column_type();
+ const auto sparse = map_type->create_column();
+ const int version = BeExecVersionManager::get_newest_version();
+ std::string buffer(
+ 2 * sizeof(uint32_t) +
map_type->get_uncompressed_serialized_bytes(*sparse, version),
+ '\0');
+ char* end = map_type->serialize(*sparse, buffer.data() + 2 *
sizeof(uint32_t), version);
+ PBlock wire;
+ wire.set_be_exec_version(version);
+ auto* meta = wire.add_column_metas();
+ meta->set_name("legacy_v1");
+ meta->set_type(PGenericType::VARIANT);
+ // Copy just the actual old payload, without a serialized doc-value Map or
spare capacity.
+ wire.set_column_values(buffer.data(), end - buffer.data());
+ expect_legacy_rejected(wire);
+}
+
+TEST(BlockVariantWireFormatTest, RejectsLegacyV1InsideComplexTypes) {
+ const DataTypePtr variant = std::make_shared<DataTypeVariant>();
+ const DataTypePtr array =
std::make_shared<DataTypeArray>(make_nullable(variant));
+ const DataTypes types = {
+ std::make_shared<DataTypeArray>(array),
+ std::make_shared<DataTypeMap>(std::make_shared<DataTypeString>(),
array),
+ std::make_shared<DataTypeStruct>(DataTypes {array}, Strings
{"v"})};
+ for (const auto& type : types) {
+ auto column = type->create_column();
+ column->insert_default();
+ Block source({{std::move(column), type, "nested"}});
+ for (auto compression : {segment_v2::NO_COMPRESSION,
segment_v2::SNAPPY}) {
+ PBlock wire = serialize_wire(source, compression);
+ auto* root = wire.mutable_column_metas(0);
+ auto* child = root->mutable_children(root->type() ==
PGenericType::MAP ? 1 : 0);
+ child->mutable_children(0)->clear_variant_enable_doc_mode();
+ expect_legacy_rejected(wire);
+ }
+ }
+}
+
+TEST(BlockVariantWireFormatTest, RoundTripsSupportedV1AndV2) {
+ for (bool v2 : {false, true}) {
+ for (bool doc_mode : {false, true}) {
+ for (bool nullable : {false, true}) {
+ DataTypePtr type =
+ v2 ?
DataTypePtr(std::make_shared<DataTypeVariantV2>(0, doc_mode))
+ : DataTypePtr(std::make_shared<DataTypeVariant>(0,
doc_mode));
+ if (nullable) {
+ type = make_nullable(type);
+ }
+ auto column = type->create_column();
+ column->insert_default();
+ Block source({{std::move(column), type, "v"}});
+ for (auto compression : {segment_v2::NO_COMPRESSION,
segment_v2::SNAPPY}) {
+ PBlock wire = serialize_wire(source, compression);
+ if (v2) {
+ // V2 has its own marker and does not require a V1
doc-value marker.
+
wire.mutable_column_metas(0)->clear_variant_enable_doc_mode();
+ }
+ Block decoded;
+ ASSERT_TRUE(decode_wire(wire, decoded).ok());
+ ASSERT_EQ(decoded.rows(), source.rows());
+ // 4.1 does not implement compare_at for Variant columns,
so compare the
+ // re-serialized payload with the payload that was sent
instead.
+ PBlock reserialized = serialize_wire(decoded, compression);
+ EXPECT_EQ(reserialized.column_values(),
wire.column_values());
+ }
+ }
+ }
+ }
+}
+} // namespace doris
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
