This is an automated email from the ASF dual-hosted git repository.
HappenLee pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/master by this push:
new 231b5345bab [fix](be) Skip condition cache for complex object literals
(#68930)
231b5345bab is described below
commit 231b5345bab954fa9d937ab3b8dda5b34ac454cb
Author: HappenLee <[email protected]>
AuthorDate: Thu Oct 15 11:03:07 2026 +0800
[fix](be) Skip condition cache for complex object literals (#68930)
Problem Summary: With the condition cache enabled, valid scan predicates
containing unsupported literal columns can fail while preparing their
digest. For example, `bitmap_count(IF(id = 1, to_bitmap(1), CAST(NULL AS
BITMAP))) > 0` reaches a column xxHash method that is not implemented.
Checking only the top-level SQL type also misses VARBINARY, nested
containers, and AGG_STATE values serialized as fixed-length object
columns.
Inspect the literal's actual column and recursively visit its subcolumns
using the existing read-only traversal interface. Return the existing
unsupported-digest sentinel (0) for bitmap, HLL, quantile state,
VARBINARY, and fixed-length object columns, including through constant,
nullable, array, map, and struct wrappers. Parent expressions propagate
zero and the affected scan evaluates its predicates without the
condition cache. Ordinary scalar/container literals and AGG_STATE values
serialized as hashable strings retain their existing digest behavior.
Keep the general column hashing contract unchanged: it also serves
aggregation and does not define zero as an unsupported value. The extra
check walks the literal's column structure rather than table rows.
### Release note
Fix valid queries containing unsupported object, binary,
aggregate-state, or nested literals in scan predicates failing when the
condition cache is enabled. Such scans now bypass the condition cache.
### Check List (For Author)
- Test:
- Unit Test: Add typed NULL and non-NULL object/VARBINARY literals,
parent digest propagation, nested Array/Map/Struct coverage, supported
and unsupported AGG_STATE serialization, and scalar/container digest
controls.
- Regression test: Compare six ordered queries with condition cache
disabled and across repeated enabled execution and cache transitions.
Include IF/CASE, VARBINARY, folded object arrays with row-dependent
indexing, and supported array/projection controls.
- Local validation: clang-format 16, build hygiene, git diff --check,
and Groovy syntax compilation passed.
- clang-tidy passed the repository's changed-line check using the
generated UT compilation flags with the missing PCH removed and the
toolchain resource directory supplied. The production file also has no
full-file diagnostics. Full-file test analysis reports an existing
constructor virtual-call diagnostic in unchanged
data_type_agg_state.h:66.
- BE UT execution was attempted with run-be-ut.sh -j48 --run
--filter='VLiteralDigestTest.*', but Ninja stopped on the missing local
third-party archive libsimdutf.a. The SQL regression was not run locally
because this worktree has no runnable build of the change. No runtime
pass is claimed; CI validation is required before merging.
- Behavior changed: Yes. Unsupported literal digests disable the
affected scan's condition cache instead of failing the query.
- Does this need documentation: No.
---
be/src/exprs/vliteral.cpp | 28 ++++
be/test/exprs/vliteral_digest_test.cpp | 159 +++++++++++++++++++++
.../test_condition_cache_complex_literal.groovy | 74 ++++++++++
3 files changed, 261 insertions(+)
diff --git a/be/src/exprs/vliteral.cpp b/be/src/exprs/vliteral.cpp
index 9b93d709727..ee8f7e7641e 100644
--- a/be/src/exprs/vliteral.cpp
+++ b/be/src/exprs/vliteral.cpp
@@ -28,6 +28,9 @@
#include "core/block/block.h"
#include "core/column/column.h"
+#include "core/column/column_complex.h"
+#include "core/column/column_fixed_length_object.h"
+#include "core/column/column_varbinary.h"
#include "core/data_type/data_type_decimal.h"
#include "core/field.h"
#include "core/types.h"
@@ -35,6 +38,26 @@
namespace doris {
+namespace {
+
+bool supports_literal_digest(const IColumn& column) {
+ // These concrete columns inherit IColumn's throwing xxHash implementation.
+ // Inspect the column rather than the SQL type: AGG_STATE can use either a
+ // hashable string column or an unsupported fixed-length object column.
+ if (is_column<ColumnBitmap>(column) || is_column<ColumnHLL>(column) ||
+ is_column<ColumnQuantileState>(column) ||
is_column<ColumnVarbinary>(column) ||
+ is_column<ColumnFixedLengthObject>(column)) {
+ return false;
+ }
+
+ bool supported = true;
+ column.for_each_subcolumn(
+ [&](const IColumn& child) { supported = supported &&
supports_literal_digest(child); });
+ return supported;
+}
+
+} // namespace
+
class VExprContext;
Status VLiteral::prepare(RuntimeState* state, const RowDescriptor& desc,
VExprContext* context) {
@@ -100,6 +123,11 @@ bool VLiteral::equals(const VExpr& other) {
}
uint64_t VLiteral::get_digest(uint64_t seed) const {
+ // Wrappers and containers delegate hashing to their nested columns. A zero
+ // digest disables the condition cache without changing the column hash
contract.
+ if (!supports_literal_digest(*_column_ptr)) {
+ return 0;
+ }
_column_ptr->update_xxHash_with_value(0, 1, seed, nullptr);
return seed;
}
diff --git a/be/test/exprs/vliteral_digest_test.cpp
b/be/test/exprs/vliteral_digest_test.cpp
new file mode 100644
index 00000000000..d520cbf23aa
--- /dev/null
+++ b/be/test/exprs/vliteral_digest_test.cpp
@@ -0,0 +1,159 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#include <gtest/gtest.h>
+
+#include <utility>
+#include <vector>
+
+#include "agent/be_exec_version_manager.h"
+#include "core/data_type/data_type_agg_state.h"
+#include "core/data_type/data_type_array.h"
+#include "core/data_type/data_type_factory.hpp"
+#include "core/data_type/data_type_map.h"
+#include "core/data_type/data_type_nullable.h"
+#include "core/data_type/data_type_number.h"
+#include "core/data_type/data_type_string.h"
+#include "core/data_type/data_type_struct.h"
+#include "exprs/vliteral.h"
+#include "testutil/mock/mock_fn_call.h"
+
+namespace doris {
+
+TEST(VLiteralDigestTest, ComplexNullDisablesParentDigest) {
+ for (auto type : {TYPE_BITMAP, TYPE_HLL, TYPE_QUANTILE_STATE,
TYPE_VARBINARY}) {
+ SCOPED_TRACE(type_to_string(type));
+ auto data_type = DataTypeFactory::instance().create_data_type(type,
true);
+ auto literal = VLiteral::create_shared(data_type, Field());
+ EXPECT_EQ(literal->get_digest(42), 0);
+
+ // A later scalar child must not revive a digest disabled by the
complex literal.
+ auto parent = MockFnCall::create("if");
+ parent->add_child(literal);
+
parent->add_child(VLiteral::create_shared(std::make_shared<DataTypeInt32>(),
+
Field::create_field<TYPE_INT>(1)));
+ EXPECT_EQ(parent->get_digest(42), 0);
+ }
+}
+
+TEST(VLiteralDigestTest, NonNullComplexValuesDisableDigest) {
+ auto bitmap = VLiteral::create_shared(
+ DataTypeFactory::instance().create_data_type(TYPE_BITMAP, false),
+ Field::create_field<TYPE_BITMAP>(BitmapValue(1)));
+ auto hll =
+
VLiteral::create_shared(DataTypeFactory::instance().create_data_type(TYPE_HLL,
false),
+
Field::create_field<TYPE_HLL>(HyperLogLog()));
+ auto quantile = VLiteral::create_shared(
+ DataTypeFactory::instance().create_data_type(TYPE_QUANTILE_STATE,
false),
+ Field::create_field<TYPE_QUANTILE_STATE>(QuantileState()));
+ EXPECT_EQ(bitmap->get_digest(42), 0);
+ EXPECT_EQ(hll->get_digest(42), 0);
+ EXPECT_EQ(quantile->get_digest(42), 0);
+
+ auto binary = VLiteral::create_shared(
+ DataTypeFactory::instance().create_data_type(TYPE_VARBINARY,
false),
+ Field::create_field<TYPE_VARBINARY>(StringView("a", 1)));
+ EXPECT_EQ(binary->get_digest(42), 0);
+}
+
+TEST(VLiteralDigestTest, NestedUnsupportedColumnsDisableDigest) {
+ auto bitmap_type =
DataTypeFactory::instance().create_data_type(TYPE_BITMAP, true);
+ auto int_type = std::make_shared<DataTypeInt32>();
+ auto array_type = std::make_shared<DataTypeArray>(bitmap_type);
+ auto array = Field::create_field<TYPE_ARRAY>(
+ Array {Field::create_field<TYPE_BITMAP>(BitmapValue(1)), Field()});
+ auto nested_array_type = std::make_shared<DataTypeArray>(array_type);
+ auto nested_array = Field::create_field<TYPE_ARRAY>(Array {array});
+ auto map_type = std::make_shared<DataTypeMap>(int_type, array_type);
+ auto map = Field::create_field<TYPE_MAP>(
+ Map {Field::create_field<TYPE_ARRAY>(Array
{Field::create_field<TYPE_INT>(1)}),
+ Field::create_field<TYPE_ARRAY>(Array {array})});
+ auto struct_type = std::make_shared<DataTypeStruct>(DataTypes {int_type,
map_type});
+ auto structure =
+ Field::create_field<TYPE_STRUCT>(Struct
{Field::create_field<TYPE_INT>(1), map});
+
+ for (const auto& [type, value] :
+ std::vector<std::pair<DataTypePtr, Field>> {{array_type, array},
+ {nested_array_type,
nested_array},
+ {map_type, map},
+ {struct_type, structure},
+
{make_nullable(array_type), Field()}}) {
+ SCOPED_TRACE(type->get_name());
+ auto literal = VLiteral::create_shared(type, value);
+ EXPECT_EQ(literal->get_digest(42), 0);
+ }
+}
+
+TEST(VLiteralDigestTest, AggStateDigestUsesSerializedColumn) {
+ const auto version = BeExecVersionManager::get_newest_version();
+ auto sum_type = std::make_shared<DataTypeAggState>(
+ DataTypes {std::make_shared<DataTypeInt32>()}, false, "sum",
version);
+ auto sum_null = VLiteral::create_shared(make_nullable(sum_type), Field());
+ EXPECT_EQ(sum_null->get_digest(42), 0);
+
+ // HLL aggregate states serialize as ColumnString, unlike direct HLL
literals.
+ auto hll_type = DataTypeFactory::instance().create_data_type(TYPE_HLL,
false);
+ auto hll_state_type =
+ std::make_shared<DataTypeAggState>(DataTypes {hll_type}, false,
"hll_union", version);
+ auto hll_state_null =
VLiteral::create_shared(make_nullable(hll_state_type), Field());
+ EXPECT_NE(hll_state_null->get_digest(42), 0);
+ EXPECT_EQ(hll_state_null->get_digest(42), hll_state_null->get_digest(42));
+}
+
+TEST(VLiteralDigestTest, SupportedContainersKeepDistinctDigests) {
+ auto int_type = std::make_shared<DataTypeInt32>();
+ auto array_type = std::make_shared<DataTypeArray>(int_type);
+ auto one = Field::create_field<TYPE_INT>(1);
+ auto two = Field::create_field<TYPE_INT>(2);
+ auto first = VLiteral::create_shared(array_type,
Field::create_field<TYPE_ARRAY>(Array {one}));
+ auto second = VLiteral::create_shared(array_type,
Field::create_field<TYPE_ARRAY>(Array {two}));
+ EXPECT_NE(first->get_digest(42), 0);
+ EXPECT_NE(first->get_digest(42), second->get_digest(42));
+ EXPECT_EQ(first->get_digest(42), first->get_digest(42));
+
+ auto map_type = std::make_shared<DataTypeMap>(int_type, int_type);
+ auto map = Field::create_field<TYPE_MAP>(Map
{Field::create_field<TYPE_ARRAY>(Array {one}),
+
Field::create_field<TYPE_ARRAY>(Array {two})});
+ auto map_literal = VLiteral::create_shared(map_type, map);
+ EXPECT_NE(map_literal->get_digest(42), 0);
+ auto struct_type = std::make_shared<DataTypeStruct>(DataTypes {int_type,
map_type});
+ auto struct_literal = VLiteral::create_shared(
+ struct_type, Field::create_field<TYPE_STRUCT>(Struct {one, map}));
+ EXPECT_NE(struct_literal->get_digest(42), 0);
+}
+
+TEST(VLiteralDigestTest, ScalarValuesKeepDistinctStableDigests) {
+ auto int_type = std::make_shared<DataTypeInt32>();
+ auto one = VLiteral::create_shared(int_type,
Field::create_field<TYPE_INT>(1));
+ auto two = VLiteral::create_shared(int_type,
Field::create_field<TYPE_INT>(2));
+ EXPECT_NE(one->get_digest(42), 0);
+ EXPECT_EQ(one->get_digest(42), one->get_digest(42));
+ EXPECT_NE(one->get_digest(42), two->get_digest(42));
+ EXPECT_NE(one->get_digest(42), one->get_digest(43));
+
+ auto string_type = std::make_shared<DataTypeString>();
+ auto a = VLiteral::create_shared(string_type,
Field::create_field<TYPE_STRING>(String("a")));
+ auto b = VLiteral::create_shared(string_type,
Field::create_field<TYPE_STRING>(String("b")));
+ EXPECT_NE(a->get_digest(42), 0);
+ EXPECT_NE(a->get_digest(42), b->get_digest(42));
+
+ auto scalar_null = VLiteral::create_shared(
+ DataTypeFactory::instance().create_data_type(TYPE_INT, true),
Field());
+ EXPECT_NE(scalar_null->get_digest(42), 0);
+}
+
+} // namespace doris
diff --git
a/regression-test/suites/query_p0/condition_cache/test_condition_cache_complex_literal.groovy
b/regression-test/suites/query_p0/condition_cache/test_condition_cache_complex_literal.groovy
new file mode 100644
index 00000000000..269adb82cc0
--- /dev/null
+++
b/regression-test/suites/query_p0/condition_cache/test_condition_cache_complex_literal.groovy
@@ -0,0 +1,74 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+suite("test_condition_cache_complex_literal") {
+ sql "set enable_sql_cache=false"
+ sql "drop table if exists condition_cache_complex_literal"
+ sql """
+ create table condition_cache_complex_literal (id int)
+ duplicate key(id)
+ distributed by hash(id) buckets 1
+ properties("replication_num"="1")
+ """
+ sql "insert into condition_cache_complex_literal values (1), (2)"
+
+ def queries = [
+ """
+ select id from condition_cache_complex_literal
+ where bitmap_count(if(id=1, to_bitmap(1), cast(null as bitmap))) > 0
+ order by id
+ """,
+ """
+ select id from condition_cache_complex_literal
+ where bitmap_count(case when id=1 then to_bitmap(1)
+ else cast(null as bitmap) end) > 0
+ order by id
+ """,
+ """
+ select id from condition_cache_complex_literal
+ where md5sum(X'61', to_binary(cast(id as string))) <> ''
+ order by id
+ """,
+ // FE folds this array into a literal, while the row-dependent index
+ // keeps it in the scan predicate. Even its NULL bitmap element lacks
xxHash.
+ """
+ select id from condition_cache_complex_literal
+ where bitmap_count(element_at(array(cast(null as bitmap)), id)) = 0
+ order by id
+ """,
+ """
+ select id from condition_cache_complex_literal
+ where element_at(array(1, 2), id) = id
+ order by id
+ """,
+ """
+ select id, bitmap_count(if(id=1, to_bitmap(1), cast(null as
bitmap)))
+ from condition_cache_complex_literal order by id
+ """
+ ]
+
+ // Compare each query with uncached execution to verify that cache
transitions
+ // and repeated enabled execution preserve the row results.
+ sql "set enable_condition_cache=false"
+ def uncachedResults = queries.collect { query -> sql(query) }
+ for (boolean enabled : [true, true, false, true]) {
+ sql "set enable_condition_cache=${enabled}"
+ queries.eachWithIndex { query, index ->
+ assertEquals(uncachedResults[index], sql(query))
+ }
+ }
+}
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
