This is an automated email from the ASF dual-hosted git repository.
eldenmoon pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/master by this push:
new d1d5ab33dbc [fix](variant) Keep integer element_at indexes out of
variant sub-column paths (#67955)
d1d5ab33dbc is described below
commit d1d5ab33dbc306b1df24f9af3a6d444d26a4d077
Author: lihangyu <[email protected]>
AuthorDate: Wed Sep 16 09:33:42 2026 +0800
[fix](variant) Keep integer element_at indexes out of variant sub-column
paths (#67955)
---
.../rules/rewrite/VariantSubPathPruning.java | 10 ++--
.../rules/rewrite/PruneNestedColumnTest.java | 17 ++++++
.../rules/rewrite/VariantPruningLogicTest.java | 4 +-
.../data/variant_p0/element_function.out | 34 +++++++++++
.../suites/variant_p0/element_function.groovy | 67 ++++++++++++++++++++++
5 files changed, 125 insertions(+), 7 deletions(-)
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/VariantSubPathPruning.java
b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/VariantSubPathPruning.java
index cbbd44baefc..3036a2ba4d9 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/VariantSubPathPruning.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/VariantSubPathPruning.java
@@ -32,7 +32,6 @@ import
org.apache.doris.nereids.trees.expressions.StatementScopeIdGenerator;
import org.apache.doris.nereids.trees.expressions.functions.ExpressionTrait;
import org.apache.doris.nereids.trees.expressions.functions.Function;
import org.apache.doris.nereids.trees.expressions.functions.scalar.ElementAt;
-import org.apache.doris.nereids.trees.expressions.literal.Literal;
import org.apache.doris.nereids.trees.expressions.literal.StringLikeLiteral;
import org.apache.doris.nereids.trees.expressions.literal.VarcharLiteral;
import
org.apache.doris.nereids.trees.expressions.visitor.DefaultExpressionVisitor;
@@ -792,14 +791,13 @@ public class VariantSubPathPruning implements
CustomRewriter {
if (!(elementAt.left() instanceof ElementAt || elementAt.left()
instanceof SlotReference)) {
return null;
}
+ // Storage sub-paths address object keys only. An integer index
selects an array element of the
+ // VARIANT value, so the sub-path stops before it and the index is
applied to the extracted value.
Expression key = elementAt.right();
- if (key instanceof StringLikeLiteral) {
- subPath.add(((StringLikeLiteral) key).getStringValue());
- } else if (key instanceof Literal &&
key.getDataType().isIntegerLikeType()) {
- subPath.add(((Literal) key).getStringValue());
- } else {
+ if (!(key instanceof StringLikeLiteral)) {
return null;
}
+ subPath.add(((StringLikeLiteral) key).getStringValue());
if (elementAt.left() instanceof SlotReference) {
// ElementAt's left child is SlotReference
// reverse subPath because we put them by reverse order
diff --git
a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/PruneNestedColumnTest.java
b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/PruneNestedColumnTest.java
index a3e82f92d0c..6814b523a43 100644
---
a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/PruneNestedColumnTest.java
+++
b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/PruneNestedColumnTest.java
@@ -356,6 +356,23 @@ public class PruneNestedColumnTest extends
TestWithFeService implements MemoPatt
));
}
+ @Test
+ public void testVariantIntegerIndexStopsSubColumnPath() throws Exception {
+ // DORIS-28435: an integer index selects an array element, so only the
object-key prefix becomes a storage
+ // sub-column; a sub-column items.1 would look up a missing object key
and return NULL.
+ assertVariantSubColumnSlots("select element_at(element_at(v, 'items'),
1), v['items'][-1] from variant_tbl",
+ ImmutableList.of(ImmutableList.of("items")));
+ }
+
+ @Test
+ public void testVariantIntegerIndexPredicateStopsSubColumnPath() throws
Exception {
+ // Filters stop at the integer index too, also when a string key
follows it; the object-key sub-column
+ // items.1, spelled like the index, stays a separate slot.
+ assertVariantSubColumnSlots("select id from variant_tbl"
+ + " where v['items'][1] = 2 and v['items'][-1]['k'] =
1 and v['items']['1'] = 'x'",
+ ImmutableList.of(ImmutableList.of("items"),
ImmutableList.of("items", "1")));
+ }
+
@Test
public void testVariantPredicateAccessPath() throws Exception {
assertColumn("select 1 from variant_tbl where v['k'] is not null",
diff --git
a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/VariantPruningLogicTest.java
b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/VariantPruningLogicTest.java
index a29988dc2a9..c6720d6d3a6 100644
---
a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/VariantPruningLogicTest.java
+++
b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/VariantPruningLogicTest.java
@@ -59,10 +59,12 @@ public class VariantPruningLogicTest extends
TestWithFeService {
@Test
public void testVariantNumericIndexSubPath() throws Exception {
+ // A numeric index selects an array element, not an object key, so the
storage sub-column stops at arr and
+ // [0]['x'] is applied to it at runtime; the access path still names
the whole expression.
assertVariantSubColumnSlots(
"select v['arr'][0]['x'] from variant_tbl",
ImmutableList.of(
- ImmutableList.of("arr", "0", "x")
+ ImmutableList.of("arr")
)
);
assertAllAccessPathsContain(
diff --git a/regression-test/data/variant_p0/element_function.out
b/regression-test/data/variant_p0/element_function.out
index 095c7b20356..007761a663b 100644
--- a/regression-test/data/variant_p0/element_function.out
+++ b/regression-test/data/variant_p0/element_function.out
@@ -2,3 +2,37 @@
-- !sql --
1
+-- !nested_integer_index --
+1 2 4 \N \N 3 \N
+2 [5,6] 7 \N \N s \N
+3 \N \N \N \N \N key one
+4 \N \N \N \N \N \N
+5 {"k":1} {"k":2} \N \N {"k":2} \N
+6 2 4 \N \N null \N
+
+-- !nested_integer_index_const --
+2 4
+
+-- !nested_integer_index_filter --
+1
+6
+
+-- !nested_integer_index_filter_negative --
+1 3
+6 null
+
+-- !nested_integer_index_filter_null --
+3
+4
+
+-- !nested_integer_index_filter_object --
+5
+
+-- !nested_integer_index_filter_with_key --
+1
+3
+6
+
+-- !nested_integer_index_filter_topn --
+1 [2,3,4]
+
diff --git a/regression-test/suites/variant_p0/element_function.groovy
b/regression-test/suites/variant_p0/element_function.groovy
index a90e655bd7f..4574bbe5eae 100644
--- a/regression-test/suites/variant_p0/element_function.groovy
+++ b/regression-test/suites/variant_p0/element_function.groovy
@@ -59,4 +59,71 @@ suite("regression_test_variant_element_at", "p0") {
def obj = sql """select sort_json_object_keys(json_extract(
cast(${variantV2Function}('{"o":{"name":"john"}}') as json),
'\$.o'))"""
assertEquals('{"name":"john"}', obj[0][0])
+
+ // DORIS-28435: an integer index on a VARIANT array that was itself
extracted with element_at is 1-based like
+ // ARRAY element_at and counts from the end when negative; 0, out-of-range
indexes, non-array values and
+ // missing paths give NULL, while a string index still reads object keys.
On a stored column the planner
+ // must not turn the integer index into the storage sub-path items.1.
+ sql "DROP TABLE IF EXISTS element_at_nested_index_test"
+ sql """
+ CREATE TABLE element_at_nested_index_test (
+ id INT,
+ json_variant VARIANT
+ )
+ DUPLICATE KEY(id)
+ DISTRIBUTED BY HASH(id) BUCKETS 1
+ PROPERTIES ("replication_num" = "1")
+ """
+ sql """INSERT INTO element_at_nested_index_test VALUES
+ (1, ${variantV2Function}('{"items": [2, 3, 4]}')),
+ (2, ${variantV2Function}('{"items": [[5, 6], "s", 7]}')),
+ (3, ${variantV2Function}('{"items": {"1": "key one"}}')),
+ (4, ${variantV2Function}('{"other": 1}')),
+ (5, ${variantV2Function}('{"items": [{"k": 1}, {"k": 2}]}')),
+ (6, ${variantV2Function}('{"items": [2, null, 4]}'))"""
+ order_qt_nested_integer_index """
+ SELECT id,
+ element_at(element_at(json_variant, 'items'), 1),
+ element_at(element_at(json_variant, 'items'), -1),
+ element_at(element_at(json_variant, 'items'), 0),
+ element_at(element_at(json_variant, 'items'), 4),
+ json_variant['items'][2],
+ element_at(element_at(json_variant, 'items'), '1')
+ FROM element_at_nested_index_test
+ """
+ qt_nested_integer_index_const """
+ SELECT element_at(element_at(${variantV2Function}('{"items": [2, 3,
4]}'), 'items'), 1),
+ element_at(element_at(${variantV2Function}('{"items": [2, 3,
4]}'), 'items'), -1)
+ """
+
+ // Filters read items and apply the index the same way: as a pushed-down
predicate, after a string key that
+ // follows the index, next to the object-key sub-column items.1 in one
filter, and under a TopN.
+ order_qt_nested_integer_index_filter """
+ SELECT id FROM element_at_nested_index_test
+ WHERE CAST(element_at(element_at(json_variant, 'items'), 1) AS INT) = 2
+ """
+ order_qt_nested_integer_index_filter_negative """
+ SELECT id, json_variant['items'][2] FROM element_at_nested_index_test
+ WHERE CAST(json_variant['items'][-1] AS INT) = 4
+ """
+ order_qt_nested_integer_index_filter_null """
+ SELECT id FROM element_at_nested_index_test
+ WHERE element_at(element_at(json_variant, 'items'), 1) IS NULL
+ """
+ order_qt_nested_integer_index_filter_object """
+ SELECT id FROM element_at_nested_index_test WHERE
CAST(json_variant['items'][1]['k'] AS INT) = 1
+ """
+ order_qt_nested_integer_index_filter_with_key """
+ SELECT id FROM element_at_nested_index_test
+ WHERE CAST(json_variant['items']['1'] AS STRING) = 'key one' OR
CAST(json_variant['items'][1] AS INT) = 2
+ """
+ qt_nested_integer_index_filter_topn """
+ SELECT id, CAST(json_variant['items'] AS STRING) FROM
element_at_nested_index_test
+ WHERE CAST(json_variant['items'][1] AS INT) = 2 ORDER BY id LIMIT 1
+ """
+ // MATCH needs a storage column, and a path with an integer index is not
one.
+ test {
+ sql "SELECT id FROM element_at_nested_index_test WHERE
json_variant['items'][2] MATCH_ANY 's'"
+ exception "Only support match left operand is SlotRef"
+ }
}
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
