This is an automated email from the ASF dual-hosted git repository.
morrysnow pushed a commit to branch branch-2.1
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/branch-2.1 by this push:
new 8b35b0e4774 branch-2.1: [fix](function) fixed some nested type func's
param type which is not suitable and make result wrong #44923 (#45798)
8b35b0e4774 is described below
commit 8b35b0e4774697d2a6b2047dec3cb47d7524ba4e
Author: amory <[email protected]>
AuthorDate: Tue Dec 24 14:57:33 2024 +0800
branch-2.1: [fix](function) fixed some nested type func's param type which
is not suitable and make result wrong #44923 (#45798)
Cherry-picked from #44923
---
.../expressions/functions/scalar/ArrayApply.java | 16 ++++++++--
.../functions/scalar/ArrayContains.java | 15 ++++++++--
.../functions/scalar/ArrayPosition.java | 16 ++++++++--
.../functions/scalar/ArrayPushBack.java | 16 ++++++++--
.../functions/scalar/ArrayPushFront.java | 16 ++++++++--
.../expressions/functions/scalar/ArrayRemove.java | 16 ++++++++--
.../expressions/functions/scalar/CountEqual.java | 19 ++++++++++--
.../functions/scalar/MapContainsKey.java | 17 +++++++++--
.../functions/scalar/MapContainsValue.java | 17 +++++++++--
.../org/apache/doris/nereids/types/DataType.java | 35 ++++++++++++++++++++++
.../nereids_function_p0/scalar_function/Array.out | 27 +++++++++++++++++
.../scalar_function/Array.groovy | 16 ++++++++++
12 files changed, 208 insertions(+), 18 deletions(-)
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/ArrayApply.java
b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/ArrayApply.java
index 07e4c16d776..55e5fe3b1d2 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/ArrayApply.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/ArrayApply.java
@@ -41,11 +41,16 @@ import java.util.List;
*/
public class ArrayApply extends ScalarFunction
implements BinaryExpression, ExplicitlyCastableSignature,
PropagateNullable {
- public static final List<FunctionSignature> SIGNATURES = ImmutableList.of(
+ public static final List<FunctionSignature> FOLLOW_DATATYPE_SIGNATURE =
ImmutableList.of(
FunctionSignature.retArgType(0)
.args(ArrayType.of(new AnyDataType(0)),
VarcharType.SYSTEM_DEFAULT,
new FollowToAnyDataType(0)));
+ public static final List<FunctionSignature> MIN_COMMON_TYPE_SIGNATURES =
ImmutableList.of(
+ FunctionSignature.retArgType(0)
+ .args(ArrayType.of(new AnyDataType(0)),
VarcharType.SYSTEM_DEFAULT,
+ new AnyDataType(0)));
+
/**
* constructor
*/
@@ -93,6 +98,13 @@ public class ArrayApply extends ScalarFunction
@Override
public List<FunctionSignature> getSignatures() {
- return SIGNATURES;
+ if (getArgument(0).getDataType().isArrayType()
+ &&
+ ((ArrayType) getArgument(0).getDataType()).getItemType()
+
.isSameTypeForComplexTypeParam(getArgument(2).getDataType())) {
+ // return least common type
+ return MIN_COMMON_TYPE_SIGNATURES;
+ }
+ return FOLLOW_DATATYPE_SIGNATURE;
}
}
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/ArrayContains.java
b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/ArrayContains.java
index 494a4a863c5..4eee7066584 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/ArrayContains.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/ArrayContains.java
@@ -38,10 +38,14 @@ import java.util.List;
public class ArrayContains extends ScalarFunction
implements BinaryExpression, ExplicitlyCastableSignature {
- public static final List<FunctionSignature> SIGNATURES = ImmutableList.of(
+ public static final List<FunctionSignature> FOLLOW_DATATYPE_SIGNATURE =
ImmutableList.of(
FunctionSignature.ret(BooleanType.INSTANCE)
.args(ArrayType.of(new AnyDataType(0)), new
FollowToAnyDataType(0))
);
+ public static final List<FunctionSignature> MIN_COMMON_TYPE_SIGNATURES =
ImmutableList.of(
+ FunctionSignature.ret(BooleanType.INSTANCE)
+ .args(ArrayType.of(new AnyDataType(0)), new AnyDataType(0))
+ );
/**
* constructor with 2 arguments.
@@ -71,6 +75,13 @@ public class ArrayContains extends ScalarFunction
@Override
public List<FunctionSignature> getSignatures() {
- return SIGNATURES;
+ if (getArgument(0).getDataType().isArrayType()
+ &&
+ ((ArrayType) getArgument(0).getDataType()).getItemType()
+
.isSameTypeForComplexTypeParam(getArgument(1).getDataType())) {
+ // return least common type
+ return MIN_COMMON_TYPE_SIGNATURES;
+ }
+ return FOLLOW_DATATYPE_SIGNATURE;
}
}
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/ArrayPosition.java
b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/ArrayPosition.java
index 3a94e84d511..b782a560719 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/ArrayPosition.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/ArrayPosition.java
@@ -38,11 +38,16 @@ import java.util.List;
public class ArrayPosition extends ScalarFunction
implements BinaryExpression, ExplicitlyCastableSignature {
- public static final List<FunctionSignature> SIGNATURES = ImmutableList.of(
+ public static final List<FunctionSignature> FOLLOW_DATATYPE_SIGNATURE =
ImmutableList.of(
FunctionSignature.ret(BigIntType.INSTANCE)
.args(ArrayType.of(new AnyDataType(0)), new
FollowToAnyDataType(0))
);
+ public static final List<FunctionSignature> MIN_COMMON_TYPE_SIGNATURES =
ImmutableList.of(
+ FunctionSignature.ret(BigIntType.INSTANCE)
+ .args(ArrayType.of(new AnyDataType(0)), new AnyDataType(0))
+ );
+
/**
* constructor with 2 arguments.
*/
@@ -71,6 +76,13 @@ public class ArrayPosition extends ScalarFunction
@Override
public List<FunctionSignature> getSignatures() {
- return SIGNATURES;
+ if (getArgument(0).getDataType().isArrayType()
+ &&
+ ((ArrayType) getArgument(0).getDataType()).getItemType()
+
.isSameTypeForComplexTypeParam(getArgument(1).getDataType())) {
+ // return least common type
+ return MIN_COMMON_TYPE_SIGNATURES;
+ }
+ return FOLLOW_DATATYPE_SIGNATURE;
}
}
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/ArrayPushBack.java
b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/ArrayPushBack.java
index 932c2446132..a96c48c4b27 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/ArrayPushBack.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/ArrayPushBack.java
@@ -38,11 +38,16 @@ import java.util.List;
public class ArrayPushBack extends ScalarFunction
implements UnaryExpression, ExplicitlyCastableSignature,
AlwaysNullable {
- public static final List<FunctionSignature> SIGNATURES = ImmutableList.of(
+ public static final List<FunctionSignature> FOLLOW_DATATYPE_SIGNATURE =
ImmutableList.of(
FunctionSignature.retArgType(0)
.args(ArrayType.of(new AnyDataType(0)), new
FollowToAnyDataType(0))
);
+ public static final List<FunctionSignature> MIN_COMMON_TYPE_SIGNATURES =
ImmutableList.of(
+ FunctionSignature.retArgType(0)
+ .args(ArrayType.of(new AnyDataType(0)), new AnyDataType(0))
+ );
+
/**
* constructor with 1 argument.
*/
@@ -66,6 +71,13 @@ public class ArrayPushBack extends ScalarFunction
@Override
public List<FunctionSignature> getSignatures() {
- return SIGNATURES;
+ if (getArgument(0).getDataType().isArrayType()
+ &&
+ ((ArrayType) getArgument(0).getDataType()).getItemType()
+
.isSameTypeForComplexTypeParam(getArgument(1).getDataType())) {
+ // return least common type
+ return MIN_COMMON_TYPE_SIGNATURES;
+ }
+ return FOLLOW_DATATYPE_SIGNATURE;
}
}
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/ArrayPushFront.java
b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/ArrayPushFront.java
index 26e1cdd91e3..e20fea5cc20 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/ArrayPushFront.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/ArrayPushFront.java
@@ -38,11 +38,16 @@ import java.util.List;
public class ArrayPushFront extends ScalarFunction
implements UnaryExpression, ExplicitlyCastableSignature,
AlwaysNullable {
- public static final List<FunctionSignature> SIGNATURES = ImmutableList.of(
+ public static final List<FunctionSignature> FOLLOW_DATATYPE_SIGNATURE =
ImmutableList.of(
FunctionSignature.retArgType(0)
.args(ArrayType.of(new AnyDataType(0)), new
FollowToAnyDataType(0))
);
+ public static final List<FunctionSignature> MIN_COMMON_TYPE_SIGNATURES =
ImmutableList.of(
+ FunctionSignature.retArgType(0)
+ .args(ArrayType.of(new AnyDataType(0)), new AnyDataType(0))
+ );
+
/**
* constructor with 1 argument.
*/
@@ -66,6 +71,13 @@ public class ArrayPushFront extends ScalarFunction
@Override
public List<FunctionSignature> getSignatures() {
- return SIGNATURES;
+ if (getArgument(0).getDataType().isArrayType()
+ &&
+ ((ArrayType) getArgument(0).getDataType()).getItemType()
+
.isSameTypeForComplexTypeParam(getArgument(1).getDataType())) {
+ // return least common type
+ return MIN_COMMON_TYPE_SIGNATURES;
+ }
+ return FOLLOW_DATATYPE_SIGNATURE;
}
}
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/ArrayRemove.java
b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/ArrayRemove.java
index acf731b8bb4..f5996776867 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/ArrayRemove.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/ArrayRemove.java
@@ -38,11 +38,16 @@ import java.util.List;
public class ArrayRemove extends ScalarFunction
implements BinaryExpression, ExplicitlyCastableSignature,
PropagateNullable {
- public static final List<FunctionSignature> SIGNATURES = ImmutableList.of(
+ public static final List<FunctionSignature> FOLLOW_DATATYPE_SIGNATURE =
ImmutableList.of(
FunctionSignature.retArgType(0).args(
ArrayType.of(new AnyDataType(0)), new
FollowToAnyDataType(0))
);
+ public static final List<FunctionSignature> MIN_COMMON_TYPE_SIGNATURES =
ImmutableList.of(
+ FunctionSignature.retArgType(0).args(
+ ArrayType.of(new AnyDataType(0)), new AnyDataType(0))
+ );
+
/**
* constructor with 2 arguments.
*/
@@ -66,6 +71,13 @@ public class ArrayRemove extends ScalarFunction
@Override
public List<FunctionSignature> getSignatures() {
- return SIGNATURES;
+ if (getArgument(0).getDataType().isArrayType()
+ &&
+ ((ArrayType) getArgument(0).getDataType()).getItemType()
+
.isSameTypeForComplexTypeParam(getArgument(1).getDataType())) {
+ // return least common type
+ return MIN_COMMON_TYPE_SIGNATURES;
+ }
+ return FOLLOW_DATATYPE_SIGNATURE;
}
}
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/CountEqual.java
b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/CountEqual.java
index 2ba0c907243..bb76aecde31 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/CountEqual.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/CountEqual.java
@@ -38,11 +38,16 @@ import java.util.List;
public class CountEqual extends ScalarFunction
implements BinaryExpression, ExplicitlyCastableSignature {
- public static final List<FunctionSignature> SIGNATURES = ImmutableList.of(
+ public static final List<FunctionSignature> FOLLOW_DATATYPE_SIGNATURE =
ImmutableList.of(
FunctionSignature.ret(BigIntType.INSTANCE)
.args(ArrayType.of(new AnyDataType(0)), new
FollowToAnyDataType(0))
);
+ public static final List<FunctionSignature> MIN_COMMON_TYPE_SIGNATURES =
ImmutableList.of(
+ FunctionSignature.ret(BigIntType.INSTANCE)
+ .args(ArrayType.of(new AnyDataType(0)), new AnyDataType(0))
+ );
+
/**
* constructor with 2 arguments.
*/
@@ -71,6 +76,16 @@ public class CountEqual extends ScalarFunction
@Override
public List<FunctionSignature> getSignatures() {
- return SIGNATURES;
+ // to find out element type in array vs param type,
+ // if they are different, return first array element type,
+ // else return least common type between element type and param
+ if (getArgument(0).getDataType().isArrayType()
+ &&
+ ((ArrayType) getArgument(0).getDataType()).getItemType()
+
.isSameTypeForComplexTypeParam(getArgument(1).getDataType())) {
+ // return least common type
+ return MIN_COMMON_TYPE_SIGNATURES;
+ }
+ return FOLLOW_DATATYPE_SIGNATURE;
}
}
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/MapContainsKey.java
b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/MapContainsKey.java
index b489bd47963..798101e1a2b 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/MapContainsKey.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/MapContainsKey.java
@@ -38,12 +38,18 @@ import java.util.List;
public class MapContainsKey extends ScalarFunction
implements BinaryExpression, ExplicitlyCastableSignature {
- public static final List<FunctionSignature> SIGNATURES = ImmutableList.of(
+ public static final List<FunctionSignature> FOLLOW_DATATYPE_SIGNATURE =
ImmutableList.of(
FunctionSignature.ret(BooleanType.INSTANCE)
.args(MapType.of(new AnyDataType(0),
AnyDataType.INSTANCE_WITHOUT_INDEX),
new FollowToAnyDataType(0))
);
+ public static final List<FunctionSignature> MIN_COMMON_TYPE_SIGNATURES =
ImmutableList.of(
+ FunctionSignature.ret(BooleanType.INSTANCE)
+ .args(MapType.of(new AnyDataType(0),
AnyDataType.INSTANCE_WITHOUT_INDEX),
+ new AnyDataType(0))
+ );
+
/**
* constructor with 2 arguments.
*/
@@ -72,6 +78,13 @@ public class MapContainsKey extends ScalarFunction
@Override
public List<FunctionSignature> getSignatures() {
- return SIGNATURES;
+ if (getArgument(0).getDataType().isMapType()
+ &&
+ ((MapType) getArgument(0).getDataType()).getKeyType()
+
.isSameTypeForComplexTypeParam(getArgument(1).getDataType())) {
+ // return least common type
+ return MIN_COMMON_TYPE_SIGNATURES;
+ }
+ return FOLLOW_DATATYPE_SIGNATURE;
}
}
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/MapContainsValue.java
b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/MapContainsValue.java
index de77386c889..0a8bcdc49c5 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/MapContainsValue.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/MapContainsValue.java
@@ -38,12 +38,18 @@ import java.util.List;
public class MapContainsValue extends ScalarFunction
implements BinaryExpression, ExplicitlyCastableSignature {
- public static final List<FunctionSignature> SIGNATURES = ImmutableList.of(
+ public static final List<FunctionSignature> FOLLOW_DATATYPE_SIGNATURE =
ImmutableList.of(
FunctionSignature.ret(BooleanType.INSTANCE)
.args(MapType.of(AnyDataType.INSTANCE_WITHOUT_INDEX, new
AnyDataType(0)),
new FollowToAnyDataType(0))
);
+ public static final List<FunctionSignature> MIN_COMMON_TYPE_SIGNATURES =
ImmutableList.of(
+ FunctionSignature.ret(BooleanType.INSTANCE)
+ .args(MapType.of(AnyDataType.INSTANCE_WITHOUT_INDEX, new
AnyDataType(0)),
+ new AnyDataType(0))
+ );
+
/**
* constructor with 2 arguments.
*/
@@ -72,6 +78,13 @@ public class MapContainsValue extends ScalarFunction
@Override
public List<FunctionSignature> getSignatures() {
- return SIGNATURES;
+ if (getArgument(0).getDataType().isMapType()
+ &&
+ ((MapType) getArgument(0).getDataType()).getValueType()
+
.isSameTypeForComplexTypeParam(getArgument(1).getDataType())) {
+ // return least common type
+ return MIN_COMMON_TYPE_SIGNATURES;
+ }
+ return FOLLOW_DATATYPE_SIGNATURE;
}
}
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/nereids/types/DataType.java
b/fe/fe-core/src/main/java/org/apache/doris/nereids/types/DataType.java
index 5631a9cdcb6..29d549681f3 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/types/DataType.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/types/DataType.java
@@ -648,6 +648,41 @@ public abstract class DataType {
}
}
+ /**
+ * whether the param dataType is same-like type for nested in complex type
+ * same-like type means: string-like, date-like, number type
+ */
+ public boolean isSameTypeForComplexTypeParam(DataType paramType) {
+ if (this.isArrayType() && paramType.isArrayType()) {
+ return ((ArrayType) this).getItemType()
+ .isSameTypeForComplexTypeParam(((ArrayType)
paramType).getItemType());
+ } else if (this.isMapType() && paramType.isMapType()) {
+ MapType thisMapType = (MapType) this;
+ MapType otherMapType = (MapType) paramType;
+ return
thisMapType.getKeyType().isSameTypeForComplexTypeParam(otherMapType.getKeyType())
+ &&
thisMapType.getValueType().isSameTypeForComplexTypeParam(otherMapType.getValueType());
+ } else if (this.isStructType() && paramType.isStructType()) {
+ StructType thisStructType = (StructType) this;
+ StructType otherStructType = (StructType) paramType;
+ if (thisStructType.getFields().size() !=
otherStructType.getFields().size()) {
+ return false;
+ }
+ for (int i = 0; i < thisStructType.getFields().size(); i++) {
+ if
(!thisStructType.getFields().get(i).getDataType().isSameTypeForComplexTypeParam(
+ otherStructType.getFields().get(i).getDataType())) {
+ return false;
+ }
+ }
+ return true;
+ } else if (this.isStringLikeType() && paramType.isStringLikeType()) {
+ return true;
+ } else if (this.isDateLikeType() && paramType.isDateLikeType()) {
+ return true;
+ } else {
+ return this.isNumericType() && paramType.isNumericType();
+ }
+ }
+
/** getAllPromotions */
public List<DataType> getAllPromotions() {
if (this instanceof ArrayType) {
diff --git a/regression-test/data/nereids_function_p0/scalar_function/Array.out
b/regression-test/data/nereids_function_p0/scalar_function/Array.out
index 4e7054255bd..ebabba9b7da 100644
--- a/regression-test/data/nereids_function_p0/scalar_function/Array.out
+++ b/regression-test/data/nereids_function_p0/scalar_function/Array.out
@@ -15579,3 +15579,30 @@ false
\N
\N
+-- !sql --
+0 0
+
+-- !sql --
+[258] []
+
+-- !sql --
+false false
+
+-- !sql --
+[257, 258] [258, 1, 2, 3]
+
+-- !sql --
+[1, 258, 257] [1, 2, 3, 258]
+
+-- !sql --
+[1, 258] [1, 2, 3]
+
+-- !sql --
+0 0
+
+-- !sql --
+false false
+
+-- !sql --
+false false
+
diff --git
a/regression-test/suites/nereids_function_p0/scalar_function/Array.groovy
b/regression-test/suites/nereids_function_p0/scalar_function/Array.groovy
index 446dbcc6f74..b2a1d4e68fa 100644
--- a/regression-test/suites/nereids_function_p0/scalar_function/Array.groovy
+++ b/regression-test/suites/nereids_function_p0/scalar_function/Array.groovy
@@ -1375,4 +1375,20 @@ suite("nereids_scalar_fn_Array") {
order_qt_sql_array_overlaps_5 """select arrays_overlap(b, c) from
fn_test_array_with_large_decimal order by id"""
order_qt_sql_array_overlaps_6 """select arrays_overlap(c, b) from
fn_test_array_with_large_decimal order by id"""
+ // tests for nereids array functions for number overflow cases
+ qt_sql """ SELECT array_position([1,258],257),array_position([2],258);"""
+ qt_sql """ select array_apply([258], '>' , 257), array_apply([1,2,3], '>',
258);"""
+ qt_sql """ select array_contains([258], 257), array_contains([1,2,3],
258);"""
+ // pushfront and pushback
+ qt_sql """ select array_pushfront([258], 257), array_pushfront([1,2,3],
258);"""
+ qt_sql """ select array_pushback([1,258], 257), array_pushback([1,2,3],
258);"""
+ // array_remove
+ qt_sql """ select array_remove([1,258], 257), array_remove([1,2,3],
258);"""
+ // countequal
+ qt_sql """ select countequal([1,258], 257), countequal([1,2,3], 258);"""
+ // map_contains_key
+ qt_sql """ select map_contains_key(map(1,258), 257),
map_contains_key(map(2,1), 258);"""
+ // map_contains_value
+ qt_sql """ select map_contains_value(map(1,1), 257),
map_contains_value(map(1,2), 258);"""
+
}
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
