This is an automated email from the ASF dual-hosted git repository.
yiguolei pushed a commit to branch branch-2.1
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/branch-2.1 by this push:
new a40a4bbc67c branch-2.1: [fix](Nereids) fold constant for string
function process emoji character by mistake #49087 (#49344)
a40a4bbc67c is described below
commit a40a4bbc67c9be8a845adb54acf72f4107b68d7d
Author: LiBinfeng <[email protected]>
AuthorDate: Sat Mar 22 07:44:55 2025 +0800
branch-2.1: [fix](Nereids) fold constant for string function process emoji
character by mistake #49087 (#49344)
pick: #49087
Related PR: #40441
Problem Summary:
wrong calculation of emoji character length in some String function when
do constant folding in FE. For example:
select STRLEFT('๐๐๐', 2);
should return ๐๐, but fe return ๐ only when folding constant
fixed functions:
- left
- strleft
- right
- strright
- locate
- character_length
- split_by_string
- overlay
- replace_empty
---
.../doris/catalog/BuiltinScalarFunctions.java | 8 +-
.../functions/executable/StringArithmetic.java | 139 +++++++++++----------
.../expressions/functions/scalar/StrLeft.java | 70 -----------
.../expressions/functions/scalar/StrRight.java | 70 -----------
.../expressions/visitor/ScalarFunctionVisitor.java | 10 --
.../nereids/rules/expression/FoldConstantTest.java | 92 ++++++++++++++
.../string_functions/test_string_function.out | Bin 4644 -> 4890 bytes
.../fold_constant_string_arithmatic.groovy | 118 ++++++++++++++++-
8 files changed, 284 insertions(+), 223 deletions(-)
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/catalog/BuiltinScalarFunctions.java
b/fe/fe-core/src/main/java/org/apache/doris/catalog/BuiltinScalarFunctions.java
index 08e5d6268b5..9b2a1bfa560 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/catalog/BuiltinScalarFunctions.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/catalog/BuiltinScalarFunctions.java
@@ -400,8 +400,6 @@ import
org.apache.doris.nereids.trees.expressions.functions.scalar.StPolygonfrom
import org.apache.doris.nereids.trees.expressions.functions.scalar.StX;
import org.apache.doris.nereids.trees.expressions.functions.scalar.StY;
import org.apache.doris.nereids.trees.expressions.functions.scalar.StartsWith;
-import org.apache.doris.nereids.trees.expressions.functions.scalar.StrLeft;
-import org.apache.doris.nereids.trees.expressions.functions.scalar.StrRight;
import org.apache.doris.nereids.trees.expressions.functions.scalar.StrToDate;
import org.apache.doris.nereids.trees.expressions.functions.scalar.Strcmp;
import
org.apache.doris.nereids.trees.expressions.functions.scalar.StructElement;
@@ -738,7 +736,7 @@ public class BuiltinScalarFunctions implements
FunctionHelper {
scalar(L2Distance.class, "l2_distance"),
scalar(LastDay.class, "last_day"),
scalar(Least.class, "least"),
- scalar(Left.class, "left"),
+ scalar(Left.class, "left", "strleft"),
scalar(Length.class, "length"),
scalar(Crc32.class, "crc32"),
scalar(Like.class, "like"),
@@ -820,7 +818,7 @@ public class BuiltinScalarFunctions implements
FunctionHelper {
scalar(Replace.class, "replace"),
scalar(ReplaceEmpty.class, "replace_empty"),
scalar(Reverse.class, "reverse"),
- scalar(Right.class, "right"),
+ scalar(Right.class, "right", "strright"),
scalar(Round.class, "round"),
scalar(RoundBankers.class, "round_bankers"),
scalar(Rpad.class, "rpad"),
@@ -880,8 +878,6 @@ public class BuiltinScalarFunctions implements
FunctionHelper {
scalar(StY.class, "st_y"),
scalar(StartsWith.class, "starts_with"),
scalar(Strcmp.class, "strcmp"),
- scalar(StrLeft.class, "strleft"),
- scalar(StrRight.class, "strright"),
scalar(StrToDate.class, "str_to_date"),
scalar(SubBitmap.class, "sub_bitmap"),
scalar(SubReplace.class, "sub_replace"),
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/executable/StringArithmetic.java
b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/executable/StringArithmetic.java
index 532ec04d8aa..ed3e3894ecc 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/executable/StringArithmetic.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/executable/StringArithmetic.java
@@ -43,6 +43,7 @@ import org.apache.doris.nereids.types.ArrayType;
import org.apache.doris.qe.ConnectContext;
import com.google.common.collect.ImmutableList;
+import com.google.common.collect.Lists;
import java.io.UnsupportedEncodingException;
import java.net.MalformedURLException;
@@ -61,7 +62,7 @@ import java.util.regex.Pattern;
* concat
*/
public class StringArithmetic {
- private static Expression castStringLikeLiteral(StringLikeLiteral first,
String value) {
+ private static Literal castStringLikeLiteral(StringLikeLiteral first,
String value) {
if (first instanceof StringLiteral) {
return new StringLiteral(value);
} else if (first instanceof VarcharLiteral) {
@@ -80,7 +81,7 @@ public class StringArithmetic {
}
private static String substringImpl(String first, int second, int third) {
- int stringLength = first.length();
+ int stringLength = first.codePointCount(0, first.length());
if (stringLength == 0) {
return "";
}
@@ -102,8 +103,11 @@ public class StringArithmetic {
} else {
rightIndex = third + leftIndex;
}
+ // at here leftIndex and rightIndex can not be exceeding boundary
+ int finalLeftIndex = first.codePointCount(0, (int) leftIndex);
+ int finalRightIndex = first.codePointCount(0, (int) rightIndex);
// left index and right index are in integer range because of
definition, so we can safely cast it to int
- return first.substring((int) leftIndex, (int) rightIndex);
+ return first.substring(finalLeftIndex, finalRightIndex);
}
/**
@@ -294,12 +298,14 @@ public class StringArithmetic {
*/
@ExecFunction(name = "left")
public static Expression left(StringLikeLiteral first, IntegerLiteral
second) {
+ int inputLength = first.getValue().codePointCount(0,
first.getValue().length());
if (second.getValue() <= 0) {
return castStringLikeLiteral(first, "");
- } else if (second.getValue() < first.getValue().length()) {
- return castStringLikeLiteral(first, first.getValue().substring(0,
second.getValue()));
- } else {
+ } else if (second.getValue() >= inputLength) {
return first;
+ } else {
+ int index = first.getValue().codePointCount(0, second.getValue());
+ return castStringLikeLiteral(first, first.getValue().substring(0,
index));
}
}
@@ -308,17 +314,20 @@ public class StringArithmetic {
*/
@ExecFunction(name = "right")
public static Expression right(StringLikeLiteral first, IntegerLiteral
second) {
- if (second.getValue() < (- first.getValue().length()) ||
Math.abs(second.getValue()) == 0) {
+ int inputLength = first.getValue().codePointCount(0,
first.getValue().length());
+ if (second.getValue() < (- inputLength) || Math.abs(second.getValue())
== 0) {
return castStringLikeLiteral(first, "");
- } else if (second.getValue() > first.getValue().length()) {
+ } else if (second.getValue() >= inputLength) {
return first;
} else {
- if (second.getValue() > 0) {
+ if (second.getValue() >= 0) {
+ int index = first.getValue().codePointCount(0,
second.getValue());
return castStringLikeLiteral(first, first.getValue().substring(
- first.getValue().length() - second.getValue(),
first.getValue().length()));
+ inputLength - index, inputLength));
} else {
+ int index =
first.getValue().codePointCount(Math.abs(second.getValue()) - 1,
first.getValue().length());
return castStringLikeLiteral(first, first.getValue().substring(
- Math.abs(second.getValue()) - 1,
first.getValue().length()));
+ Math.abs(index) - 1, inputLength));
}
}
}
@@ -338,7 +347,7 @@ public class StringArithmetic {
public static Expression locate(StringLikeLiteral first, StringLikeLiteral
second, IntegerLiteral third) {
int result = second.getValue().indexOf(first.getValue()) + 1;
if (third.getValue() <= 0 || !substringImpl(second.getValue(),
third.getValue(),
- second.getValue().length()).contains(first.getValue())) {
+ second.getValue().codePointCount(0,
second.getValue().length())).contains(first.getValue())) {
result = 0;
}
return new IntegerLiteral(result);
@@ -409,7 +418,7 @@ public class StringArithmetic {
*/
@ExecFunction(name = "character_length")
public static Expression characterLength(StringLikeLiteral first) {
- return new IntegerLiteral(first.getValue().length());
+ return new IntegerLiteral(first.getValue().codePointCount(0,
first.getValue().length()));
}
private static boolean isAlphabetic(char c) {
@@ -666,6 +675,23 @@ public class StringArithmetic {
return new VarcharLiteral(sb.toString());
}
+ /**
+ * split by char by empty string considering emoji
+ * @param str input string to be split
+ * @return ArrayLiteral
+ */
+ public static List<String> splitByGrapheme(StringLikeLiteral str) {
+ List<String> result =
Lists.newArrayListWithExpectedSize(str.getValue().length());
+ int length = str.getValue().length();
+ for (int i = 0; i < length; ) {
+ int codePoint = str.getValue().codePointAt(i);
+ int charCount = Character.charCount(codePoint);
+ result.add(new String(new int[]{codePoint}, 0, 1));
+ i += charCount;
+ }
+ return result;
+ }
+
/**
* Executable arithmetic functions split_by_string
*/
@@ -674,11 +700,17 @@ public class StringArithmetic {
if (first.getValue().isEmpty()) {
return new ArrayLiteral(ImmutableList.of(),
ArrayType.of(first.getDataType()));
}
- int limit = second.getValue().isEmpty() ? 0 : -1;
- String[] result =
first.getValue().split(Pattern.quote(second.getValue()), limit);
+ if (second.getValue().isEmpty()) {
+ List<Literal> result =
Lists.newArrayListWithExpectedSize(first.getValue().length());
+ for (String resultStr : splitByGrapheme(first)) {
+ result.add(castStringLikeLiteral(first, resultStr));
+ }
+ return new ArrayLiteral(result);
+ }
+ String[] result =
first.getValue().split(Pattern.quote(second.getValue()), -1);
List<Literal> items = new ArrayList<>();
for (String s : result) {
- items.add((Literal) castStringLikeLiteral(first, s));
+ items.add(castStringLikeLiteral(first, s));
}
return new ArrayLiteral(items);
}
@@ -775,60 +807,26 @@ public class StringArithmetic {
}
}
- /**
- * Executable arithmetic functions strLeft
- */
- @ExecFunction(name = "strleft")
- public static Expression strLeft(StringLikeLiteral first, IntegerLiteral
second) {
- if (second.getValue() <= 0) {
- return castStringLikeLiteral(first, "");
- } else if (second.getValue() > first.getValue().length()) {
- return first;
- } else {
- return castStringLikeLiteral(first, first.getValue().substring(0,
second.getValue()));
- }
- }
-
- /**
- * Executable arithmetic functions strRight
- */
- @ExecFunction(name = "strright")
- public static Expression strRight(StringLikeLiteral first, IntegerLiteral
second) {
- if (second.getValue() < (- first.getValue().length()) ||
Math.abs(second.getValue()) == 0) {
- return castStringLikeLiteral(first, "");
- } else if (second.getValue() > first.getValue().length()) {
- return first;
- } else {
- if (second.getValue() > 0) {
- return castStringLikeLiteral(first, first.getValue().substring(
- first.getValue().length() - second.getValue(),
first.getValue().length()));
- } else {
- return castStringLikeLiteral(first, first.getValue().substring(
- Math.abs(second.getValue()) - 1,
first.getValue().length()));
- }
- }
- }
-
/**
* Executable arithmetic functions overlay
*/
@ExecFunction(name = "overlay")
- public static Expression overlay(StringLikeLiteral first,
- IntegerLiteral second, IntegerLiteral
third, StringLikeLiteral four) {
+ public static Expression overlay(StringLikeLiteral originStr,
+ IntegerLiteral pos, IntegerLiteral
len, StringLikeLiteral insertStr) {
StringBuilder sb = new StringBuilder();
- if (second.getValue() <= 0 || second.getValue() >
first.getValue().length()) {
- return first;
+ int totalLength = originStr.getValue().codePointCount(0,
originStr.getValue().length());
+ if (pos.getValue() <= 0 || pos.getValue() > totalLength) {
+ return originStr;
} else {
- if (third.getValue() < 0 || third.getValue() >
(first.getValue().length() - third.getValue())) {
- sb.append(first.getValue().substring(0, second.getValue() -
1));
- sb.append(four.getValue());
- return castStringLikeLiteral(first, sb.toString());
+ if (len.getValue() < 0 || len.getValue() > (totalLength -
pos.getValue())) {
+ sb.append(substringImpl(originStr.getValue(), 1,
pos.getValue() - 1));
+ sb.append(insertStr.getValue());
+ return castStringLikeLiteral(originStr, sb.toString());
} else {
- sb.append(first.getValue().substring(0, second.getValue() -
1));
- sb.append(four.getValue());
- sb.append(first.getValue().substring(second.getValue()
- + third.getValue() - 1, first.getValue().length()));
- return castStringLikeLiteral(first, sb.toString());
+ sb.append(substringImpl(originStr.getValue(), 1,
pos.getValue() - 1));
+ sb.append(insertStr.getValue());
+ sb.append(substringImpl(originStr.getValue(), pos.getValue() +
len.getValue(), totalLength));
+ return castStringLikeLiteral(originStr, sb.toString());
}
}
}
@@ -937,7 +935,7 @@ public class StringArithmetic {
*/
@ExecFunction(name = "append_trailing_char_if_absent")
public static Expression appendTrailingCharIfAbsent(StringLikeLiteral
first, StringLikeLiteral second) {
- if (second.getValue().length() != 1) {
+ if (second.getValue().codePointCount(0, second.getValue().length()) !=
1) {
return new NullLiteral(first.getDataType());
}
if (first.getValue().endsWith(second.getValue())) {
@@ -1001,6 +999,19 @@ public class StringArithmetic {
*/
@ExecFunction(name = "replace_empty")
public static Expression replaceEmpty(StringLikeLiteral first,
StringLikeLiteral second, StringLikeLiteral third) {
+ if (second.getValue().isEmpty()) {
+ if (first.getValue().isEmpty()) {
+ return castStringLikeLiteral(first, third.getValue());
+ }
+ List<String> inputs = splitByGrapheme(first);
+ StringBuilder sb = new StringBuilder();
+ sb.append(third.getValue());
+ for (String input : inputs) {
+ sb.append(input);
+ sb.append(third.getValue());
+ }
+ return castStringLikeLiteral(first, sb.toString());
+ }
return castStringLikeLiteral(first,
first.getValue().replace(second.getValue(), third.getValue()));
}
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/StrLeft.java
b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/StrLeft.java
deleted file mode 100644
index e8188dbc0f3..00000000000
---
a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/StrLeft.java
+++ /dev/null
@@ -1,70 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements. See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership. The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License. You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-package org.apache.doris.nereids.trees.expressions.functions.scalar;
-
-import org.apache.doris.catalog.FunctionSignature;
-import org.apache.doris.nereids.trees.expressions.Expression;
-import
org.apache.doris.nereids.trees.expressions.functions.ExplicitlyCastableSignature;
-import org.apache.doris.nereids.trees.expressions.functions.PropagateNullable;
-import org.apache.doris.nereids.trees.expressions.shape.BinaryExpression;
-import org.apache.doris.nereids.trees.expressions.visitor.ExpressionVisitor;
-import org.apache.doris.nereids.types.IntegerType;
-import org.apache.doris.nereids.types.StringType;
-import org.apache.doris.nereids.types.VarcharType;
-
-import com.google.common.base.Preconditions;
-import com.google.common.collect.ImmutableList;
-
-import java.util.List;
-
-/**
- * ScalarFunction 'strleft'. This class is generated by GenerateFunction.
- */
-public class StrLeft extends ScalarFunction
- implements BinaryExpression, ExplicitlyCastableSignature,
PropagateNullable {
-
- public static final List<FunctionSignature> SIGNATURES = ImmutableList.of(
-
FunctionSignature.ret(VarcharType.SYSTEM_DEFAULT).args(VarcharType.SYSTEM_DEFAULT,
IntegerType.INSTANCE),
-
FunctionSignature.ret(StringType.INSTANCE).args(StringType.INSTANCE,
IntegerType.INSTANCE));
-
- /**
- * constructor with 2 arguments.
- */
- public StrLeft(Expression arg0, Expression arg1) {
- super("strleft", arg0, arg1);
- }
-
- /**
- * withChildren.
- */
- @Override
- public StrLeft withChildren(List<Expression> children) {
- Preconditions.checkArgument(children.size() == 2);
- return new StrLeft(children.get(0), children.get(1));
- }
-
- @Override
- public List<FunctionSignature> getSignatures() {
- return SIGNATURES;
- }
-
- @Override
- public <R, C> R accept(ExpressionVisitor<R, C> visitor, C context) {
- return visitor.visitStrLeft(this, context);
- }
-}
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/StrRight.java
b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/StrRight.java
deleted file mode 100644
index 0cb563ce94f..00000000000
---
a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/StrRight.java
+++ /dev/null
@@ -1,70 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements. See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership. The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License. You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-package org.apache.doris.nereids.trees.expressions.functions.scalar;
-
-import org.apache.doris.catalog.FunctionSignature;
-import org.apache.doris.nereids.trees.expressions.Expression;
-import
org.apache.doris.nereids.trees.expressions.functions.ExplicitlyCastableSignature;
-import org.apache.doris.nereids.trees.expressions.functions.PropagateNullable;
-import org.apache.doris.nereids.trees.expressions.shape.BinaryExpression;
-import org.apache.doris.nereids.trees.expressions.visitor.ExpressionVisitor;
-import org.apache.doris.nereids.types.IntegerType;
-import org.apache.doris.nereids.types.StringType;
-import org.apache.doris.nereids.types.VarcharType;
-
-import com.google.common.base.Preconditions;
-import com.google.common.collect.ImmutableList;
-
-import java.util.List;
-
-/**
- * ScalarFunction 'strright'. This class is generated by GenerateFunction.
- */
-public class StrRight extends ScalarFunction
- implements BinaryExpression, ExplicitlyCastableSignature,
PropagateNullable {
-
- public static final List<FunctionSignature> SIGNATURES = ImmutableList.of(
-
FunctionSignature.ret(VarcharType.SYSTEM_DEFAULT).args(VarcharType.SYSTEM_DEFAULT,
IntegerType.INSTANCE),
-
FunctionSignature.ret(StringType.INSTANCE).args(StringType.INSTANCE,
IntegerType.INSTANCE));
-
- /**
- * constructor with 2 arguments.
- */
- public StrRight(Expression arg0, Expression arg1) {
- super("strright", arg0, arg1);
- }
-
- /**
- * withChildren.
- */
- @Override
- public StrRight withChildren(List<Expression> children) {
- Preconditions.checkArgument(children.size() == 2);
- return new StrRight(children.get(0), children.get(1));
- }
-
- @Override
- public List<FunctionSignature> getSignatures() {
- return SIGNATURES;
- }
-
- @Override
- public <R, C> R accept(ExpressionVisitor<R, C> visitor, C context) {
- return visitor.visitStrRight(this, context);
- }
-}
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/visitor/ScalarFunctionVisitor.java
b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/visitor/ScalarFunctionVisitor.java
index a45d6ccdff6..30ce433c070 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/visitor/ScalarFunctionVisitor.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/visitor/ScalarFunctionVisitor.java
@@ -398,8 +398,6 @@ import
org.apache.doris.nereids.trees.expressions.functions.scalar.StPolygonfrom
import org.apache.doris.nereids.trees.expressions.functions.scalar.StX;
import org.apache.doris.nereids.trees.expressions.functions.scalar.StY;
import org.apache.doris.nereids.trees.expressions.functions.scalar.StartsWith;
-import org.apache.doris.nereids.trees.expressions.functions.scalar.StrLeft;
-import org.apache.doris.nereids.trees.expressions.functions.scalar.StrRight;
import org.apache.doris.nereids.trees.expressions.functions.scalar.StrToDate;
import org.apache.doris.nereids.trees.expressions.functions.scalar.Strcmp;
import
org.apache.doris.nereids.trees.expressions.functions.scalar.StructElement;
@@ -1958,14 +1956,6 @@ public interface ScalarFunctionVisitor<R, C> {
return visitScalarFunction(startsWith, context);
}
- default R visitStrLeft(StrLeft strLeft, C context) {
- return visitScalarFunction(strLeft, context);
- }
-
- default R visitStrRight(StrRight strRight, C context) {
- return visitScalarFunction(strRight, context);
- }
-
default R visitStrToDate(StrToDate strToDate, C context) {
return visitScalarFunction(strToDate, context);
}
diff --git
a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/expression/FoldConstantTest.java
b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/expression/FoldConstantTest.java
index 3e5654580bf..3a83877849c 100644
---
a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/expression/FoldConstantTest.java
+++
b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/expression/FoldConstantTest.java
@@ -38,6 +38,7 @@ import
org.apache.doris.nereids.trees.expressions.functions.scalar.Asin;
import org.apache.doris.nereids.trees.expressions.functions.scalar.Bin;
import org.apache.doris.nereids.trees.expressions.functions.scalar.BitCount;
import org.apache.doris.nereids.trees.expressions.functions.scalar.Ceil;
+import
org.apache.doris.nereids.trees.expressions.functions.scalar.CharacterLength;
import org.apache.doris.nereids.trees.expressions.functions.scalar.Coalesce;
import org.apache.doris.nereids.trees.expressions.functions.scalar.ConvertTz;
import org.apache.doris.nereids.trees.expressions.functions.scalar.Cos;
@@ -47,16 +48,21 @@ import
org.apache.doris.nereids.trees.expressions.functions.scalar.Exp;
import org.apache.doris.nereids.trees.expressions.functions.scalar.Floor;
import
org.apache.doris.nereids.trees.expressions.functions.scalar.FromUnixtime;
import org.apache.doris.nereids.trees.expressions.functions.scalar.HoursAdd;
+import org.apache.doris.nereids.trees.expressions.functions.scalar.Left;
import org.apache.doris.nereids.trees.expressions.functions.scalar.Ln;
+import org.apache.doris.nereids.trees.expressions.functions.scalar.Locate;
import org.apache.doris.nereids.trees.expressions.functions.scalar.Log;
import org.apache.doris.nereids.trees.expressions.functions.scalar.MinutesAdd;
import org.apache.doris.nereids.trees.expressions.functions.scalar.Power;
+import
org.apache.doris.nereids.trees.expressions.functions.scalar.ReplaceEmpty;
+import org.apache.doris.nereids.trees.expressions.functions.scalar.Right;
import org.apache.doris.nereids.trees.expressions.functions.scalar.Round;
import org.apache.doris.nereids.trees.expressions.functions.scalar.SecondsAdd;
import org.apache.doris.nereids.trees.expressions.functions.scalar.Sign;
import org.apache.doris.nereids.trees.expressions.functions.scalar.Sin;
import org.apache.doris.nereids.trees.expressions.functions.scalar.Sqrt;
import org.apache.doris.nereids.trees.expressions.functions.scalar.StrToDate;
+import org.apache.doris.nereids.trees.expressions.functions.scalar.Substring;
import org.apache.doris.nereids.trees.expressions.functions.scalar.Tan;
import org.apache.doris.nereids.trees.expressions.functions.scalar.ToDays;
import
org.apache.doris.nereids.trees.expressions.functions.scalar.UnixTimestamp;
@@ -332,6 +338,92 @@ class FoldConstantTest extends ExpressionRewriteTestHelper
{
AppendTrailingCharIfAbsent a = new
AppendTrailingCharIfAbsent(StringLiteral.of("1"), StringLiteral.of("3"));
rewritten = executor.rewrite(a, context);
Assertions.assertEquals(new StringLiteral("13"), rewritten);
+
+ Left left = new Left(StringLiteral.of("hello world"),
IntegerLiteral.of(5));
+ rewritten = executor.rewrite(left, context);
+ Assertions.assertEquals(new StringLiteral("hello"), rewritten);
+ left = new Left(StringLiteral.of("test"), IntegerLiteral.of(10));
+ rewritten = executor.rewrite(left, context);
+ Assertions.assertEquals(new StringLiteral("test"), rewritten);
+ left = new Left(StringLiteral.of("data"), IntegerLiteral.of(0));
+ rewritten = executor.rewrite(left, context);
+ Assertions.assertEquals(new StringLiteral(""), rewritten);
+ left = new Left(StringLiteral.of("data"), IntegerLiteral.of(-3));
+ rewritten = executor.rewrite(left, context);
+ Assertions.assertEquals(new StringLiteral(""), rewritten);
+
+ Right right = new Right(StringLiteral.of("hello world"),
IntegerLiteral.of(5));
+ rewritten = executor.rewrite(right, context);
+ Assertions.assertEquals(new StringLiteral("world"), rewritten);
+ right = new Right(StringLiteral.of("test"), IntegerLiteral.of(10));
+ rewritten = executor.rewrite(right, context);
+ Assertions.assertEquals(new StringLiteral("test"), rewritten);
+ right = new Right(StringLiteral.of("data"), IntegerLiteral.of(0));
+ rewritten = executor.rewrite(right, context);
+ Assertions.assertEquals(new StringLiteral(""), rewritten);
+ right = new Right(StringLiteral.of("data"), IntegerLiteral.of(-3));
+ rewritten = executor.rewrite(right, context);
+ Assertions.assertEquals(new StringLiteral("ata"), rewritten);
+
+ Substring substr = new Substring(
+ StringLiteral.of("database"),
+ IntegerLiteral.of(1),
+ IntegerLiteral.of(4)
+ );
+ rewritten = executor.rewrite(substr, context);
+ Assertions.assertEquals(new StringLiteral("data"), rewritten);
+ substr = new Substring(
+ StringLiteral.of("database"),
+ IntegerLiteral.of(-4),
+ IntegerLiteral.of(4)
+ );
+ rewritten = executor.rewrite(substr, context);
+ Assertions.assertEquals(new StringLiteral("base"), rewritten);
+ substr = new Substring(
+ StringLiteral.of("example"),
+ IntegerLiteral.of(3),
+ IntegerLiteral.of(10)
+ );
+ rewritten = executor.rewrite(substr, context);
+ Assertions.assertEquals(new StringLiteral("ample"), rewritten);
+
+ Locate locate = new Locate(
+ StringLiteral.of("world"),
+ StringLiteral.of("hello world")
+ );
+ rewritten = executor.rewrite(locate, context);
+ Assertions.assertEquals(new IntegerLiteral(7), rewritten);
+ locate = new Locate(
+ StringLiteral.of("test"),
+ StringLiteral.of("hello world")
+ );
+ rewritten = executor.rewrite(locate, context);
+ Assertions.assertEquals(new IntegerLiteral(0), rewritten);
+ locate = new Locate(
+ StringLiteral.of("l"),
+ StringLiteral.of("hello world"),
+ IntegerLiteral.of(3)
+ );
+ rewritten = executor.rewrite(locate, context);
+ Assertions.assertEquals(new IntegerLiteral(3), rewritten);
+
+ CharacterLength len = new CharacterLength(StringLiteral.of("hello"));
+ rewritten = executor.rewrite(len, context);
+ Assertions.assertEquals(new IntegerLiteral(5), rewritten);
+ len = new CharacterLength(StringLiteral.of(""));
+ rewritten = executor.rewrite(len, context);
+ Assertions.assertEquals(new IntegerLiteral(0), rewritten);
+ len = new CharacterLength(StringLiteral.of("๐"));
+ rewritten = executor.rewrite(len, context);
+ Assertions.assertEquals(new IntegerLiteral(1), rewritten);
+
+ ReplaceEmpty replace = new ReplaceEmpty(
+ StringLiteral.of(""),
+ StringLiteral.of(""),
+ StringLiteral.of("default")
+ );
+ rewritten = executor.rewrite(replace, context);
+ Assertions.assertEquals(new StringLiteral("default"), rewritten);
}
@Test
diff --git
a/regression-test/data/query_p0/sql_functions/string_functions/test_string_function.out
b/regression-test/data/query_p0/sql_functions/string_functions/test_string_function.out
index 86953f95ab0..dacc36966a2 100644
Binary files
a/regression-test/data/query_p0/sql_functions/string_functions/test_string_function.out
and
b/regression-test/data/query_p0/sql_functions/string_functions/test_string_function.out
differ
diff --git
a/regression-test/suites/nereids_p0/expression/fold_constant/fold_constant_string_arithmatic.groovy
b/regression-test/suites/nereids_p0/expression/fold_constant/fold_constant_string_arithmatic.groovy
index 8d440289073..0f0650f4a44 100644
---
a/regression-test/suites/nereids_p0/expression/fold_constant/fold_constant_string_arithmatic.groovy
+++
b/regression-test/suites/nereids_p0/expression/fold_constant/fold_constant_string_arithmatic.groovy
@@ -259,7 +259,6 @@ suite("fold_constant_string_arithmatic") {
testFoldConst("select initcap('semi;test')")
testFoldConst("select initcap('quote''test')")
testFoldConst("select initcap('slash/test')")
- testFoldConst("select initcap('back\slash')")
testFoldConst("select initcap('emoji<d83d><dc3c>test')")
testFoldConst("select initcap('ๆฐๅญ123test')")
testFoldConst("select initcap(' leading space')")
@@ -498,7 +497,7 @@ suite("fold_constant_string_arithmatic") {
testFoldConst("select null_or_empty(NULL)")
testFoldConst("select null_or_empty('\b')")
testFoldConst("select null_or_empty(' \b')")
-
+
// parse_url
testFoldConst("select
parse_url(cast('http://www.example.com/path?query=abc' as string), cast('HOST'
as string))")
testFoldConst("select parse_url('http://www.example.com/path?query=abc',
'HOST')")
@@ -934,6 +933,60 @@ suite("fold_constant_string_arithmatic") {
testFoldConst("select strleft(' Hello World', 5)")
testFoldConst("select strleft('Hello World ', 50)")
testFoldConst("select strleft(NULL, 1)")
+ testFoldConst("select strleft('๐๐๐', 2)")
+ testFoldConst("select strleft('ฮฑฮฒฮณฮด', 3)")
+ testFoldConst("select strleft('ไฝ ๅฅฝไธ็', 4)")
+ testFoldConst("select strleft('ใใใซใกใฏไธ็', 5)")
+ testFoldConst("select strleft('์๋
ํ์ธ์', 3)")
+ testFoldConst("select strleft('ะฟัะธะฒะตั', 4)")
+ testFoldConst("select strleft('ืฉืืื', 3)")
+ testFoldConst("select strleft('๐๐๐๐๐', 4)")
+ testFoldConst("select strleft('ฮฑฮฒฮณฮดฮตฮถ', 4)")
+ testFoldConst("select strleft('ไฝ ๅฅฝๅ๏ผไธ็', 6)")
+ testFoldConst("select strleft('ใใใซใกใฏใ็ด ๆดใใใไธๆฅ', 7)")
+ testFoldConst("select strleft('์๋
ํ์ธ์ ์ฌ๋ฌ๋ถ', 5)")
+ testFoldConst("select strleft('ะฟัะธะฒะตั ะผะธั', 6)")
+ testFoldConst("select strleft('ืฉืืื ืขืืื', 4)")
+ testFoldConst("select strleft(null, 2)")
+ testFoldConst("select strleft('๐๐๐๐๐๐', 0)")
+ testFoldConst("select strleft('ฮฑฮฒฮณฮดฮตฮถฮท', -1)")
+ testFoldConst("select strleft('ไฝ ๅฅฝ๏ผ็พๅฅฝ็ไธๅคฉ', -2)")
+ testFoldConst("select strleft('ใใใซใกใฏใ็ด ๆดใใใไธๆฅ', -3)")
+ testFoldConst("select strleft('์๋
ํ์ธ์ ์ฌ๋ฌ๋ถ ์๋
ํ๊ฐ์ธ์', -4)")
+ testFoldConst("select strleft('ะฟัะธะฒะตั ะฒัะตะผ ะดััะทััะผ', -5)")
+ testFoldConst("select strleft('ืฉืืื ืืื ืืืืจืื', -3)")
+ testFoldConst("select strleft('', 2)")
+ testFoldConst("select strleft('๐๐', -1)")
+ testFoldConst("select strleft('ฮฑฮฒ', 0)")
+ testFoldConst("select strleft('ไฝ ๅฅฝ', -1)")
+ testFoldConst("select strleft('ใใใซใกใฏ', 0)")
+ testFoldConst("select strleft('์๋
ํ์ธ์', -1)")
+ testFoldConst("select strleft('ะฟัะธะฒะตั', 0)")
+ testFoldConst("select strleft('ืฉืืื', -1)")
+ testFoldConst("select strleft('๐๐๐๐๐๐๐', 5)")
+ testFoldConst("select strleft('ฮฑฮฒฮณฮดฮตฮถฮทฮธ', 5)")
+ testFoldConst("select strleft('ไฝ ๅฅฝ๏ผไธ็๏ผๆฌข่ฟ', 6)")
+ testFoldConst("select strleft('ใใใซใกใฏใไธ็๏ผใใใใ', 7)")
+ testFoldConst("select strleft('์๋
ํ์ธ์ ์ธ๊ณ!', 5)")
+ testFoldConst("select strleft('ะฟัะธะฒะตั, ะผะธั!', 6)")
+ testFoldConst("select strleft('ืฉืืื ืขืืื!', 4)")
+ testFoldConst("select strleft('๐๐๐๐๐๐๐๐', 6)")
+ testFoldConst("select strleft('ฮฑฮฒฮณฮดฮตฮถฮทฮธฮน', 6)")
+ testFoldConst("select strleft('ไฝ ๅฅฝๅ๏ผ็พๅฅฝ็ไธ็', 7)")
+ testFoldConst("select strleft('ใใใซใกใฏใ็ด ๆดใใใไธ็ใ', 8)")
+ testFoldConst("select strleft('์๋
ํ์ธ์, ์๋ฆ๋ค์ด ์ธ์', 7)")
+ testFoldConst("select strleft('ะฟัะธะฒะตั, ะฟัะตะบัะฐัะฝัะน ะผะธั', 8)")
+ testFoldConst("select strleft('ืฉืืื ืืขืืื ืืคื', 5)")
+ testFoldConst("select strleft('', -1)")
+ testFoldConst("select strleft('๐๐', 0)")
+ testFoldConst("select strleft('ฮฑฮฒ', -1)")
+ testFoldConst("select strleft('ไฝ ๅฅฝ', 0)")
+ testFoldConst("select strleft('ใใใซใกใฏ', -1)")
+ testFoldConst("select strleft('์๋
ํ์ธ์', 0)")
+ testFoldConst("select strleft('ะฟัะธะฒะตั', -1)")
+ testFoldConst("select strleft('ืฉืืื', 0)")
+ testFoldConst("select strleft('ะฟัะธะฒะตั', 2147483647)")
+ testFoldConst("select strleft('ะฟัะธะฒะตั', 2147483648)")
// strright
testFoldConst("select strright('good morning', NULL)")
@@ -944,7 +997,61 @@ suite("fold_constant_string_arithmatic") {
testFoldConst("select strright(' Hello World', 5)")
testFoldConst("select strright('Hello World ', 5)")
testFoldConst("select strright(NULL, 1)")
-
+ testFoldConst("select strright('๐๐๐', 2)")
+ testFoldConst("select strright('ฮฑฮฒฮณฮด', 3)")
+ testFoldConst("select strright('ไฝ ๅฅฝไธ็', 2)")
+ testFoldConst("select strright('ใใใซใกใฏไธ็', 3)")
+ testFoldConst("select strright('์๋
ํ์ธ์', 3)")
+ testFoldConst("select strright('ะฟัะธะฒะตั', 4)")
+ testFoldConst("select strright('ืฉืืื', 3)")
+ testFoldConst("select strright('๐๐๐๐๐', 4)")
+ testFoldConst("select strright('ฮฑฮฒฮณฮดฮตฮถ', 4)")
+ testFoldConst("select strright('ไฝ ๅฅฝๅ๏ผไธ็', 6)")
+ testFoldConst("select strright('ใใใซใกใฏใ็ด ๆดใใใไธๆฅ', 7)")
+ testFoldConst("select strright('์๋
ํ์ธ์ ์ฌ๋ฌ๋ถ', 5)")
+ testFoldConst("select strright('ะฟัะธะฒะตั ะผะธั', 6)")
+ testFoldConst("select strright('ืฉืืื ืืื ืืืืจืื', 10)")
+ testFoldConst("select strright(null, 2)")
+ testFoldConst("select strright('๐๐๐๐๐๐', 0)")
+ testFoldConst("select strright('ฮฑฮฒฮณฮดฮตฮถฮท', -1)")
+ testFoldConst("select strright('ไฝ ๅฅฝ๏ผ็พๅฅฝ็ไธๅคฉ', -2)")
+ testFoldConst("select strright('ใใใซใกใฏใ็ด ๆดใใใไธๆฅ', -3)")
+ testFoldConst("select strright('์๋
ํ์ธ์ ์ฌ๋ฌ๋ถ ์๋
ํ๊ฐ์ธ์', -4)")
+ testFoldConst("select strright('ะฟัะธะฒะตั ะฒัะตะผ ะดััะทััะผ', -5)")
+ testFoldConst("select strright('ืฉืืื ืขืืื!', -3)")
+ testFoldConst("select strright('', 2)")
+ testFoldConst("select strright('๐๐', -1)")
+ testFoldConst("select strright('ฮฑฮฒ', 0)")
+ testFoldConst("select strright('ไฝ ๅฅฝ', -1)")
+ testFoldConst("select strright('ใใใซใกใฏ', 0)")
+ testFoldConst("select strright('์๋
ํ์ธ์', -1)")
+ testFoldConst("select strright('ะฟัะธะฒะตั', 0)")
+ testFoldConst("select strright('ืฉืืื', -1)")
+ testFoldConst("select strright('๐๐๐๐๐๐๐', 5)")
+ testFoldConst("select strright('ฮฑฮฒฮณฮดฮตฮถฮทฮธ', 5)")
+ testFoldConst("select strright('ไฝ ๅฅฝ๏ผไธ็๏ผๆฌข่ฟ', 6)")
+ testFoldConst("select strright('ใใใซใกใฏใไธ็๏ผใใใใ', 7)")
+ testFoldConst("select strright('์๋
ํ์ธ์ ์ธ๊ณ!', 5)")
+ testFoldConst("select strright('ะฟัะธะฒะตั, ะผะธั!', 6)")
+ testFoldConst("select strright('ืฉืืื ืขืืื!', 4)")
+ testFoldConst("select strright('๐๐๐๐๐๐๐๐', 6)")
+ testFoldConst("select strright('ฮฑฮฒฮณฮดฮตฮถฮทฮธฮน', 6)")
+ testFoldConst("select strright('ไฝ ๅฅฝๅ๏ผ็พๅฅฝ็ไธ็', 7)")
+ testFoldConst("select strright('ใใใซใกใฏใ็ด ๆดใใใไธ็ใ', 8)")
+ testFoldConst("select strright('์๋
ํ์ธ์, ์๋ฆ๋ค์ด ์ธ์', 7)")
+ testFoldConst("select strright('ะฟัะธะฒะตั, ะฟัะตะบัะฐัะฝัะน ะผะธั', 8)")
+ testFoldConst("select strright('ืฉืืื ืืขืืื ืืคื', 5)")
+ testFoldConst("select strright('', -1)")
+ testFoldConst("select strright('๐๐', 0)")
+ testFoldConst("select strright('ฮฑฮฒ', -1)")
+ testFoldConst("select strright('ไฝ ๅฅฝ', 0)")
+ testFoldConst("select strright('ใใใซใกใฏ', -1)")
+ testFoldConst("select strright('์๋
ํ์ธ์', 0)")
+ testFoldConst("select strright('ะฟัะธะฒะตั', -1)")
+ testFoldConst("select strright('ืฉืืื', 0)")
+ testFoldConst("select strright('ะฟัะธะฒะตั', 2147483647)")
+ testFoldConst("select strright('ะฟัะธะฒะตั', 2147483648)")
+
// sub_replace
testFoldConst("select sub_replace(CAST('doris' AS STRING), CAST('***' AS
STRING), 1, 2)")
testFoldConst("select sub_replace(CAST('doris' AS STRING), CAST('***' AS
STRING), 1, 2)")
@@ -1602,4 +1709,9 @@ suite("fold_constant_string_arithmatic") {
testFoldConst("select
extract_url_parameter('http://user:[email protected]?๐=b&c=d&e=f&g=h&i=j&k=l',
'')")
testFoldConst("select
extract_url_parameter('http://user:[email protected]?๐=b&c=d&e=f&g=h&i=j&k=l',
null)")
+ // emoji
+ testFoldConst("select replace_empty('๐abc', '', 'def')")
+ testFoldConst("select split_by_string('a๐a๐a', '')")
+ testFoldConst("select character_length('a๐a๐a')")
+ testFoldConst("select replace_empty('a๐a๐a', '', '2')")
}
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
