This is an automated email from the ASF dual-hosted git repository. yiguolei pushed a commit to branch branch-2.1 in repository https://gitbox.apache.org/repos/asf/doris.git
commit d5275c55b45ee7ae8828085ace67f6e99feb28a2 Author: zhangstar333 <[email protected]> AuthorDate: Tue Apr 23 20:51:40 2024 +0800 [bug](fold) fix fold date/datetime error as null (#33845) the LocalDateTime/LocalDate value maybe null, so need check it firstly. if it's null, could return NullLiteral directly. --- .../expression/rules/FoldConstantRuleOnBE.java | 50 +++++++++++++--------- .../expressions/functions/scalar/Tokenize.java | 10 ++++- .../query_p0/explain/test_pushdown_explain.groovy | 13 ++++++ 3 files changed, 51 insertions(+), 22 deletions(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/rules/FoldConstantRuleOnBE.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/rules/FoldConstantRuleOnBE.java index ac9d7c4427b..3c2f0d546dd 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/rules/FoldConstantRuleOnBE.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/rules/FoldConstantRuleOnBE.java @@ -33,11 +33,12 @@ import org.apache.doris.nereids.rules.expression.ExpressionMatchingContext; import org.apache.doris.nereids.rules.expression.ExpressionPatternMatcher; import org.apache.doris.nereids.rules.expression.ExpressionPatternRuleFactory; import org.apache.doris.nereids.trees.expressions.Alias; +import org.apache.doris.nereids.trees.expressions.ArrayItemReference; import org.apache.doris.nereids.trees.expressions.Cast; import org.apache.doris.nereids.trees.expressions.Expression; +import org.apache.doris.nereids.trees.expressions.Match; import org.apache.doris.nereids.trees.expressions.functions.generator.TableGeneratingFunction; import org.apache.doris.nereids.trees.expressions.functions.scalar.Sleep; -import org.apache.doris.nereids.trees.expressions.functions.scalar.Tokenize; import org.apache.doris.nereids.trees.expressions.literal.ArrayLiteral; import org.apache.doris.nereids.trees.expressions.literal.BigIntLiteral; import org.apache.doris.nereids.trees.expressions.literal.BooleanLiteral; @@ -197,15 +198,14 @@ public class FoldConstantRuleOnBE implements ExpressionPatternRuleFactory { || expr.getDataType().isIPv6Type()) { return; } - if (skipSleepFunction(expr) || (expr instanceof TableGeneratingFunction)) { + // first need pass PlanTranslatorContext value, + // and ArrayItemReference translate, can't findColumnRef + // Match need give more info rather then as left child a NULL, in + // match_phrase_prefix/MATCH_PHRASE/MATCH_PHRASE/MATCH_ANY + if (skipSleepFunction(expr) || (expr instanceof TableGeneratingFunction) + || (expr instanceof ArrayItemReference) || (expr instanceof Match)) { return; } - // Tokenize function want check the second child literal must be string type - // and properties format, it's a little special, - // maybe check in checkLegalityBeforeTypeCoercion function? - if (expr instanceof Tokenize) { - expr.checkLegalityAfterRewrite(); - } String id = idGenerator.getNextId().toString(); constMap.put(id, expr); Expr staleExpr; @@ -286,7 +286,7 @@ public class FoldConstantRuleOnBE implements ExpressionPatternRuleFactory { List<Literal> resultExpression = getResultExpression(type, resultContent); if (resultExpression.isEmpty()) { ret = constMap.get(e1.getKey()); - LOG.debug("Be constant folding convert {} to {} failed query_id: {}", e1.getKey(), ret, + LOG.warn("Be constant folding convert {} to {} failed query_id: {}", e1.getKey(), ret, DebugUtil.printId(context.queryId())); } else { ret = resultExpression.get(0); @@ -320,8 +320,7 @@ public class FoldConstantRuleOnBE implements ExpressionPatternRuleFactory { if (type.isNullType()) { int num = resultContent.getNullMapCount(); for (int i = 0; i < num; ++i) { - Literal literal = new NullLiteral(type); - res.add(literal); + res.add(new NullLiteral(type)); } } else if (type.isBooleanType()) { int num = resultContent.getUint32ValueCount(); @@ -412,19 +411,27 @@ public class FoldConstantRuleOnBE implements ExpressionPatternRuleFactory { for (int i = 0; i < num; ++i) { long uint64Value = resultContent.getUint64Value(i); LocalDateTime dateTimeV2 = convertToJavaDateTimeV2(uint64Value); - Literal literal = new DateTimeV2Literal((DateTimeV2Type) type, dateTimeV2.getYear(), - dateTimeV2.getMonthValue(), dateTimeV2.getDayOfMonth(), dateTimeV2.getHour(), - dateTimeV2.getMinute(), dateTimeV2.getSecond(), dateTimeV2.getNano() / 1000); - res.add(literal); + if (dateTimeV2 == null && resultContent.hasHasNull()) { + res.add(new NullLiteral(type)); + } else { + Literal literal = new DateTimeV2Literal((DateTimeV2Type) type, dateTimeV2.getYear(), + dateTimeV2.getMonthValue(), dateTimeV2.getDayOfMonth(), dateTimeV2.getHour(), + dateTimeV2.getMinute(), dateTimeV2.getSecond(), dateTimeV2.getNano() / 1000); + res.add(literal); + } } } else if (type.isDateV2Type()) { int num = resultContent.getUint32ValueCount(); for (int i = 0; i < num; ++i) { int uint32Value = resultContent.getUint32Value(i); LocalDate localDate = convertToJavaDateV2(uint32Value); - DateV2Literal dateV2Literal = new DateV2Literal(localDate.getYear(), localDate.getMonthValue(), - localDate.getDayOfMonth()); - res.add(dateV2Literal); + if (localDate == null && resultContent.hasHasNull()) { + res.add(new NullLiteral(type)); + } else { + DateV2Literal dateV2Literal = new DateV2Literal(localDate.getYear(), localDate.getMonthValue(), + localDate.getDayOfMonth()); + res.add(dateV2Literal); + } } } else if (type.isIPv4Type()) { int num = resultContent.getUint32ValueCount(); @@ -438,8 +445,11 @@ public class FoldConstantRuleOnBE implements ExpressionPatternRuleFactory { for (int i = 0; i < num; ++i) { String stringValue = resultContent.getStringValue(i); // maybe need handle NULL_IN_CSV_FOR_ORDINARY_TYPE = "\\N"; - JsonLiteral jsonLiteral = new JsonLiteral(stringValue); - res.add(jsonLiteral); + if ("\\N".equalsIgnoreCase(stringValue) && resultContent.hasHasNull()) { + res.add(new NullLiteral(type)); + } else { + res.add(new JsonLiteral(stringValue)); + } } } else if (type.isStringLikeType()) { int num = resultContent.getStringValueCount(); diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/Tokenize.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/Tokenize.java index 8186f87864f..4902a53210c 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/Tokenize.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/Tokenize.java @@ -23,6 +23,7 @@ import org.apache.doris.nereids.parser.NereidsParser; import org.apache.doris.nereids.trees.expressions.Expression; import org.apache.doris.nereids.trees.expressions.functions.ExplicitlyCastableSignature; import org.apache.doris.nereids.trees.expressions.functions.PropagateNullable; +import org.apache.doris.nereids.trees.expressions.literal.NullLiteral; import org.apache.doris.nereids.trees.expressions.literal.StringLikeLiteral; import org.apache.doris.nereids.trees.expressions.shape.BinaryExpression; import org.apache.doris.nereids.trees.expressions.visitor.ExpressionVisitor; @@ -54,8 +55,13 @@ public class Tokenize extends ScalarFunction } @Override - public void checkLegalityAfterRewrite() { - if (!(child(1) instanceof StringLikeLiteral)) { + public void checkLegalityBeforeTypeCoercion() { + Expression rightChild = child(1); + // tokenize(k7, null) could return NULL + if (rightChild instanceof NullLiteral) { + return; + } + if (!(rightChild instanceof StringLikeLiteral)) { throw new AnalysisException("tokenize second argument must be string literal"); } String properties = ((StringLikeLiteral) child(1)).value; diff --git a/regression-test/suites/query_p0/explain/test_pushdown_explain.groovy b/regression-test/suites/query_p0/explain/test_pushdown_explain.groovy index 93f4662942f..f5ff46b77fe 100644 --- a/regression-test/suites/query_p0/explain/test_pushdown_explain.groovy +++ b/regression-test/suites/query_p0/explain/test_pushdown_explain.groovy @@ -23,4 +23,17 @@ suite("test_pushdown_explain") { contains "PREDICATES:" } qt_select "select k1 from baseall where k1 = 1" + + explain { + sql("select cast(0 as datetime)") + contains "NULL" + } + explain { + sql("select cast(0 as date)") + contains "NULL" + } + explain { + sql("select cast(\"6.a8\" as json)") + contains "NULL" + } } --------------------------------------------------------------------- To unsubscribe, e-mail: [email protected] For additional commands, e-mail: [email protected]
