This is an automated email from the ASF dual-hosted git repository.
morrysnow pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/master by this push:
new 7942bd0bf9 [fix](planner) cast string literal to date like type should
not be an implict cast (#20709)
7942bd0bf9 is described below
commit 7942bd0bf9fba005848094cd6fdfe517eca34de9
Author: starocean999 <[email protected]>
AuthorDate: Tue Jun 13 17:57:14 2023 +0800
[fix](planner) cast string literal to date like type should not be an
implict cast (#20709)
1. cast string literal to date like type should not be an implict cast
2. the string representation of float like type should not be scientific
notation
3. the data type of like function's regex expr should be string type even
if it's a null literal
4. add -Xss4m in fe.conf to prevent stack overflow in some case
---
conf/fe.conf | 4 +-
fe/fe-core/src/main/cup/sql_parser.cup | 6 +-
.../org/apache/doris/analysis/FloatLiteral.java | 5 +-
.../org/apache/doris/analysis/LikePredicate.java | 2 +
.../apache/doris/analysis/ArrayLiteralTest.java | 2 +-
.../org/apache/doris/planner/QueryPlanTest.java | 2 +-
.../doris/rewrite/RewriteDateLiteralRuleTest.java | 8 +-
.../data/correctness_p0/test_cast_null.out | 13 ++++
.../suites/correctness_p0/test_cast_null.groovy | 87 ++++++++++++++++++++++
9 files changed, 117 insertions(+), 12 deletions(-)
diff --git a/conf/fe.conf b/conf/fe.conf
index 5894320ced..e5a097d974 100644
--- a/conf/fe.conf
+++ b/conf/fe.conf
@@ -26,10 +26,10 @@ CUR_DATE=`date +%Y%m%d-%H%M%S`
# the output dir of stderr and stdout
LOG_DIR = ${DORIS_HOME}/log
-JAVA_OPTS="-Xmx8192m -XX:+UseMembar -XX:SurvivorRatio=8
-XX:MaxTenuringThreshold=7 -XX:+PrintGCDateStamps -XX:+PrintGCDetails
-XX:+UseConcMarkSweepGC -XX:+UseParNewGC -XX:+CMSClassUnloadingEnabled
-XX:-CMSParallelRemarkEnabled -XX:CMSInitiatingOccupancyFraction=80
-XX:SoftRefLRUPolicyMSPerMB=0 -Xloggc:$DORIS_HOME/log/fe.gc.log.$CUR_DATE"
+JAVA_OPTS="-Xss4m -Xmx8192m -XX:+UseMembar -XX:SurvivorRatio=8
-XX:MaxTenuringThreshold=7 -XX:+PrintGCDateStamps -XX:+PrintGCDetails
-XX:+UseConcMarkSweepGC -XX:+UseParNewGC -XX:+CMSClassUnloadingEnabled
-XX:-CMSParallelRemarkEnabled -XX:CMSInitiatingOccupancyFraction=80
-XX:SoftRefLRUPolicyMSPerMB=0 -Xloggc:$DORIS_HOME/log/fe.gc.log.$CUR_DATE"
# For jdk 9+, this JAVA_OPTS will be used as default JVM options
-JAVA_OPTS_FOR_JDK_9="-Xmx8192m -XX:SurvivorRatio=8 -XX:MaxTenuringThreshold=7
-XX:+CMSClassUnloadingEnabled -XX:-CMSParallelRemarkEnabled
-XX:CMSInitiatingOccupancyFraction=80 -XX:SoftRefLRUPolicyMSPerMB=0
-Xlog:gc*:$DORIS_HOME/log/fe.gc.log.$CUR_DATE:time"
+JAVA_OPTS_FOR_JDK_9="-Xss4m -Xmx8192m -XX:SurvivorRatio=8
-XX:MaxTenuringThreshold=7 -XX:+CMSClassUnloadingEnabled
-XX:-CMSParallelRemarkEnabled -XX:CMSInitiatingOccupancyFraction=80
-XX:SoftRefLRUPolicyMSPerMB=0
-Xlog:gc*:$DORIS_HOME/log/fe.gc.log.$CUR_DATE:time"
##
## the lowercase properties are read by main program.
diff --git a/fe/fe-core/src/main/cup/sql_parser.cup
b/fe/fe-core/src/main/cup/sql_parser.cup
index 3d0f8dd392..85c2aace7f 100644
--- a/fe/fe-core/src/main/cup/sql_parser.cup
+++ b/fe/fe-core/src/main/cup/sql_parser.cup
@@ -6468,11 +6468,11 @@ non_pred_expr ::=
| function_call_expr:e
{: RESULT = e; :}
| KW_DATE STRING_LITERAL:l
- {: RESULT = new CastExpr(Type.DATE, new StringLiteral(l)); :}
+ {: RESULT = new CastExpr(TypeDef.create(PrimitiveType.DATE), new
StringLiteral(l)); :}
| KW_DATEV2 STRING_LITERAL:l
- {: RESULT = new CastExpr(Type.DATEV2, new StringLiteral(l)); :}
+ {: RESULT = new CastExpr(TypeDef.create(PrimitiveType.DATEV2), new
StringLiteral(l)); :}
| KW_TIMESTAMP STRING_LITERAL:l
- {: RESULT = new CastExpr(Type.DATETIME, new StringLiteral(l)); :}
+ {: RESULT = new CastExpr(TypeDef.create(PrimitiveType.DATETIME), new
StringLiteral(l)); :}
| KW_EXTRACT LPAREN function_name:fn_name KW_FROM func_arg_list:exprs RPAREN
{: RESULT = new FunctionCallExpr(fn_name, exprs); :}
//| function_name:fn_name LPAREN RPAREN
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/analysis/FloatLiteral.java
b/fe/fe-core/src/main/java/org/apache/doris/analysis/FloatLiteral.java
index 5cbf8ab4c6..c87505b22d 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/analysis/FloatLiteral.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/FloatLiteral.java
@@ -33,6 +33,7 @@ import java.io.IOException;
import java.math.BigDecimal;
import java.nio.ByteBuffer;
import java.nio.ByteOrder;
+import java.text.NumberFormat;
public class FloatLiteral extends LiteralExpr {
private double value;
@@ -134,7 +135,9 @@ public class FloatLiteral extends LiteralExpr {
if (type.equals(Type.TIME) || type.equals(Type.TIMEV2)) {
return timeStrFromFloat(value);
}
- return Double.toString(value);
+ NumberFormat nf = NumberFormat.getInstance();
+ nf.setGroupingUsed(false);
+ return nf.format(value);
}
@Override
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/analysis/LikePredicate.java
b/fe/fe-core/src/main/java/org/apache/doris/analysis/LikePredicate.java
index 22a84d9eb3..e81772d874 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/analysis/LikePredicate.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/LikePredicate.java
@@ -146,6 +146,8 @@ public class LikePredicate extends Predicate {
} catch (PatternSyntaxException e) {
throw new AnalysisException("Invalid regular expression in '"
+ this.toSql() + "'");
}
+ } else if (getChild(1) instanceof NullLiteral) {
+ getChild(1).setType(Type.STRING);
}
}
diff --git
a/fe/fe-core/src/test/java/org/apache/doris/analysis/ArrayLiteralTest.java
b/fe/fe-core/src/test/java/org/apache/doris/analysis/ArrayLiteralTest.java
index 73e4f8f843..53b9ede98d 100644
--- a/fe/fe-core/src/test/java/org/apache/doris/analysis/ArrayLiteralTest.java
+++ b/fe/fe-core/src/test/java/org/apache/doris/analysis/ArrayLiteralTest.java
@@ -35,7 +35,7 @@ public class ArrayLiteralTest {
DateLiteral dateLiteral = new DateLiteral("2022-10-10", Type.DATE);
DateLiteral datetimeLiteral = new DateLiteral("2022-10-10 12:10:10",
Type.DATETIME);
ArrayLiteral arrayLiteral1 = new ArrayLiteral(intLiteral1,
floatLiteral);
- Assert.assertEquals("[\"1.0\", \"2.15\"]",
arrayLiteral1.getStringValueForArray());
+ Assert.assertEquals("[\"1\", \"2.15\"]",
arrayLiteral1.getStringValueForArray());
ArrayLiteral arrayLiteral2 = new ArrayLiteral(boolLiteral,
boolLiteral);
Assert.assertEquals("[\"1\", \"1\"]",
arrayLiteral2.getStringValueForArray());
diff --git
a/fe/fe-core/src/test/java/org/apache/doris/planner/QueryPlanTest.java
b/fe/fe-core/src/test/java/org/apache/doris/planner/QueryPlanTest.java
index ac4373a490..1144cdeb08 100644
--- a/fe/fe-core/src/test/java/org/apache/doris/planner/QueryPlanTest.java
+++ b/fe/fe-core/src/test/java/org/apache/doris/planner/QueryPlanTest.java
@@ -953,7 +953,7 @@ public class QueryPlanTest extends TestWithFeService {
// 3.1 test float,float in case expr
String sql31 = "select case cast(100 as float) when 1 then 'a' when 2
then 'b' else 'other' end as col31;";
Assert.assertTrue(StringUtils.containsIgnoreCase(getSQLPlanOrErrorMsg("explain
" + sql31),
- "constant exprs: \n CASE 100.0 WHEN 1.0 THEN 'a' WHEN
2.0 THEN 'b' ELSE 'other' END"));
+ "constant exprs: \n CASE 100 WHEN 1 THEN 'a' WHEN 2
THEN 'b' ELSE 'other' END"));
// 4.1 test null in case expr return else
String sql41 = "select case null when 1 then 'a' when 2 then 'b' else
'other' end as col41";
diff --git
a/fe/fe-core/src/test/java/org/apache/doris/rewrite/RewriteDateLiteralRuleTest.java
b/fe/fe-core/src/test/java/org/apache/doris/rewrite/RewriteDateLiteralRuleTest.java
index 55a875fd2d..e8e550dd2d 100644
---
a/fe/fe-core/src/test/java/org/apache/doris/rewrite/RewriteDateLiteralRuleTest.java
+++
b/fe/fe-core/src/test/java/org/apache/doris/rewrite/RewriteDateLiteralRuleTest.java
@@ -168,21 +168,21 @@ public class RewriteDateLiteralRuleTest {
public void testWithDoubleFormatDate() throws Exception {
String query = "select * from " + DB_NAME + ".tb1 where k1 >
20210301.22";
String planString = dorisAssert.query(query).explainQuery();
- Assert.assertTrue(planString.contains("`k1` > 2.021030122E7"));
+ Assert.assertTrue(planString.contains("`k1` > 20210301.22"));
query = "select k1 > 20210331.22 from " + DB_NAME + ".tb1";
planString = dorisAssert.query(query).explainQuery();
- Assert.assertTrue(planString.contains("`k1` > 2.021033122E7"));
+ Assert.assertTrue(planString.contains("`k1` > 20210331.22"));
}
public void testWithDoubleFormatDateV2() throws Exception {
String query = "select * from " + DB_NAME + ".tb2 where k1 >
20210301.22";
String planString = dorisAssert.query(query).explainQuery();
- Assert.assertTrue(planString.contains("`k1` > 2.021030122E7"));
+ Assert.assertTrue(planString.contains("`k1` > 20210301.22"));
query = "select k1 > 20210331.22 from " + DB_NAME + ".tb2";
planString = dorisAssert.query(query).explainQuery();
- Assert.assertTrue(planString.contains("`k1` > 2.021033122E7"));
+ Assert.assertTrue(planString.contains("`k1` > 20210331.22"));
}
public void testWithInvalidFormatDate() throws Exception {
diff --git a/regression-test/data/correctness_p0/test_cast_null.out
b/regression-test/data/correctness_p0/test_cast_null.out
new file mode 100644
index 0000000000..1a685b0c3a
--- /dev/null
+++ b/regression-test/data/correctness_p0/test_cast_null.out
@@ -0,0 +1,13 @@
+-- This file is automatically generated. You should know what you did if you
want to edit this
+-- !sql1 --
+19700506
+
+-- !sql2 --
+\N
+
+-- !sql3 --
+19700506
+
+-- !sql4 --
+\N
+
diff --git a/regression-test/suites/correctness_p0/test_cast_null.groovy
b/regression-test/suites/correctness_p0/test_cast_null.groovy
new file mode 100644
index 0000000000..4bf78a30c8
--- /dev/null
+++ b/regression-test/suites/correctness_p0/test_cast_null.groovy
@@ -0,0 +1,87 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+suite("test_cast_null") {
+ sql """
+ drop table if exists test_table_t53;
+ """
+
+ sql """
+ drop table if exists test_table_t0;
+ """
+
+ sql """
+ CREATE TABLE `test_table_t53` (
+ `c0` char(238) NOT NULL
+ ) ENGINE=OLAP
+ AGGREGATE KEY(`c0`)
+ COMMENT 'OLAP'
+ DISTRIBUTED BY HASH(`c0`) BUCKETS 7
+ PROPERTIES (
+ "replication_allocation" = "tag.location.default: 1",
+ "storage_format" = "V2",
+ "light_schema_change" = "true",
+ "disable_auto_compaction" = "false",
+ "enable_single_replica_compaction" = "false"
+ );
+ """
+
+ sql """
+ CREATE TABLE `test_table_t0` (
+ `c0` varchar(238) NULL
+ ) ENGINE=OLAP
+ DUPLICATE KEY(`c0`)
+ COMMENT 'OLAP'
+ DISTRIBUTED BY HASH(`c0`) BUCKETS 14
+ PROPERTIES (
+ "replication_allocation" = "tag.location.default: 1",
+ "storage_format" = "V2",
+ "light_schema_change" = "true",
+ "disable_auto_compaction" = "false",
+ "enable_single_replica_compaction" = "false"
+ );
+ """
+
+ sql """
+ set enable_nereids_planner=false;
+ """
+
+ explain {
+ sql """SELECT * FROM test_table_t53 LEFT JOIN test_table_t0 ON (('I4')
LIKE (CAST(CAST(DATE '1970-05-06' AS FLOAT) AS VARCHAR) ));"""
+ contains "19700506"
+ }
+
+ qt_sql1 """
+ select CAST(CAST(DATE '1970-05-06' AS FLOAT) AS VARCHAR);
+ """
+
+ qt_sql2 """
+ select 'abc' like null;
+ """
+
+ sql """
+ set enable_nereids_planner=true;
+ """
+
+ qt_sql3 """
+ select CAST(CAST(DATE '1970-05-06' AS FLOAT) AS VARCHAR);
+ """
+
+ qt_sql4 """
+ select 'abc' like null;
+ """
+}
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
