This is an automated email from the ASF dual-hosted git repository.
panxiaolei pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/master by this push:
new 0691586eb7 [Chore](regression-test) add createMV action && add some mv
case from fe ut MaterializedViewFunctionTest (#16825)
0691586eb7 is described below
commit 0691586eb7d3834a9b405ef75f4b3d2df24f7001
Author: Pxl <[email protected]>
AuthorDate: Fri Feb 24 16:35:37 2023 +0800
[Chore](regression-test) add createMV action && add some mv case from fe ut
MaterializedViewFunctionTest (#16825)
1. add createMV action
2. add some mv case from fe ut MaterializedViewFunctionTest
3. reduce mv scheduler interval time from 10s to 0.3s
---
.../doris/alter/MaterializedViewHandler.java | 4 +-
.../org/apache/doris/alter/BatchRollupJobTest.java | 4 +-
.../ut/testProjectionMV1/testProjectionMV1.out | 11 +++
.../ut/testProjectionMV2/testProjectionMV2.out | 14 +++
.../ut/testProjectionMV3/testProjectionMV3.out | 14 +++
.../ut/testProjectionMV4/testProjectionMV4.out | 12 +++
.../doris/regression/action/CreateMVAction.groovy | 110 +++++++++++++++++++++
.../org/apache/doris/regression/suite/Suite.groovy | 9 ++
.../ut/testProjectionMV1/testProjectionMV1.groovy | 53 ++++++++++
.../ut/testProjectionMV2/testProjectionMV2.groovy | 59 +++++++++++
.../ut/testProjectionMV3/testProjectionMV3.groovy | 61 ++++++++++++
.../ut/testProjectionMV4/testProjectionMV4.groovy | 61 ++++++++++++
12 files changed, 409 insertions(+), 3 deletions(-)
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/alter/MaterializedViewHandler.java
b/fe/fe-core/src/main/java/org/apache/doris/alter/MaterializedViewHandler.java
index c2b654e23f..5a7ff739ca 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/alter/MaterializedViewHandler.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/alter/MaterializedViewHandler.java
@@ -93,8 +93,10 @@ public class MaterializedViewHandler extends AlterHandler {
private static final Logger LOG =
LogManager.getLogger(MaterializedViewHandler.class);
public static final String NEW_STORAGE_FORMAT_INDEX_NAME_PREFIX = "__v2_";
+ public static int scheduler_interval_millisecond = 333;
+
public MaterializedViewHandler() {
- super("materialized view");
+ super("materialized view", scheduler_interval_millisecond);
}
// for batch submit rollup job, tableId -> jobId
diff --git
a/fe/fe-core/src/test/java/org/apache/doris/alter/BatchRollupJobTest.java
b/fe/fe-core/src/test/java/org/apache/doris/alter/BatchRollupJobTest.java
index 987d90215b..ac3522ad83 100644
--- a/fe/fe-core/src/test/java/org/apache/doris/alter/BatchRollupJobTest.java
+++ b/fe/fe-core/src/test/java/org/apache/doris/alter/BatchRollupJobTest.java
@@ -92,7 +92,7 @@ public class BatchRollupJobTest {
while (!alterJobV2.getJobState().isFinalState()) {
System.out.println(
"rollup job " + alterJobV2.getJobId() + " is running.
state: " + alterJobV2.getJobState());
- Thread.sleep(5000);
+ Thread.sleep(1000);
}
System.out.println("rollup job " + alterJobV2.getJobId() + " is
done. state: " + alterJobV2.getJobState());
Assert.assertEquals(AlterJobV2.JobState.FINISHED,
alterJobV2.getJobState());
@@ -142,7 +142,7 @@ public class BatchRollupJobTest {
while (!alterJobV2.getJobState().isFinalState()) {
System.out.println(
"rollup job " + alterJobV2.getJobId() + " is running.
state: " + alterJobV2.getJobState());
- Thread.sleep(5000);
+ Thread.sleep(1000);
}
System.out.println("rollup job " + alterJobV2.getJobId() + " is
done. state: " + alterJobV2.getJobState());
Assert.assertEquals(AlterJobV2.JobState.FINISHED,
alterJobV2.getJobState());
diff --git
a/regression-test/data/materialized_view_p0/ut/testProjectionMV1/testProjectionMV1.out
b/regression-test/data/materialized_view_p0/ut/testProjectionMV1/testProjectionMV1.out
new file mode 100644
index 0000000000..ba455b7e85
--- /dev/null
+++
b/regression-test/data/materialized_view_p0/ut/testProjectionMV1/testProjectionMV1.out
@@ -0,0 +1,11 @@
+-- This file is automatically generated. You should know what you did if you
want to edit this
+-- !select_star --
+2020-01-01 1 a 1 1 1
+2020-01-01 1 a 1 1 1
+2020-01-02 2 b 2 2 2
+
+-- !select_mv --
+1 1
+1 1
+2 2
+
diff --git
a/regression-test/data/materialized_view_p0/ut/testProjectionMV2/testProjectionMV2.out
b/regression-test/data/materialized_view_p0/ut/testProjectionMV2/testProjectionMV2.out
new file mode 100644
index 0000000000..934d3208d4
--- /dev/null
+++
b/regression-test/data/materialized_view_p0/ut/testProjectionMV2/testProjectionMV2.out
@@ -0,0 +1,14 @@
+-- This file is automatically generated. You should know what you did if you
want to edit this
+-- !select_star --
+2020-01-01 1 a 1 1 1
+2020-01-01 1 a 1 1 1
+2020-01-02 2 b 2 2 2
+
+-- !select_mv --
+2
+2
+
+-- !select_base --
+a
+a
+
diff --git
a/regression-test/data/materialized_view_p0/ut/testProjectionMV3/testProjectionMV3.out
b/regression-test/data/materialized_view_p0/ut/testProjectionMV3/testProjectionMV3.out
new file mode 100644
index 0000000000..9537036f41
--- /dev/null
+++
b/regression-test/data/materialized_view_p0/ut/testProjectionMV3/testProjectionMV3.out
@@ -0,0 +1,14 @@
+-- This file is automatically generated. You should know what you did if you
want to edit this
+-- !select_star --
+2020-01-01 1 a 1 1 1
+2020-01-01 1 a 1 1 1
+2020-01-02 2 b 2 2 2
+
+-- !select_mv --
+2 a
+2 a
+
+-- !select_mv2 --
+a
+a
+
diff --git
a/regression-test/data/materialized_view_p0/ut/testProjectionMV4/testProjectionMV4.out
b/regression-test/data/materialized_view_p0/ut/testProjectionMV4/testProjectionMV4.out
new file mode 100644
index 0000000000..45a962ed59
--- /dev/null
+++
b/regression-test/data/materialized_view_p0/ut/testProjectionMV4/testProjectionMV4.out
@@ -0,0 +1,12 @@
+-- This file is automatically generated. You should know what you did if you
want to edit this
+-- !select_star --
+2020-01-01 1 a 1 1 1
+2020-01-01 1 a 1 1 1
+2020-01-02 2 b 2 2 2
+
+-- !select_mv --
+b
+
+-- !select_base --
+2
+
diff --git
a/regression-test/framework/src/main/groovy/org/apache/doris/regression/action/CreateMVAction.groovy
b/regression-test/framework/src/main/groovy/org/apache/doris/regression/action/CreateMVAction.groovy
new file mode 100644
index 0000000000..b4b37df016
--- /dev/null
+++
b/regression-test/framework/src/main/groovy/org/apache/doris/regression/action/CreateMVAction.groovy
@@ -0,0 +1,110 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package org.apache.doris.regression.action
+
+import groovy.transform.stc.ClosureParams
+import groovy.transform.stc.FromString
+import org.apache.doris.regression.suite.SuiteContext
+import org.apache.doris.regression.util.JdbcUtils
+import groovy.util.logging.Slf4j
+import java.sql.ResultSetMetaData
+import java.util.stream.Collectors
+import java.sql.Connection
+
+@Slf4j
+class CreateMVAction implements SuiteAction {
+ private String sql
+ private String exception
+ private SuiteContext context
+
+ CreateMVAction(SuiteContext context, String sql) {
+ this.context = context
+ this.sql = sql
+ }
+
+ CreateMVAction(SuiteContext context, String sql, String exception) {
+ this.context = context
+ this.sql = sql
+ this.exception = exception
+ }
+
+ @Override
+ void run() {
+ def result = doRun(sql)
+ String resultString = result.result
+
+ if (exception != null) {
+ if (result.exception == null) {
+ throw new IllegalStateException("Expect exception on create
mv, but create success");
+ }
+ def msg = result.exception?.toString()
+ Assert.assertTrue("Expect exception msg contains '${exception}',
but meet '${msg}'",
+ msg != null && exception != null &&
msg.contains(exception))
+ return
+ }
+
+ def tryTimes = 0
+ def sqlResult = "null"
+ while (!sqlResult.contains("FINISHED")) {
+ def tmp = doRun("SHOW ALTER TABLE MATERIALIZED VIEW ORDER BY
CreateTime DESC LIMIT 1;")
+ sqlResult = tmp.result[0]
+ log.info("result: ${sqlResult}")
+ if (tryTimes == 60 || sqlResult.contains("CANCELLED")) {
+ throw new IllegalStateException("MV create check times over
limit");
+ }
+ Thread.sleep(1200)
+ tryTimes++
+ }
+
+ Thread.sleep(1000)
+ }
+
+ ActionResult doRun(String sql) {
+ Connection conn = context.getConnection()
+ List<List<Object>> result = null
+ ResultSetMetaData meta = null
+ Throwable ex = null
+
+ long startTime = System.currentTimeMillis()
+ try {
+ log.info("sql:\n${sql}".toString())
+ (result, meta) = JdbcUtils.executeToList(conn, sql)
+ } catch (Throwable t) {
+ ex = t
+ }
+ long endTime = System.currentTimeMillis()
+
+ return new ActionResult(result, ex, startTime, endTime, meta)
+ }
+
+ class ActionResult {
+ List<List<Object>> result
+ Throwable exception
+ long startTime
+ long endTime
+ ResultSetMetaData meta
+
+ ActionResult(List<List<Object>> result, Throwable exception, long
startTime, long endTime, ResultSetMetaData meta) {
+ this.result = result
+ this.exception = exception
+ this.startTime = startTime
+ this.endTime = endTime
+ this.meta = meta
+ }
+ }
+}
diff --git
a/regression-test/framework/src/main/groovy/org/apache/doris/regression/suite/Suite.groovy
b/regression-test/framework/src/main/groovy/org/apache/doris/regression/suite/Suite.groovy
index 0cbde97baa..c4a1eaf23e 100644
---
a/regression-test/framework/src/main/groovy/org/apache/doris/regression/suite/Suite.groovy
+++
b/regression-test/framework/src/main/groovy/org/apache/doris/regression/suite/Suite.groovy
@@ -25,6 +25,7 @@ import com.google.common.collect.ImmutableList
import org.apache.doris.regression.action.BenchmarkAction
import org.apache.doris.regression.util.DataUtils
import org.apache.doris.regression.util.OutputUtils
+import org.apache.doris.regression.action.CreateMVAction
import org.apache.doris.regression.action.ExplainAction
import org.apache.doris.regression.action.RestoreAction
import org.apache.doris.regression.action.StreamLoadAction
@@ -271,6 +272,14 @@ class Suite implements GroovyInterceptable {
runAction(new ExplainAction(context), actionSupplier)
}
+ void createMV(String sql) {
+ (new CreateMVAction(context, sql)).run()
+ }
+
+ void createMV(String sql, String expection) {
+ (new CreateMVAction(context, sql, expection)).run()
+ }
+
void test(Closure actionSupplier) {
runAction(new TestAction(context), actionSupplier)
}
diff --git
a/regression-test/suites/materialized_view_p0/ut/testProjectionMV1/testProjectionMV1.groovy
b/regression-test/suites/materialized_view_p0/ut/testProjectionMV1/testProjectionMV1.groovy
new file mode 100644
index 0000000000..c23b533832
--- /dev/null
+++
b/regression-test/suites/materialized_view_p0/ut/testProjectionMV1/testProjectionMV1.groovy
@@ -0,0 +1,53 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+import org.codehaus.groovy.runtime.IOGroovyMethods
+
+suite ("testProjectionMV1") {
+ sql """ DROP TABLE IF EXISTS emps; """
+
+ sql """
+ create table emps (
+ time_col date,
+ empid int,
+ name varchar,
+ deptno int,
+ salary int,
+ commission int)
+ partition by range (time_col) (partition p1 values less than
MAXVALUE) distributed by hash(time_col) buckets 3 properties('replication_num'
= '1');
+ """
+
+ sql """insert into emps values("2020-01-01",1,"a",1,1,1);"""
+ sql """insert into emps values("2020-01-02",2,"b",2,2,2);"""
+
+ createMV("create materialized view emps_mv as select deptno, empid from
emps order by deptno;")
+
+ sql """insert into emps values("2020-01-01",1,"a",1,1,1);"""
+
+ explain {
+ sql("select * from emps order by empid;")
+ contains "(emps)"
+ }
+ qt_select_star "select * from emps order by empid;"
+
+
+ explain {
+ sql("select empid, deptno from emps order by empid;")
+ contains "(emps_mv)"
+ }
+ qt_select_mv "select empid, deptno from emps order by empid;"
+}
diff --git
a/regression-test/suites/materialized_view_p0/ut/testProjectionMV2/testProjectionMV2.groovy
b/regression-test/suites/materialized_view_p0/ut/testProjectionMV2/testProjectionMV2.groovy
new file mode 100644
index 0000000000..4b1936f5e0
--- /dev/null
+++
b/regression-test/suites/materialized_view_p0/ut/testProjectionMV2/testProjectionMV2.groovy
@@ -0,0 +1,59 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+import org.codehaus.groovy.runtime.IOGroovyMethods
+
+suite ("testProjectionMV2") {
+ sql """ DROP TABLE IF EXISTS emps; """
+
+ sql """
+ create table emps (
+ time_col date,
+ empid int,
+ name varchar,
+ deptno int,
+ salary int,
+ commission int)
+ partition by range (time_col) (partition p1 values less than
MAXVALUE) distributed by hash(time_col) buckets 3 properties('replication_num'
= '1');
+ """
+
+ sql """insert into emps values("2020-01-01",1,"a",1,1,1);"""
+ sql """insert into emps values("2020-01-02",2,"b",2,2,2);"""
+
+ createMV("create materialized view emps_mv as select deptno, empid from
emps order by deptno;")
+
+ sql """insert into emps values("2020-01-01",1,"a",1,1,1);"""
+
+ explain {
+ sql("select * from emps order by empid;")
+ contains "(emps)"
+ }
+ qt_select_star "select * from emps order by empid;"
+
+
+ explain {
+ sql("select empid + 1 from emps where deptno = 1 order by empid;")
+ contains "(emps_mv)"
+ }
+ qt_select_mv "select empid + 1 from emps where deptno = 1 order by empid;"
+
+ explain {
+ sql("select name from emps where deptno -1 = 0 order by empid;")
+ contains "(emps)"
+ }
+ qt_select_base "select name from emps where deptno -1 = 0 order by empid;"
+}
diff --git
a/regression-test/suites/materialized_view_p0/ut/testProjectionMV3/testProjectionMV3.groovy
b/regression-test/suites/materialized_view_p0/ut/testProjectionMV3/testProjectionMV3.groovy
new file mode 100644
index 0000000000..261168e48f
--- /dev/null
+++
b/regression-test/suites/materialized_view_p0/ut/testProjectionMV3/testProjectionMV3.groovy
@@ -0,0 +1,61 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+import org.codehaus.groovy.runtime.IOGroovyMethods
+
+suite ("testProjectionMV3") {
+ sql """ DROP TABLE IF EXISTS emps; """
+
+ sql """
+ create table emps (
+ time_col date,
+ empid int,
+ name varchar,
+ deptno int,
+ salary int,
+ commission int)
+ partition by range (time_col) (partition p1 values less than
MAXVALUE) distributed by hash(time_col) buckets 3 properties('replication_num'
= '1');
+ """
+
+ sql """insert into emps values("2020-01-01",1,"a",1,1,1);"""
+ sql """insert into emps values("2020-01-02",2,"b",2,2,2);"""
+
+ def result = "null"
+
+ createMV("create materialized view emps_mv as select deptno, empid, name
from emps order by deptno;")
+
+ sql """insert into emps values("2020-01-01",1,"a",1,1,1);"""
+
+ explain {
+ sql("select * from emps order by empid;")
+ contains "(emps)"
+ }
+ qt_select_star "select * from emps order by empid;"
+
+
+ explain {
+ sql("select empid + 1, name from emps where deptno = 1 order by
empid;")
+ contains "(emps_mv)"
+ }
+ qt_select_mv "select empid + 1, name from emps where deptno = 1 order by
empid;"
+
+ explain {
+ sql("select name from emps where deptno -1 = 0 order by empid;")
+ contains "(emps_mv)"
+ }
+ qt_select_mv2 "select name from emps where deptno -1 = 0 order by empid;"
+}
diff --git
a/regression-test/suites/materialized_view_p0/ut/testProjectionMV4/testProjectionMV4.groovy
b/regression-test/suites/materialized_view_p0/ut/testProjectionMV4/testProjectionMV4.groovy
new file mode 100644
index 0000000000..969e547630
--- /dev/null
+++
b/regression-test/suites/materialized_view_p0/ut/testProjectionMV4/testProjectionMV4.groovy
@@ -0,0 +1,61 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+import org.codehaus.groovy.runtime.IOGroovyMethods
+
+suite ("testProjectionMV4") {
+ sql """ DROP TABLE IF EXISTS emps; """
+
+ sql """
+ create table emps (
+ time_col date,
+ empid int,
+ name varchar,
+ deptno int,
+ salary int,
+ commission int)
+ partition by range (time_col) (partition p1 values less than
MAXVALUE) distributed by hash(time_col) buckets 3 properties('replication_num'
= '1');
+ """
+
+ sql """insert into emps values("2020-01-01",1,"a",1,1,1);"""
+ sql """insert into emps values("2020-01-02",2,"b",2,2,2);"""
+
+ def result = "null"
+
+ createMV("create materialized view emps_mv as select name, deptno, salary
from emps;")
+
+ sql """insert into emps values("2020-01-01",1,"a",1,1,1);"""
+
+ explain {
+ sql("select * from emps order by empid;")
+ contains "(emps)"
+ }
+ qt_select_star "select * from emps order by empid;"
+
+
+ explain {
+ sql("select name from emps where deptno > 1 and salary > 1 order by
name;")
+ contains "(emps_mv)"
+ }
+ qt_select_mv "select name from emps where deptno > 1 and salary > 1 order
by name;"
+
+ explain {
+ sql("select empid from emps where deptno > 1 and empid > 1 order by
empid;")
+ contains "(emps)"
+ }
+ qt_select_base "select empid from emps where deptno > 1 and empid > 1
order by empid;"
+}
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
