This is an automated email from the ASF dual-hosted git repository.
yiguolei pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/master by this push:
new db2920a8145 [enhance](mtmv)mtmv support hive default partition (#32051)
db2920a8145 is described below
commit db2920a8145b92c2bc532a8839f73b5b78f6b5bc
Author: zhangdong <[email protected]>
AuthorDate: Tue Mar 12 18:59:39 2024 +0800
[enhance](mtmv)mtmv support hive default partition (#32051)
---
.../org/apache/doris/catalog/PartitionInfo.java | 3 +
.../doris/datasource/hive/HMSExternalTable.java | 6 +-
.../data/mtmv_p0/test_hive_default_mtmv.out | 12 +++
.../suites/mtmv_p0/test_hive_default_mtmv.groovy | 111 +++++++++++++++++++++
4 files changed, 127 insertions(+), 5 deletions(-)
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/catalog/PartitionInfo.java
b/fe/fe-core/src/main/java/org/apache/doris/catalog/PartitionInfo.java
index b7dcba3bf96..70aee07fad2 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/catalog/PartitionInfo.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/PartitionInfo.java
@@ -20,6 +20,7 @@ package org.apache.doris.catalog;
import org.apache.doris.analysis.DateLiteral;
import org.apache.doris.analysis.Expr;
import org.apache.doris.analysis.MaxLiteral;
+import org.apache.doris.analysis.NullLiteral;
import org.apache.doris.analysis.PartitionDesc;
import org.apache.doris.analysis.PartitionValue;
import org.apache.doris.analysis.SinglePartitionDesc;
@@ -374,6 +375,8 @@ public class PartitionInfo implements Writable {
return PartitionValue.MAX_VALUE;
} else if (expr instanceof DateLiteral) {
return new PartitionValue(expr.getStringValue());
+ } else if (expr instanceof NullLiteral) {
+ return new PartitionValue("NULL", true);
} else {
return new PartitionValue(expr.getRealValue().toString());
}
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HMSExternalTable.java
b/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HMSExternalTable.java
index 47b684c264b..50eac67deff 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HMSExternalTable.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HMSExternalTable.java
@@ -105,7 +105,6 @@ public class HMSExternalTable extends ExternalTable
implements MTMVRelatedTableI
private static final String USE_HIVE_SYNC_PARTITION =
"use_hive_sync_partition";
-
static {
SUPPORTED_HIVE_FILE_FORMATS = Sets.newHashSet();
SUPPORTED_HIVE_FILE_FORMATS.add("org.apache.hadoop.hive.ql.io.parquet.MapredParquetInputFormat");
@@ -778,10 +777,7 @@ public class HMSExternalTable extends ExternalTable
implements MTMVRelatedTableI
.getMetaStoreCache((HMSExternalCatalog) getCatalog());
HiveMetaStoreCache.HivePartitionValues hivePartitionValues =
cache.getPartitionValues(
getDbName(), getName(), getPartitionColumnTypes());
-
- return hivePartitionValues.getIdToPartitionItem().entrySet().stream()
- .filter(entry -> !entry.getValue().isDefaultPartition())
- .collect(Collectors.toMap(Map.Entry::getKey,
Map.Entry::getValue));
+ return hivePartitionValues.getIdToPartitionItem();
}
@Override
diff --git a/regression-test/data/mtmv_p0/test_hive_default_mtmv.out
b/regression-test/data/mtmv_p0/test_hive_default_mtmv.out
new file mode 100644
index 00000000000..e8e078dd6b9
--- /dev/null
+++ b/regression-test/data/mtmv_p0/test_hive_default_mtmv.out
@@ -0,0 +1,12 @@
+-- This file is automatically generated. You should know what you did if you
want to edit this
+-- !test --
+1 1 2020 1
+2 2 2020 \N
+
+-- !null --
+2 2 2020 \N
+
+-- !1 --
+1 1 2020 1
+2 2 2020 \N
+
diff --git a/regression-test/suites/mtmv_p0/test_hive_default_mtmv.groovy
b/regression-test/suites/mtmv_p0/test_hive_default_mtmv.groovy
new file mode 100644
index 00000000000..abc3f950ef8
--- /dev/null
+++ b/regression-test/suites/mtmv_p0/test_hive_default_mtmv.groovy
@@ -0,0 +1,111 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+suite("test_hive_default_mtmv",
"p0,external,hive,external_docker,external_docker_hive") {
+ String enabled = context.config.otherConfigs.get("enableHiveTest")
+ if (enabled == null || !enabled.equalsIgnoreCase("true")) {
+ logger.info("diable Hive test.")
+ return;
+ }
+ // prepare data in hive
+ def hive_database = "mtmv_default_partition_db"
+ def hive_table = "test_hive_default_mtmv_t1"
+
+ def drop_table_str = """ drop table if exists
${hive_database}.${hive_table} """
+ def drop_database_str = """ drop database if exists ${hive_database}"""
+ def create_database_str = """ create database ${hive_database}"""
+ def create_table_str = """ CREATE TABLE ${hive_database}.${hive_table} (
+ user_id INT,
+ num INT
+ )
+ partitioned by(year int,month int)
+ STORED AS ORC;
+ """
+
+ def insert_str1 = """
+ insert into ${hive_database}.${hive_table}
PARTITION(year=2020,month) values(1,1,1);
+ """
+ def insert_str2 = """
+ insert into ${hive_database}.${hive_table}
PARTITION(year=2020,month) values(2,2,null);
+ """
+
+ logger.info("hive sql: " + drop_table_str)
+ hive_docker """ ${drop_table_str} """
+ logger.info("hive sql: " + drop_database_str)
+ hive_docker """ ${drop_database_str} """
+ logger.info("hive sql: " + create_database_str)
+ hive_docker """ ${create_database_str}"""
+ logger.info("hive sql: " + create_table_str)
+ hive_docker """ ${create_table_str} """
+ logger.info("hive sql: " + insert_str1)
+ hive_docker """ ${insert_str1} """
+ logger.info("hive sql: " + insert_str2)
+ hive_docker """ ${insert_str2} """
+
+
+ // prepare catalog
+ String hms_port = context.config.otherConfigs.get("hms_port")
+ String catalog_name = "hive_default_mtmv_test_catalog"
+ String externalEnvIp = context.config.otherConfigs.get("externalEnvIp")
+
+ sql """drop catalog if exists ${catalog_name}"""
+ sql """create catalog if not exists ${catalog_name} properties (
+ "type"="hms",
+ 'hive.metastore.uris' = 'thrift://${externalEnvIp}:${hms_port}'
+ );"""
+
+ order_qt_test "SELECT * FROM
${catalog_name}.${hive_database}.${hive_table} order by num"
+
+
+ // prepare mtmv
+ def mvName = "test_hive_default_mtmv"
+ def dbName = "regression_test_mtmv_p0"
+ sql """drop materialized view if exists ${mvName};"""
+
+ sql """
+ CREATE MATERIALIZED VIEW ${mvName}
+ BUILD DEFERRED REFRESH AUTO ON MANUAL
+ partition by(`month`)
+ DISTRIBUTED BY RANDOM BUCKETS 2
+ PROPERTIES ('replication_num' = '1')
+ AS
+ SELECT user_id,num,year,month FROM
${catalog_name}.${hive_database}.${hive_table};
+ """
+ def showPartitionsResult = sql """show partitions from ${mvName}"""
+ logger.info("showPartitionsResult: " + showPartitionsResult.toString())
+ assertTrue(showPartitionsResult.toString().contains("p_NULL"))
+ assertTrue(showPartitionsResult.toString().contains("p_1"))
+
+ sql """
+ REFRESH MATERIALIZED VIEW ${mvName} partitions(p_NULL);
+ """
+ def jobName = getJobName(dbName, mvName);
+ waitingMTMVTaskFinished(jobName)
+ order_qt_null "SELECT * FROM ${mvName} order by num"
+
+ sql """
+ REFRESH MATERIALIZED VIEW ${mvName} partitions(p_1);
+ """
+ waitingMTMVTaskFinished(jobName)
+ order_qt_1 "SELECT * FROM ${mvName} order by num"
+
+ sql """drop materialized view if exists ${mvName};"""
+
+ sql """drop catalog if exists ${catalog_name}"""
+
+}
+
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
