This is an automated email from the ASF dual-hosted git repository.
yiguolei pushed a commit to branch branch-2.1
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/branch-2.1 by this push:
new 79056d4d7a1 branch-2.1: [feat](hive) add catalog level partition cache
property #50724 (#50762)
79056d4d7a1 is described below
commit 79056d4d7a1541f33ee4b3f87d8ce73cc65b2613
Author: github-actions[bot]
<41898282+github-actions[bot]@users.noreply.github.com>
AuthorDate: Fri May 9 22:01:49 2025 +0800
branch-2.1: [feat](hive) add catalog level partition cache property #50724
(#50762)
Cherry-picked from #50724
Co-authored-by: Mingyu Chen (Rayner) <[email protected]>
---
.../main/java/org/apache/doris/common/Config.java | 2 +-
.../doris/datasource/hive/HMSExternalCatalog.java | 29 ++-
.../doris/datasource/hive/HiveMetaStoreCache.java | 17 +-
.../hive/test_hive_meta_cache.out | Bin 0 -> 858 bytes
.../export/hive_read/orc/test_hive_read_orc.groovy | 2 +-
.../hive/test_hive_meta_cache.groovy | 235 +++++++++++++++++++++
.../hive/test_hive_star_qualifier.groovy | 2 +-
7 files changed, 269 insertions(+), 18 deletions(-)
diff --git a/fe/fe-common/src/main/java/org/apache/doris/common/Config.java
b/fe/fe-common/src/main/java/org/apache/doris/common/Config.java
index f25bb4de3b3..509a82b5467 100644
--- a/fe/fe-common/src/main/java/org/apache/doris/common/Config.java
+++ b/fe/fe-common/src/main/java/org/apache/doris/common/Config.java
@@ -2086,7 +2086,7 @@ public class Config extends ConfigBase {
"Max cache number of partition at table level in Hive Metastore."})
public static long max_hive_partition_cache_num = 10000;
- @ConfField(description = {"Hudi/Iceberg 表级别缓存的最大数量。",
+ @ConfField(description = {"Hudi/Iceberg/Paimon 表级别缓存的最大数量。",
"Max cache number of hudi/iceberg table."})
public static long max_external_table_cache_num = 1000;
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HMSExternalCatalog.java
b/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HMSExternalCatalog.java
index 505436903ce..61a7c030477 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HMSExternalCatalog.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HMSExternalCatalog.java
@@ -73,20 +73,21 @@ public class HMSExternalCatalog extends ExternalCatalog {
private static final Logger LOG =
LogManager.getLogger(HMSExternalCatalog.class);
public static final String FILE_META_CACHE_TTL_SECOND =
"file.meta.cache.ttl-second";
+ public static final String PARTITION_CACHE_TTL_SECOND =
"partition.cache.ttl-second";
// broker name for file split and query scan.
public static final String BIND_BROKER_NAME = "broker.name";
// Default is false, if set to true, will get table schema from
"remoteTable" instead of from hive metastore.
- // This is because for some forward compatiblity issue of hive metastore,
there maybe
+ // This is because for some forward compatibility issue of hive metastore,
there maybe
// "storage schema reading not support" error being thrown.
// set this to true can avoid this error.
// But notice that if set to true, the default value of column will be
ignored because we cannot get default value
// from remoteTable object.
public static final String GET_SCHEMA_FROM_TABLE = "get_schema_from_table";
- // -1 means file cache no ttl set
- public static final int FILE_META_CACHE_NO_TTL = -1;
- // 0 means file cache is disabled; >0 means file cache with ttl;
- public static final int FILE_META_CACHE_TTL_DISABLE_CACHE = 0;
+ // -1 means cache with no ttl
+ public static final int CACHE_NO_TTL = -1;
+ // 0 means cache is disabled; >0 means cache with ttl;
+ public static final int CACHE_TTL_DISABLE_CACHE = 0;
private static final int FILE_SYSTEM_EXECUTOR_THREAD_NUM = 16;
private ThreadPoolExecutor fileSystemExecutor;
@@ -119,11 +120,20 @@ public class HMSExternalCatalog extends ExternalCatalog {
super.checkProperties();
// check file.meta.cache.ttl-second parameter
String fileMetaCacheTtlSecond =
catalogProperty.getOrDefault(FILE_META_CACHE_TTL_SECOND, null);
- if (Objects.nonNull(fileMetaCacheTtlSecond) &&
NumberUtils.toInt(fileMetaCacheTtlSecond, FILE_META_CACHE_NO_TTL)
- < FILE_META_CACHE_TTL_DISABLE_CACHE) {
+ if (Objects.nonNull(fileMetaCacheTtlSecond) &&
NumberUtils.toInt(fileMetaCacheTtlSecond, CACHE_NO_TTL)
+ < CACHE_TTL_DISABLE_CACHE) {
throw new DdlException(
"The parameter " + FILE_META_CACHE_TTL_SECOND + " is
wrong, value is " + fileMetaCacheTtlSecond);
}
+
+ // check partition.cache.ttl-second parameter
+ String partitionCacheTtlSecond =
catalogProperty.getOrDefault(PARTITION_CACHE_TTL_SECOND, null);
+ if (Objects.nonNull(partitionCacheTtlSecond) &&
NumberUtils.toInt(partitionCacheTtlSecond, CACHE_NO_TTL)
+ < CACHE_TTL_DISABLE_CACHE) {
+ throw new DdlException(
+ "The parameter " + PARTITION_CACHE_TTL_SECOND + " is
wrong, value is " + partitionCacheTtlSecond);
+ }
+
// check the dfs.ha properties
// 'dfs.nameservices'='your-nameservice',
// 'dfs.ha.namenodes.your-nameservice'='nn1,nn2',
@@ -282,8 +292,9 @@ public class HMSExternalCatalog extends ExternalCatalog {
public void notifyPropertiesUpdated(Map<String, String> updatedProps) {
super.notifyPropertiesUpdated(updatedProps);
String fileMetaCacheTtl =
updatedProps.getOrDefault(FILE_META_CACHE_TTL_SECOND, null);
- if (Objects.nonNull(fileMetaCacheTtl)) {
-
Env.getCurrentEnv().getExtMetaCacheMgr().getMetaStoreCache(this).setNewFileCache();
+ String partitionCacheTtl =
updatedProps.getOrDefault(PARTITION_CACHE_TTL_SECOND, null);
+ if (Objects.nonNull(fileMetaCacheTtl) ||
Objects.nonNull(partitionCacheTtl)) {
+
Env.getCurrentEnv().getExtMetaCacheMgr().getMetaStoreCache(this).init();
}
}
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HiveMetaStoreCache.java
b/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HiveMetaStoreCache.java
index ecb0fa60693..9f49c1b7f2e 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HiveMetaStoreCache.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HiveMetaStoreCache.java
@@ -136,14 +136,19 @@ public class HiveMetaStoreCache {
* we need to be very careful and try to avoid the circular dependency of
these tasks
* which will bring out thread deadlock.
**/
- private void init() {
+ public void init() {
+ long partitionCacheTtlSecond = NumberUtils.toLong(
+
(catalog.getProperties().get(HMSExternalCatalog.PARTITION_CACHE_TTL_SECOND)),
+ HMSExternalCatalog.CACHE_NO_TTL);
+
CacheFactory partitionValuesCacheFactory = new CacheFactory(
- OptionalLong.of(28800L),
+ OptionalLong.of(partitionCacheTtlSecond >=
HMSExternalCatalog.CACHE_TTL_DISABLE_CACHE
+ ? partitionCacheTtlSecond : 28800L),
OptionalLong.of(Config.external_cache_expire_time_minutes_after_access * 60L),
Config.max_hive_partition_table_cache_num,
true,
null);
- partitionValuesCache = partitionValuesCacheFactory.buildCache(key ->
loadPartitionValues(key), null,
+ partitionValuesCache =
partitionValuesCacheFactory.buildCache(this::loadPartitionValues, null,
refreshExecutor);
CacheFactory partitionCacheFactory = new CacheFactory(
@@ -170,16 +175,16 @@ public class HiveMetaStoreCache {
/***
* generate a filecache and set to fileCacheRef
*/
- public void setNewFileCache() {
+ private void setNewFileCache() {
// init or refresh job conf
setJobConf();
// if the file.meta.cache.ttl-second is equal or greater than 0, the
cache expired will be set to that value
int fileMetaCacheTtlSecond = NumberUtils.toInt(
(catalog.getProperties().get(HMSExternalCatalog.FILE_META_CACHE_TTL_SECOND)),
- HMSExternalCatalog.FILE_META_CACHE_NO_TTL);
+ HMSExternalCatalog.CACHE_NO_TTL);
CacheFactory fileCacheFactory = new CacheFactory(
- OptionalLong.of(fileMetaCacheTtlSecond >=
HMSExternalCatalog.FILE_META_CACHE_TTL_DISABLE_CACHE
+ OptionalLong.of(fileMetaCacheTtlSecond >=
HMSExternalCatalog.CACHE_TTL_DISABLE_CACHE
? fileMetaCacheTtlSecond : 28800L),
OptionalLong.of(Config.external_cache_expire_time_minutes_after_access * 60L),
Config.max_external_file_cache_num,
diff --git
a/regression-test/data/external_table_p0/hive/test_hive_meta_cache.out
b/regression-test/data/external_table_p0/hive/test_hive_meta_cache.out
new file mode 100644
index 00000000000..7ab9a456bdc
Binary files /dev/null and
b/regression-test/data/external_table_p0/hive/test_hive_meta_cache.out differ
diff --git
a/regression-test/suites/external_table_p0/export/hive_read/orc/test_hive_read_orc.groovy
b/regression-test/suites/external_table_p0/export/hive_read/orc/test_hive_read_orc.groovy
index dabdf815518..2c613bd63bd 100644
---
a/regression-test/suites/external_table_p0/export/hive_read/orc/test_hive_read_orc.groovy
+++
b/regression-test/suites/external_table_p0/export/hive_read/orc/test_hive_read_orc.groovy
@@ -21,7 +21,7 @@ import java.nio.charset.StandardCharsets
import java.nio.file.Files
import java.nio.file.Paths
-suite("test_hive_read_orc", "external,hive,external_docker") {
+suite("test_hive_read_orc",
"p0,external,hive,external_docker,external_docker_hive") {
String enabled = context.config.otherConfigs.get("enableHiveTest")
if (enabled == null || !enabled.equalsIgnoreCase("true")) {
diff --git
a/regression-test/suites/external_table_p0/hive/test_hive_meta_cache.groovy
b/regression-test/suites/external_table_p0/hive/test_hive_meta_cache.groovy
new file mode 100644
index 00000000000..3b6655f6e39
--- /dev/null
+++ b/regression-test/suites/external_table_p0/hive/test_hive_meta_cache.groovy
@@ -0,0 +1,235 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+suite("test_hive_meta_cache",
"p0,external,hive,external_docker,external_docker_hive") {
+ String catalog_name = "test_hive_meta_cache"
+ String catalog_name_no_cache = "test_hive_meta_no_cache"
+
+ String enabled = context.config.otherConfigs.get("enableHiveTest")
+ if (enabled != null && enabled.equalsIgnoreCase("true")) {
+ for (String hivePrefix : ["hive3"]) {
+ setHivePrefix(hivePrefix)
+ String externalEnvIp =
context.config.otherConfigs.get("externalEnvIp")
+ String hmsPort = context.config.otherConfigs.get(hivePrefix +
"HmsPort")
+ String hdfs_port = context.config.otherConfigs.get(hivePrefix +
"HdfsPort")
+
+ // 1. test default catalog
+ sql """drop catalog if exists ${catalog_name};"""
+ sql """
+ create catalog ${catalog_name} properties (
+ 'type'='hms',
+ 'hadoop.username' = 'hadoop',
+ 'hive.metastore.uris' = 'thrift://${externalEnvIp}:${hmsPort}',
+ 'fs.defaultFS' = 'hdfs://${externalEnvIp}:${hdfs_port}'
+ );
+ """
+ sql """switch ${catalog_name}"""
+ hive_docker """drop database if exists test_hive_meta_cache_db
CASCADE"""
+ hive_docker """create database test_hive_meta_cache_db"""
+ hive_docker """
+ CREATE TABLE test_hive_meta_cache_db.sales (
+ id INT,
+ amount DOUBLE
+ )
+ PARTITIONED BY (year INT)
+ """
+ hive_docker """ set hive.stats.column.autogather=false """
+ hive_docker """insert into test_hive_meta_cache_db.sales
partition(year=2024) values(1, 2.0)"""
+ // select 1 row
+ order_qt_sql_1row """select * from test_hive_meta_cache_db.sales"""
+ // insert into same partition
+ hive_docker """insert into test_hive_meta_cache_db.sales values(2,
2.0, 2024)"""
+ // still select 1 row
+ order_qt_sql_1row """select * from test_hive_meta_cache_db.sales"""
+ // insert into new partition
+ hive_docker """insert into test_hive_meta_cache_db.sales values(1,
3.0, 2025)"""
+ // still select 1 row
+ order_qt_sql_1row """select * from test_hive_meta_cache_db.sales"""
+ sql """refresh table test_hive_meta_cache_db.sales"""
+ // select 3 rows
+ order_qt_sql_3row """select * from test_hive_meta_cache_db.sales"""
+ sql """drop table test_hive_meta_cache_db.sales"""
+
+ // 2. test catalog with file.meta.cache.ttl-second
+ sql """drop catalog if exists ${catalog_name_no_cache};"""
+ test {
+ sql """
+ create catalog ${catalog_name_no_cache} properties (
+ 'type'='hms',
+ 'hadoop.username' = 'hadoop',
+ 'hive.metastore.uris' =
'thrift://${externalEnvIp}:${hmsPort}',
+ 'fs.defaultFS' = 'hdfs://${externalEnvIp}:${hdfs_port}',
+ 'file.meta.cache.ttl-second' = '-2'
+ );
+ """
+ exception "is wrong"
+ }
+
+ // disable file list cache
+ sql """
+ create catalog ${catalog_name_no_cache} properties (
+ 'type'='hms',
+ 'hadoop.username' = 'hadoop',
+ 'hive.metastore.uris' = 'thrift://${externalEnvIp}:${hmsPort}',
+ 'fs.defaultFS' = 'hdfs://${externalEnvIp}:${hdfs_port}',
+ 'file.meta.cache.ttl-second' = '0'
+ );
+ """
+ sql """switch ${catalog_name_no_cache}"""
+ hive_docker """drop database if exists test_hive_meta_cache_db
CASCADE"""
+ hive_docker """create database test_hive_meta_cache_db"""
+ hive_docker """
+ CREATE TABLE test_hive_meta_cache_db.sales (
+ id INT,
+ amount DOUBLE
+ )
+ PARTITIONED BY (year INT)
+ STORED AS PARQUET;
+ """
+ hive_docker """insert into test_hive_meta_cache_db.sales values(1,
2.0, 2024)"""
+ // select 1 row
+ order_qt_sql_1row """select * from test_hive_meta_cache_db.sales"""
+ // insert into same partition
+ hive_docker """insert into test_hive_meta_cache_db.sales values(2,
2.0, 2024)"""
+ // select 2 rows
+ order_qt_sql_2row """select * from test_hive_meta_cache_db.sales"""
+ // insert into new partition
+ hive_docker """insert into test_hive_meta_cache_db.sales values(1,
3.0, 2025)"""
+ // still select 2 rows
+ order_qt_sql_2row """select * from test_hive_meta_cache_db.sales"""
+ sql """refresh table test_hive_meta_cache_db.sales"""
+ // select 3 rows
+ order_qt_sql_3row """select * from test_hive_meta_cache_db.sales"""
+ sql """drop table test_hive_meta_cache_db.sales"""
+
+ // 3. test catalog with partition.cache.ttl-second
+ sql """drop catalog if exists ${catalog_name_no_cache};"""
+ test {
+ sql """
+ create catalog ${catalog_name_no_cache} properties (
+ 'type'='hms',
+ 'hadoop.username' = 'hadoop',
+ 'hive.metastore.uris' =
'thrift://${externalEnvIp}:${hmsPort}',
+ 'fs.defaultFS' = 'hdfs://${externalEnvIp}:${hdfs_port}',
+ 'partition.cache.ttl-second' = '-2'
+ );
+ """
+ exception "is wrong"
+ }
+
+ // disable partition cache
+ sql """
+ create catalog ${catalog_name_no_cache} properties (
+ 'type'='hms',
+ 'hadoop.username' = 'hadoop',
+ 'hive.metastore.uris' = 'thrift://${externalEnvIp}:${hmsPort}',
+ 'fs.defaultFS' = 'hdfs://${externalEnvIp}:${hdfs_port}',
+ 'partition.cache.ttl-second' = '0'
+ );
+ """
+ sql """switch ${catalog_name_no_cache}"""
+ hive_docker """drop database if exists test_hive_meta_cache_db
CASCADE"""
+ hive_docker """create database test_hive_meta_cache_db"""
+ hive_docker """
+ CREATE TABLE test_hive_meta_cache_db.sales (
+ id INT,
+ amount DOUBLE
+ )
+ PARTITIONED BY (year INT)
+ STORED AS PARQUET;
+ """
+ hive_docker """insert into test_hive_meta_cache_db.sales values(1,
2.0, 2024)"""
+ // select 1 row
+ order_qt_sql_1row """select * from test_hive_meta_cache_db.sales"""
+ // insert into same partition
+ hive_docker """insert into test_hive_meta_cache_db.sales values(2,
2.0, 2024)"""
+ // still select 1 row
+ order_qt_sql_1row """select * from test_hive_meta_cache_db.sales"""
+ // insert into new partition
+ hive_docker """insert into test_hive_meta_cache_db.sales values(1,
3.0, 2025)"""
+ // select 2 rows
+ order_qt_sql_2row """select * from test_hive_meta_cache_db.sales"""
+ sql """refresh table test_hive_meta_cache_db.sales"""
+ // select 3 rows
+ order_qt_sql_3row """select * from test_hive_meta_cache_db.sales"""
+ sql """drop table test_hive_meta_cache_db.sales"""
+
+ // test modify ttl property
+ sql """drop catalog if exists ${catalog_name_no_cache};"""
+ // 1. create catalog with default property fisrt
+ sql """
+ create catalog ${catalog_name_no_cache} properties (
+ 'type'='hms',
+ 'hadoop.username' = 'hadoop',
+ 'hive.metastore.uris' = 'thrift://${externalEnvIp}:${hmsPort}',
+ 'fs.defaultFS' = 'hdfs://${externalEnvIp}:${hdfs_port}'
+ );
+ """
+ sql """switch ${catalog_name_no_cache}"""
+ hive_docker """drop database if exists test_hive_meta_cache_db
CASCADE"""
+ hive_docker """create database test_hive_meta_cache_db"""
+ hive_docker """
+ CREATE TABLE test_hive_meta_cache_db.sales (
+ id INT,
+ amount DOUBLE
+ )
+ PARTITIONED BY (year INT)
+ STORED AS PARQUET;
+ """
+ hive_docker """insert into test_hive_meta_cache_db.sales values(1,
2.0, 2024)"""
+ // select 1 row
+ order_qt_sql_1row """select * from test_hive_meta_cache_db.sales"""
+ // insert into same partition
+ hive_docker """insert into test_hive_meta_cache_db.sales values(2,
2.0, 2024)"""
+ // still select 1 row
+ order_qt_sql_1row """select * from test_hive_meta_cache_db.sales"""
+ // alter wrong catalog property
+ test {
+ sql """alter catalog ${catalog_name_no_cache} set properties
("file.meta.cache.ttl-second" = "-2")"""
+ exception "is wrong"
+ }
+ // alter catalog property, disable file list cache
+ sql """alter catalog ${catalog_name_no_cache} set properties
("file.meta.cache.ttl-second" = "0")"""
+ // select 2 rows
+ order_qt_sql_2row """select * from test_hive_meta_cache_db.sales"""
+ // insert into same partition
+ hive_docker """insert into test_hive_meta_cache_db.sales values(3,
2.0, 2024)"""
+ // select 3 row
+ order_qt_sql_3row """select * from test_hive_meta_cache_db.sales"""
+
+ // insert into new partition
+ hive_docker """insert into test_hive_meta_cache_db.sales values(1,
3.0, 2025)"""
+ // still select 3 rows
+ order_qt_sql_3row """select * from test_hive_meta_cache_db.sales"""
+ // alter wrong catalog property
+ test {
+ sql """alter catalog ${catalog_name_no_cache} set properties
("partition.cache.ttl-second" = "-2")"""
+ exception "is wrong"
+ }
+ // alter catalog property, disable partition cache
+ sql """alter catalog ${catalog_name_no_cache} set properties
("partition.cache.ttl-second" = "0")"""
+ // select 4 rows
+ order_qt_sql_4row """select * from test_hive_meta_cache_db.sales"""
+ // insert into new partition
+ hive_docker """insert into test_hive_meta_cache_db.sales values(1,
4.0, 2026)"""
+ // select 5 rows
+ order_qt_sql_5row """select * from test_hive_meta_cache_db.sales"""
+ sql """drop table test_hive_meta_cache_db.sales"""
+ }
+ }
+}
+
diff --git
a/regression-test/suites/external_table_p0/hive/test_hive_star_qualifier.groovy
b/regression-test/suites/external_table_p0/hive/test_hive_star_qualifier.groovy
index 6146469c61d..5ff4f31ccee 100644
---
a/regression-test/suites/external_table_p0/hive/test_hive_star_qualifier.groovy
+++
b/regression-test/suites/external_table_p0/hive/test_hive_star_qualifier.groovy
@@ -60,7 +60,7 @@ suite("test_hive_star_qualifier",
"p0,external,hive,external_docker,external_doc
qt_test10 test10
qt_test11 test11
qt_test12 test12
- sql """drop catalog if exists ${catalog_name};"""
+ // sql """drop catalog if exists ${catalog_name};"""
}
}
}
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
