This is an automated email from the ASF dual-hosted git repository.
dataroaring pushed a commit to branch branch-3.0
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/branch-3.0 by this push:
new a160dd234c0 branch-3.0: [feat](hive) add catalog level schema cache
property #50958 #51057 (#51011)
a160dd234c0 is described below
commit a160dd234c051d52820a66f55576cc4d655f3f97
Author: github-actions[bot]
<41898282+github-actions[bot]@users.noreply.github.com>
AuthorDate: Tue Jun 24 15:21:37 2025 +0800
branch-3.0: [feat](hive) add catalog level schema cache property #50958
#51057 (#51011)
Cherry-picked from #50958 #51057
---------
Co-authored-by: Mingyu Chen (Rayner) <[email protected]>
---
.../apache/doris/datasource/ExternalCatalog.java | 29 ++++-
.../doris/datasource/ExternalMetaCacheMgr.java | 35 ++++--
.../doris/datasource/ExternalSchemaCache.java | 8 +-
.../doris/datasource/hive/HMSExternalCatalog.java | 5 -
.../doris/datasource/hive/HMSExternalTable.java | 11 +-
.../doris/datasource/hive/HiveMetaStoreCache.java | 9 +-
.../datasource/hive/HiveMetaStoreClientHelper.java | 5 +-
.../java/org/apache/doris/qe/ShowExecutor.java | 2 +-
.../hive/test_hive_meta_cache.out | Bin 858 -> 2098 bytes
.../hive/test_hive_meta_cache.groovy | 127 +++++++++++++++++++++
10 files changed, 196 insertions(+), 35 deletions(-)
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalCatalog.java
b/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalCatalog.java
index 0bf975a75d4..0b391ca0435 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalCatalog.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalCatalog.java
@@ -75,6 +75,7 @@ import com.google.gson.annotations.SerializedName;
import lombok.Data;
import org.apache.commons.lang3.NotImplementedException;
import org.apache.commons.lang3.StringUtils;
+import org.apache.commons.lang3.math.NumberUtils;
import org.apache.hadoop.conf.Configuration;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
@@ -116,6 +117,11 @@ public abstract class ExternalCatalog
//
https://help.aliyun.com/zh/emr/emr-on-ecs/user-guide/use-rootpolicy-to-access-oss-hdfs?spm=a2c4g.11186623.help-menu-search-28066.d_0
public static final String OOS_ROOT_POLICY = "oss.root_policy";
+ public static final String SCHEMA_CACHE_TTL_SECOND =
"schema.cache.ttl-second";
+ // -1 means cache with no ttl
+ public static final int CACHE_NO_TTL = -1;
+ // 0 means cache is disabled; >0 means cache with ttl;
+ public static final int CACHE_TTL_DISABLE_CACHE = 0;
// Properties that should not be shown in the `show create catalog` result
public static final Set<String> HIDDEN_PROPERTIES = Sets.newHashSet(
@@ -351,7 +357,7 @@ public abstract class ExternalCatalog
Map<String, String> properties = getCatalogProperty().getProperties();
if (properties.containsKey(CatalogMgr.METADATA_REFRESH_INTERVAL_SEC)) {
try {
- Integer metadataRefreshIntervalSec = Integer.valueOf(
+ int metadataRefreshIntervalSec = Integer.parseInt(
properties.get(CatalogMgr.METADATA_REFRESH_INTERVAL_SEC));
if (metadataRefreshIntervalSec < 0) {
throw new DdlException("Invalid properties: " +
CatalogMgr.METADATA_REFRESH_INTERVAL_SEC);
@@ -361,11 +367,13 @@ public abstract class ExternalCatalog
}
}
- // if (properties.getOrDefault(ExternalCatalog.USE_META_CACHE,
"true").equals("false")) {
- // LOG.warn("force to set use_meta_cache to true for catalog: {}
when creating", name);
- //
getCatalogProperty().addProperty(ExternalCatalog.USE_META_CACHE, "true");
- // useMetaCache = Optional.of(true);
- // }
+ // check schema.cache.ttl-second parameter
+ String schemaCacheTtlSecond =
catalogProperty.getOrDefault(SCHEMA_CACHE_TTL_SECOND, null);
+ if (java.util.Objects.nonNull(schemaCacheTtlSecond) &&
NumberUtils.toInt(schemaCacheTtlSecond, CACHE_NO_TTL)
+ < CACHE_TTL_DISABLE_CACHE) {
+ throw new DdlException(
+ "The parameter " + SCHEMA_CACHE_TTL_SECOND + " is wrong,
value is " + schemaCacheTtlSecond);
+ }
}
/**
@@ -1185,4 +1193,13 @@ public abstract class ExternalCatalog
public int hashCode() {
return Objects.hashCode(name);
}
+
+ @Override
+ public void notifyPropertiesUpdated(Map<String, String> updatedProps) {
+ CatalogIf.super.notifyPropertiesUpdated(updatedProps);
+ String schemaCacheTtl =
updatedProps.getOrDefault(SCHEMA_CACHE_TTL_SECOND, null);
+ if (java.util.Objects.nonNull(schemaCacheTtl)) {
+ Env.getCurrentEnv().getExtMetaCacheMgr().invalidSchemaCache(id);
+ }
+ }
}
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalMetaCacheMgr.java
b/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalMetaCacheMgr.java
index c2f50f929f8..b640aa08f76 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalMetaCacheMgr.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalMetaCacheMgr.java
@@ -88,7 +88,7 @@ public class ExternalMetaCacheMgr {
// catalog id -> HiveMetaStoreCache
private final Map<Long, HiveMetaStoreCache> cacheMap =
Maps.newConcurrentMap();
// catalog id -> table schema cache
- private Map<Long, ExternalSchemaCache> schemaCacheMap = Maps.newHashMap();
+ private final Map<Long, ExternalSchemaCache> schemaCacheMap =
Maps.newHashMap();
// hudi partition manager
private final HudiMetadataCacheMgr hudiMetadataCacheMgr;
// all catalogs could share the same fsCache.
@@ -221,8 +221,10 @@ public class ExternalMetaCacheMgr {
if (cacheMap.remove(catalogId) != null) {
LOG.info("remove hive metastore cache for catalog {}", catalogId);
}
- if (schemaCacheMap.remove(catalogId) != null) {
- LOG.info("remove schema cache for catalog {}", catalogId);
+ synchronized (schemaCacheMap) {
+ if (schemaCacheMap.remove(catalogId) != null) {
+ LOG.info("remove schema cache for catalog {}", catalogId);
+ }
}
hudiMetadataCacheMgr.removeCache(catalogId);
icebergMetadataCacheMgr.removeCache(catalogId);
@@ -232,9 +234,11 @@ public class ExternalMetaCacheMgr {
public void invalidateTableCache(long catalogId, String dbName, String
tblName) {
dbName = ClusterNamespace.getNameFromFullName(dbName);
- ExternalSchemaCache schemaCache = schemaCacheMap.get(catalogId);
- if (schemaCache != null) {
- schemaCache.invalidateTableCache(dbName, tblName);
+ synchronized (schemaCacheMap) {
+ ExternalSchemaCache schemaCache = schemaCacheMap.get(catalogId);
+ if (schemaCache != null) {
+ schemaCache.invalidateTableCache(dbName, tblName);
+ }
}
HiveMetaStoreCache metaCache = cacheMap.get(catalogId);
if (metaCache != null) {
@@ -251,9 +255,11 @@ public class ExternalMetaCacheMgr {
public void invalidateDbCache(long catalogId, String dbName) {
dbName = ClusterNamespace.getNameFromFullName(dbName);
- ExternalSchemaCache schemaCache = schemaCacheMap.get(catalogId);
- if (schemaCache != null) {
- schemaCache.invalidateDbCache(dbName);
+ synchronized (schemaCacheMap) {
+ ExternalSchemaCache schemaCache = schemaCacheMap.get(catalogId);
+ if (schemaCache != null) {
+ schemaCache.invalidateDbCache(dbName);
+ }
}
HiveMetaStoreCache metaCache = cacheMap.get(catalogId);
if (metaCache != null) {
@@ -269,9 +275,8 @@ public class ExternalMetaCacheMgr {
}
public void invalidateCatalogCache(long catalogId) {
- ExternalSchemaCache schemaCache = schemaCacheMap.get(catalogId);
- if (schemaCache != null) {
- schemaCache.invalidateAll();
+ synchronized (schemaCacheMap) {
+ schemaCacheMap.remove(catalogId);
}
HiveMetaStoreCache metaCache = cacheMap.get(catalogId);
if (metaCache != null) {
@@ -286,6 +291,12 @@ public class ExternalMetaCacheMgr {
}
}
+ public void invalidSchemaCache(long catalogId) {
+ synchronized (schemaCacheMap) {
+ schemaCacheMap.remove(catalogId);
+ }
+ }
+
public void addPartitionsCache(long catalogId, HMSExternalTable table,
List<String> partitionNames) {
String dbName =
ClusterNamespace.getNameFromFullName(table.getDbName());
HiveMetaStoreCache metaCache = cacheMap.get(catalogId);
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalSchemaCache.java
b/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalSchemaCache.java
index de3eeff75d9..0fb66c4e3f9 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalSchemaCache.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalSchemaCache.java
@@ -28,6 +28,7 @@ import org.apache.doris.metric.MetricRepo;
import com.github.benmanes.caffeine.cache.LoadingCache;
import com.google.common.collect.ImmutableList;
import lombok.Data;
+import org.apache.commons.lang3.math.NumberUtils;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
@@ -51,13 +52,16 @@ public class ExternalSchemaCache {
}
private void init(ExecutorService executor) {
+ long schemaCacheTtlSecond = NumberUtils.toLong(
+
(catalog.getProperties().get(ExternalCatalog.SCHEMA_CACHE_TTL_SECOND)),
ExternalCatalog.CACHE_NO_TTL);
CacheFactory schemaCacheFactory = new CacheFactory(
- OptionalLong.of(86400L),
+ OptionalLong.of(schemaCacheTtlSecond >=
ExternalCatalog.CACHE_TTL_DISABLE_CACHE
+ ? schemaCacheTtlSecond : 86400),
OptionalLong.of(Config.external_cache_expire_time_minutes_after_access * 60),
Config.max_external_schema_cache_num,
false,
null);
- schemaCache = schemaCacheFactory.buildCache(key -> loadSchema(key),
null, executor);
+ schemaCache = schemaCacheFactory.buildCache(this::loadSchema, null,
executor);
}
private void initMetrics() {
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HMSExternalCatalog.java
b/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HMSExternalCatalog.java
index 20d43263316..2442eed0b9c 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HMSExternalCatalog.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HMSExternalCatalog.java
@@ -81,11 +81,6 @@ public class HMSExternalCatalog extends ExternalCatalog {
// from remoteTable object.
public static final String GET_SCHEMA_FROM_TABLE = "get_schema_from_table";
- // -1 means cache with no ttl
- public static final int CACHE_NO_TTL = -1;
- // 0 means cache is disabled; >0 means cache with ttl;
- public static final int CACHE_TTL_DISABLE_CACHE = 0;
-
private static final int FILE_SYSTEM_EXECUTOR_THREAD_NUM = 16;
private ThreadPoolExecutor fileSystemExecutor;
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HMSExternalTable.java
b/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HMSExternalTable.java
index 35627273f93..b7353a59ddf 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HMSExternalTable.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HMSExternalTable.java
@@ -579,7 +579,7 @@ public class HMSExternalTable extends ExternalTable
implements MTMVRelatedTableI
List<FieldSchema> schema = null;
Map<String, String> colDefaultValues = Maps.newHashMap();
if (getFromTable) {
- schema = getSchemaFromRemoteTable(remoteTable);
+ schema = getSchemaFromRemoteTable();
} else {
HMSCachedClient client = ((HMSExternalCatalog)
catalog).getClient();
schema = client.getSchema(dbName, name);
@@ -597,10 +597,13 @@ public class HMSExternalTable extends ExternalTable
implements MTMVRelatedTableI
return Optional.of(new HMSSchemaCacheValue(columns, partitionColumns));
}
- private static List<FieldSchema> getSchemaFromRemoteTable(Table table) {
+ private List<FieldSchema> getSchemaFromRemoteTable() {
+ // Here we should get a new remote table instead of using
this.remoteTable
+ // Because we need to get the latest schema from HMS.
+ Table newTable = ((HMSExternalCatalog)
catalog).getClient().getTable(dbName, name);
List<FieldSchema> schema = Lists.newArrayList();
- schema.addAll(table.getSd().getCols());
- schema.addAll(table.getPartitionKeys());
+ schema.addAll(newTable.getSd().getCols());
+ schema.addAll(newTable.getPartitionKeys());
return schema;
}
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HiveMetaStoreCache.java
b/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HiveMetaStoreCache.java
index 6ef18c9f545..de867bfc2b9 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HiveMetaStoreCache.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HiveMetaStoreCache.java
@@ -37,6 +37,7 @@ import org.apache.doris.common.util.CacheBulkLoader;
import org.apache.doris.common.util.LocationPath;
import org.apache.doris.common.util.Util;
import org.apache.doris.datasource.CacheException;
+import org.apache.doris.datasource.ExternalCatalog;
import org.apache.doris.datasource.ExternalMetaCacheMgr;
import org.apache.doris.datasource.hive.AcidInfo.DeleteDeltaInfo;
import org.apache.doris.datasource.hive.HiveUtil.ACIDFileFilter;
@@ -139,10 +140,10 @@ public class HiveMetaStoreCache {
public void init() {
long partitionCacheTtlSecond = NumberUtils.toLong(
(catalog.getProperties().get(HMSExternalCatalog.PARTITION_CACHE_TTL_SECOND)),
- HMSExternalCatalog.CACHE_NO_TTL);
+ ExternalCatalog.CACHE_NO_TTL);
CacheFactory partitionValuesCacheFactory = new CacheFactory(
- OptionalLong.of(partitionCacheTtlSecond >=
HMSExternalCatalog.CACHE_TTL_DISABLE_CACHE
+ OptionalLong.of(partitionCacheTtlSecond >=
ExternalCatalog.CACHE_TTL_DISABLE_CACHE
? partitionCacheTtlSecond : 28800L),
OptionalLong.of(Config.external_cache_expire_time_minutes_after_access * 60L),
Config.max_hive_partition_table_cache_num,
@@ -181,10 +182,10 @@ public class HiveMetaStoreCache {
// if the file.meta.cache.ttl-second is equal or greater than 0, the
cache expired will be set to that value
int fileMetaCacheTtlSecond = NumberUtils.toInt(
(catalog.getProperties().get(HMSExternalCatalog.FILE_META_CACHE_TTL_SECOND)),
- HMSExternalCatalog.CACHE_NO_TTL);
+ ExternalCatalog.CACHE_NO_TTL);
CacheFactory fileCacheFactory = new CacheFactory(
- OptionalLong.of(fileMetaCacheTtlSecond >=
HMSExternalCatalog.CACHE_TTL_DISABLE_CACHE
+ OptionalLong.of(fileMetaCacheTtlSecond >=
ExternalCatalog.CACHE_TTL_DISABLE_CACHE
? fileMetaCacheTtlSecond : 28800L),
OptionalLong.of(Config.external_cache_expire_time_minutes_after_access * 60L),
Config.max_external_file_cache_num,
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HiveMetaStoreClientHelper.java
b/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HiveMetaStoreClientHelper.java
index aed35dc47ad..eb63aa1e541 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HiveMetaStoreClientHelper.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HiveMetaStoreClientHelper.java
@@ -712,7 +712,10 @@ public class HiveMetaStoreClientHelper {
return Type.UNSUPPORTED;
}
- public static String
showCreateTable(org.apache.hadoop.hive.metastore.api.Table remoteTable) {
+ public static String showCreateTable(HMSExternalTable hmsTable) {
+ // Always use the latest schema
+ HMSExternalCatalog catalog = (HMSExternalCatalog)
hmsTable.getCatalog();
+ Table remoteTable = catalog.getClient().getTable(hmsTable.getDbName(),
hmsTable.getRemoteName());
StringBuilder output = new StringBuilder();
if (remoteTable.isSetViewOriginalText() ||
remoteTable.isSetViewExpandedText()) {
output.append(String.format("CREATE VIEW `%s` AS ",
remoteTable.getTableName()));
diff --git a/fe/fe-core/src/main/java/org/apache/doris/qe/ShowExecutor.java
b/fe/fe-core/src/main/java/org/apache/doris/qe/ShowExecutor.java
index 2087fd283c3..4be871f7af8 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/qe/ShowExecutor.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/qe/ShowExecutor.java
@@ -1156,7 +1156,7 @@ public class ShowExecutor {
try {
if (table.getType() == TableType.HMS_EXTERNAL_TABLE) {
rows.add(Arrays.asList(table.getName(),
-
HiveMetaStoreClientHelper.showCreateTable(((HMSExternalTable)
table).getRemoteTable())));
+
HiveMetaStoreClientHelper.showCreateTable((HMSExternalTable) table)));
resultSet = new ShowResultSet(showStmt.getMetaData(), rows);
return;
}
diff --git
a/regression-test/data/external_table_p0/hive/test_hive_meta_cache.out
b/regression-test/data/external_table_p0/hive/test_hive_meta_cache.out
index 7ab9a456bdc..7e031ce471d 100644
Binary files
a/regression-test/data/external_table_p0/hive/test_hive_meta_cache.out and
b/regression-test/data/external_table_p0/hive/test_hive_meta_cache.out differ
diff --git
a/regression-test/suites/external_table_p0/hive/test_hive_meta_cache.groovy
b/regression-test/suites/external_table_p0/hive/test_hive_meta_cache.groovy
index 3b6655f6e39..aa5ba31af17 100644
--- a/regression-test/suites/external_table_p0/hive/test_hive_meta_cache.groovy
+++ b/regression-test/suites/external_table_p0/hive/test_hive_meta_cache.groovy
@@ -229,6 +229,133 @@ suite("test_hive_meta_cache",
"p0,external,hive,external_docker,external_docker_
// select 5 rows
order_qt_sql_5row """select * from test_hive_meta_cache_db.sales"""
sql """drop table test_hive_meta_cache_db.sales"""
+
+ // test schema cache
+ sql """drop catalog if exists ${catalog_name_no_cache};"""
+ // 1. create catalog with default property fisrt
+ sql """
+ create catalog ${catalog_name_no_cache} properties (
+ 'type'='hms',
+ 'hadoop.username' = 'hadoop',
+ 'hive.metastore.uris' = 'thrift://${externalEnvIp}:${hmsPort}',
+ 'fs.defaultFS' = 'hdfs://${externalEnvIp}:${hdfs_port}'
+ );
+ """
+ sql """switch ${catalog_name_no_cache}"""
+ hive_docker """drop database if exists test_hive_meta_cache_db
CASCADE"""
+ hive_docker """create database test_hive_meta_cache_db"""
+ hive_docker """
+ CREATE TABLE test_hive_meta_cache_db.sales (
+ id INT,
+ amount DOUBLE
+ )
+ PARTITIONED BY (year INT)
+ STORED AS PARQUET;
+ """
+ // desc table, 3 columns
+ qt_sql_3col "desc test_hive_meta_cache_db.sales";
+ // add a new column in hive
+ hive_docker "alter table test_hive_meta_cache_db.sales add
columns(k3 string)"
+ // desc table, still 3 columns
+ qt_sql_3col "desc test_hive_meta_cache_db.sales";
+ // refresh and check
+ sql "refresh table test_hive_meta_cache_db.sales";
+ // desc table, 4 columns
+ qt_sql_4col "desc test_hive_meta_cache_db.sales";
+
+ // create catalog without schema cache
+ sql """drop catalog if exists ${catalog_name_no_cache};"""
+ sql """
+ create catalog ${catalog_name_no_cache} properties (
+ 'type'='hms',
+ 'hadoop.username' = 'hadoop',
+ 'hive.metastore.uris' = 'thrift://${externalEnvIp}:${hmsPort}',
+ 'fs.defaultFS' = 'hdfs://${externalEnvIp}:${hdfs_port}',
+ 'schema.cache.ttl-second' = '0'
+ );
+ """
+ sql """switch ${catalog_name_no_cache}"""
+ // desc table, 4 columns
+ qt_sql_4col "desc test_hive_meta_cache_db.sales";
+ // add a new column in hive
+ hive_docker "alter table test_hive_meta_cache_db.sales add
columns(k4 string)"
+ // desc table, 5 columns
+ qt_sql_5col "desc test_hive_meta_cache_db.sales";
+
+ // modify property
+ // alter wrong catalog property
+ test {
+ sql """alter catalog ${catalog_name_no_cache} set properties
("schema.cache.ttl-second" = "-2")"""
+ exception "is wrong"
+ }
+ sql """alter catalog ${catalog_name_no_cache} set properties
("schema.cache.ttl-second" = "0")"""
+ // desc table, 5 columns
+ qt_sql_5col "desc test_hive_meta_cache_db.sales";
+ // add a new column in hive
+ hive_docker "alter table test_hive_meta_cache_db.sales add
columns(k5 string)"
+ // desc table, 6 columns
+ qt_sql_6col "desc test_hive_meta_cache_db.sales";
+ sql """drop table test_hive_meta_cache_db.sales"""
+
+ // test schema cache with get_schema_from_table
+ sql """drop catalog if exists ${catalog_name_no_cache};"""
+ // 1. create catalog with schema cache off and
get_schema_from_table
+ sql """
+ create catalog ${catalog_name_no_cache} properties (
+ 'type'='hms',
+ 'hadoop.username' = 'hadoop',
+ 'hive.metastore.uris' = 'thrift://${externalEnvIp}:${hmsPort}',
+ 'fs.defaultFS' = 'hdfs://${externalEnvIp}:${hdfs_port}',
+ 'schema.cache.ttl-second' = '0',
+ 'get_schema_from_table' = 'true'
+ );
+ """
+ sql """switch ${catalog_name_no_cache}"""
+ hive_docker """drop database if exists test_hive_meta_cache_db
CASCADE"""
+ hive_docker """create database test_hive_meta_cache_db"""
+ hive_docker """
+ CREATE TABLE test_hive_meta_cache_db.sales (
+ id INT,
+ amount DOUBLE
+ )
+ PARTITIONED BY (year INT)
+ STORED AS PARQUET;
+ """
+ // desc table, 3 columns
+ qt_sql_3col "desc test_hive_meta_cache_db.sales";
+ // show create table , 3 columns
+ def sql_sct01_3col = sql "show create table
test_hive_meta_cache_db.sales"
+ println "${sql_sct01_3col}"
+ assertTrue(sql_sct01_3col[0][1].contains("CREATE TABLE `sales`(\n
`id` int,\n `amount` double)\nPARTITIONED BY (\n `year` int)"));
+
+ // add a new column in hive
+ hive_docker "alter table test_hive_meta_cache_db.sales add
columns(k1 string)"
+ // desc table, 4 columns
+ qt_sql_4col "desc test_hive_meta_cache_db.sales";
+ // show create table, 4 columns
+ def sql_sct01_4col = sql "show create table
test_hive_meta_cache_db.sales"
+ println "${sql_sct01_4col}"
+ assertTrue(sql_sct01_4col[0][1].contains("CREATE TABLE `sales`(\n
`id` int,\n `amount` double,\n `k1` string)\nPARTITIONED BY (\n `year`
int)"));
+
+ // open schema cache
+ sql """alter catalog ${catalog_name_no_cache} set properties
("schema.cache.ttl-second" = "120")"""
+ // add a new column in hive
+ hive_docker "alter table test_hive_meta_cache_db.sales add
columns(k2 string)"
+ // desc table, 5 columns
+ qt_sql_5col "desc test_hive_meta_cache_db.sales";
+ // show create table, 5 columns
+ def sql_sct01_5col = sql "show create table
test_hive_meta_cache_db.sales"
+ println "${sql_sct01_5col}"
+ assertTrue(sql_sct01_5col[0][1].contains("CREATE TABLE `sales`(\n
`id` int,\n `amount` double,\n `k1` string,\n `k2` string)\nPARTITIONED BY
(\n `year` int)"));
+ // add a new column in hive
+ hive_docker "alter table test_hive_meta_cache_db.sales add
columns(k3 string)"
+ // desc table, still 5 columns
+ qt_sql_5col "desc test_hive_meta_cache_db.sales";
+ // show create table always see latest schema, 6 columns
+ def sql_sct01_6col = sql "show create table
test_hive_meta_cache_db.sales"
+ println "${sql_sct01_6col}"
+ assertTrue(sql_sct01_6col[0][1].contains("CREATE TABLE `sales`(\n
`id` int,\n `amount` double,\n `k1` string,\n `k2` string,\n `k3`
string)\nPARTITIONED BY (\n `year` int)"));
+ sql """drop table test_hive_meta_cache_db.sales"""
}
}
}
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
