This is an automated email from the ASF dual-hosted git repository.
dkuzmenko pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/hive.git
The following commit(s) were added to refs/heads/master by this push:
new 6b361ac3582 HIVE-28292: Optimize SHOW TABLES/VIEWS statements (Wechar
Yu, reviewed by Denys Kuzmenko)
6b361ac3582 is described below
commit 6b361ac3582a18852a6f090a9a20f522b84c8135
Author: Wechar Yu <[email protected]>
AuthorDate: Tue Sep 17 02:49:30 2024 +0800
HIVE-28292: Optimize SHOW TABLES/VIEWS statements (Wechar Yu, reviewed by
Denys Kuzmenko)
Closes #5273
---
.../info/show/tables/ShowTablesFormatter.java | 23 ++++-----
.../info/show/tables/ShowTablesOperation.java | 33 ++++++-------
.../hive/ql/ddl/view/show/ShowViewsOperation.java | 11 +----
.../ql/metadata/SessionHiveMetaStoreClient.java | 4 +-
.../org/apache/hadoop/hive/ql/udf/UDFLike.java | 11 ++++-
.../apache/hadoop/hive/ql/metadata/TestHive.java | 2 +
.../llap/authorization_privilege_objects.q.out | 4 +-
.../hadoop/hive/metastore/HiveMetaStoreClient.java | 54 ++++++++++++++--------
.../hive/metastore/utils/MetaStoreUtils.java | 5 ++
.../apache/hadoop/hive/metastore/HMSHandler.java | 35 ++++++--------
.../apache/hadoop/hive/metastore/ObjectStore.java | 27 ++++++-----
.../src/main/sql/derby/hive-schema-4.1.0.derby.sql | 2 +-
.../sql/derby/upgrade-4.0.0-to-4.1.0.derby.sql | 4 ++
.../src/main/sql/mssql/hive-schema-4.1.0.mssql.sql | 4 +-
.../sql/mssql/upgrade-4.0.0-to-4.1.0.mssql.sql | 5 ++
.../src/main/sql/mysql/hive-schema-4.1.0.mysql.sql | 3 +-
.../sql/mysql/upgrade-4.0.0-to-4.1.0.mysql.sql | 5 ++
.../main/sql/oracle/hive-schema-4.1.0.oracle.sql | 4 +-
.../sql/oracle/upgrade-4.0.0-to-4.1.0.oracle.sql | 5 ++
.../sql/postgres/hive-schema-4.1.0.postgres.sql | 9 +---
.../postgres/upgrade-4.0.0-to-4.1.0.postgres.sql | 5 ++
.../hive/metastore/client/TestTablesGetExists.java | 7 ---
.../hive/metastore/client/TestTablesList.java | 5 +-
23 files changed, 142 insertions(+), 125 deletions(-)
diff --git
a/ql/src/java/org/apache/hadoop/hive/ql/ddl/table/info/show/tables/ShowTablesFormatter.java
b/ql/src/java/org/apache/hadoop/hive/ql/ddl/table/info/show/tables/ShowTablesFormatter.java
index df5420cd6ba..72a19d00dcf 100644
---
a/ql/src/java/org/apache/hadoop/hive/ql/ddl/table/info/show/tables/ShowTablesFormatter.java
+++
b/ql/src/java/org/apache/hadoop/hive/ql/ddl/table/info/show/tables/ShowTablesFormatter.java
@@ -19,12 +19,12 @@
package org.apache.hadoop.hive.ql.ddl.table.info.show.tables;
import com.google.common.collect.ImmutableMap;
+
import org.apache.hadoop.hive.conf.HiveConf;
import org.apache.hadoop.hive.ql.ddl.ShowUtils;
import org.apache.hadoop.hive.ql.ddl.ShowUtils.TextMetaDataTable;
import org.apache.hadoop.hive.ql.exec.Utilities;
import org.apache.hadoop.hive.ql.metadata.HiveException;
-import org.apache.hadoop.hive.ql.metadata.Table;
import org.apache.hadoop.hive.ql.metadata.formatting.MapBuilder;
import org.apache.hadoop.hive.ql.metadata.formatting.MetaDataFormatUtils;
import org.apache.hadoop.hive.ql.session.SessionState;
@@ -36,6 +36,7 @@ import java.util.ArrayList;
import java.util.Iterator;
import java.util.List;
import java.util.Map;
+import java.util.TreeMap;
/**
* Formats SHOW TABLES results.
@@ -51,7 +52,7 @@ public abstract class ShowTablesFormatter {
public abstract void showTables(DataOutputStream out, List<String> tables)
throws HiveException;
- abstract void showTablesExtended(DataOutputStream out, List<Table> tables)
throws HiveException;
+ abstract void showTablesExtended(DataOutputStream out, Map<String, String>
tableNameToType) throws HiveException;
// ------ Implementations ------
@@ -62,16 +63,16 @@ public abstract class ShowTablesFormatter {
}
@Override
- void showTablesExtended(DataOutputStream out, List<Table> tables) throws
HiveException {
- if (tables.isEmpty()) {
+ void showTablesExtended(DataOutputStream out, Map<String, String>
tableNameToType) throws HiveException {
+ if (tableNameToType.isEmpty()) {
return;
}
List<Map<String, Object>> tableDataList = new ArrayList<>();
- for (Table table : tables) {
+ for (Map.Entry<String, String> table : tableNameToType.entrySet()) {
Map<String, Object> tableData = ImmutableMap.of(
- "Table Name", table.getTableName(),
- "Table Type", table.getTableType().toString());
+ "Table Name", table.getKey(),
+ "Table Type", table.getValue());
tableDataList.add(tableData);
}
@@ -96,8 +97,8 @@ public abstract class ShowTablesFormatter {
}
@Override
- void showTablesExtended(DataOutputStream out, List<Table> tables) throws
HiveException {
- if (tables.isEmpty()) {
+ void showTablesExtended(DataOutputStream out, Map<String, String>
tableNameToType) throws HiveException {
+ if (tableNameToType.isEmpty()) {
return;
}
@@ -106,8 +107,8 @@ public abstract class ShowTablesFormatter {
if (!SessionState.get().isHiveServerQuery()) {
mdt.addRow("# Table Name", "Table Type");
}
- for (Table table : tables) {
- mdt.addRow(table.getTableName(), table.getTableType().toString());
+ for (Map.Entry<String, String> table : tableNameToType.entrySet()) {
+ mdt.addRow(table.getKey(), table.getValue());
}
// In case the query is served by HiveServer2, don't pad it with
spaces,
// as HiveServer2 output is consumed by JDBC/ODBC clients.
diff --git
a/ql/src/java/org/apache/hadoop/hive/ql/ddl/table/info/show/tables/ShowTablesOperation.java
b/ql/src/java/org/apache/hadoop/hive/ql/ddl/table/info/show/tables/ShowTablesOperation.java
index 6dce083fad2..40f7b767e88 100644
---
a/ql/src/java/org/apache/hadoop/hive/ql/ddl/table/info/show/tables/ShowTablesOperation.java
+++
b/ql/src/java/org/apache/hadoop/hive/ql/ddl/table/info/show/tables/ShowTablesOperation.java
@@ -24,16 +24,15 @@ import org.apache.hadoop.hive.ql.ddl.ShowUtils;
import java.io.DataOutputStream;
import java.util.ArrayList;
import java.util.Collections;
-import java.util.Comparator;
import java.util.List;
-import java.util.regex.Pattern;
-import java.util.stream.Collectors;
+import java.util.Map;
+import java.util.TreeMap;
import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.hive.metastore.TableType;
import org.apache.hadoop.hive.ql.ErrorMsg;
import org.apache.hadoop.hive.ql.ddl.DDLOperation;
import org.apache.hadoop.hive.ql.metadata.HiveException;
-import org.apache.hadoop.hive.ql.metadata.Table;
import org.apache.hadoop.hive.ql.udf.UDFLike;
/**
@@ -60,12 +59,9 @@ public class ShowTablesOperation extends
DDLOperation<ShowTablesDesc> {
}
private void showTables() throws HiveException {
+ String pattern = UDFLike.likePatternToRegExp(desc.getPattern(), false,
true);
List<String> tableNames = new ArrayList<>(
- context.getDb().getTablesByType(desc.getDbName(), null,
desc.getTypeFilter()));
- if (desc.getPattern() != null) {
- Pattern pattern =
Pattern.compile(UDFLike.likePatternToRegExp(desc.getPattern()),
Pattern.CASE_INSENSITIVE);
- tableNames = tableNames.stream().filter(name ->
pattern.matcher(name).matches()).collect(Collectors.toList());
- }
+ context.getDb().getTablesByType(desc.getDbName(), pattern,
desc.getTypeFilter()));
Collections.sort(tableNames);
LOG.debug("Found {} table(s) matching the SHOW TABLES statement.",
tableNames.size());
@@ -78,20 +74,19 @@ public class ShowTablesOperation extends
DDLOperation<ShowTablesDesc> {
}
private void showTablesExtended() throws HiveException {
- List<Table> tableObjects = new ArrayList<>();
- tableObjects.addAll(context.getDb().getTableObjects(desc.getDbName(),
null, desc.getTypeFilter()));
- if (desc.getPattern() != null) {
- Pattern pattern =
Pattern.compile(UDFLike.likePatternToRegExp(desc.getPattern()),
Pattern.CASE_INSENSITIVE);
- tableObjects = tableObjects.stream()
- .filter(object -> pattern.matcher(object.getTableName()).matches())
- .collect(Collectors.toList());
+ Map<String, String> tableNameToType = new TreeMap<>();
+ String pattern = UDFLike.likePatternToRegExp(desc.getPattern(), false,
true);
+ TableType typeFilter = desc.getTypeFilter();
+ TableType[] tableTypes = typeFilter == null ? TableType.values() : new
TableType[]{typeFilter};
+ for (TableType tableType : tableTypes) {
+ List<String> tables = context.getDb().getTablesByType(desc.getDbName(),
pattern, tableType);
+ tables.forEach(name -> tableNameToType.put(name, tableType.toString()));
}
- Collections.sort(tableObjects, Comparator.comparing(Table::getTableName));
- LOG.debug("Found {} table(s) matching the SHOW EXTENDED TABLES
statement.", tableObjects.size());
+ LOG.debug("Found {} table(s) matching the SHOW EXTENDED TABLES
statement.", tableNameToType.size());
try (DataOutputStream os = ShowUtils.getOutputStream(new
Path(desc.getResFile()), context)) {
ShowTablesFormatter formatter =
ShowTablesFormatter.getFormatter(context.getConf());
- formatter.showTablesExtended(os, tableObjects);
+ formatter.showTablesExtended(os, tableNameToType);
} catch (Exception e) {
throw new HiveException(e, ErrorMsg.GENERIC_ERROR, "in database " +
desc.getDbName());
}
diff --git
a/ql/src/java/org/apache/hadoop/hive/ql/ddl/view/show/ShowViewsOperation.java
b/ql/src/java/org/apache/hadoop/hive/ql/ddl/view/show/ShowViewsOperation.java
index 0cab568cf8a..6c6e5f8d247 100644
---
a/ql/src/java/org/apache/hadoop/hive/ql/ddl/view/show/ShowViewsOperation.java
+++
b/ql/src/java/org/apache/hadoop/hive/ql/ddl/view/show/ShowViewsOperation.java
@@ -21,8 +21,6 @@ package org.apache.hadoop.hive.ql.ddl.view.show;
import java.io.DataOutputStream;
import java.util.Collections;
import java.util.List;
-import java.util.regex.Pattern;
-import java.util.stream.Collectors;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hive.metastore.TableType;
@@ -48,13 +46,8 @@ public class ShowViewsOperation extends
DDLOperation<ShowViewsDesc> {
throw new HiveException(ErrorMsg.DATABASE_NOT_EXISTS, desc.getDbName());
}
- List<String> viewNames = context.getDb().getTablesByType(desc.getDbName(),
null, TableType.VIRTUAL_VIEW);
- if (desc.getPattern() != null) {
- Pattern pattern =
Pattern.compile(UDFLike.likePatternToRegExp(desc.getPattern()),
Pattern.CASE_INSENSITIVE);
- viewNames = viewNames.stream()
- .filter(name -> pattern.matcher(name).matches())
- .collect(Collectors.toList());
- }
+ String pattern = UDFLike.likePatternToRegExp(desc.getPattern(), false,
true);
+ List<String> viewNames = context.getDb().getTablesByType(desc.getDbName(),
pattern, TableType.VIRTUAL_VIEW);
Collections.sort(viewNames);
LOG.debug("Found {} view(s) matching the SHOW VIEWS statement.",
viewNames.size());
diff --git
a/ql/src/java/org/apache/hadoop/hive/ql/metadata/SessionHiveMetaStoreClient.java
b/ql/src/java/org/apache/hadoop/hive/ql/metadata/SessionHiveMetaStoreClient.java
index 4341fd91422..af7aca6a373 100644
---
a/ql/src/java/org/apache/hadoop/hive/ql/metadata/SessionHiveMetaStoreClient.java
+++
b/ql/src/java/org/apache/hadoop/hive/ql/metadata/SessionHiveMetaStoreClient.java
@@ -321,7 +321,7 @@ public class SessionHiveMetaStoreClient extends
HiveMetaStoreClientWithLocalCach
if (tables == null || tables.size() == 0) {
return tableNames;
}
- tablePattern = tablePattern.replaceAll("\\*", ".*");
+ tablePattern = tablePattern.replaceAll("(?<!\\.)\\*", ".*");
Pattern pattern = Pattern.compile(tablePattern);
Matcher matcher = pattern.matcher("");
Set<String> combinedTableNames = new HashSet<String>();
@@ -351,7 +351,7 @@ public class SessionHiveMetaStoreClient extends
HiveMetaStoreClientWithLocalCach
if (tables == null || tables.size() == 0) {
return tableNames;
}
- tablePattern = tablePattern.replaceAll("\\*", ".*");
+ tablePattern = tablePattern.replaceAll("(?<!\\.)\\*", ".*");
Pattern pattern = Pattern.compile(tablePattern);
Matcher matcher = pattern.matcher("");
Set<String> combinedTableNames = new HashSet<String>();
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/udf/UDFLike.java
b/ql/src/java/org/apache/hadoop/hive/ql/udf/UDFLike.java
index 4889b2e164f..b455e91b452 100755
--- a/ql/src/java/org/apache/hadoop/hive/ql/udf/UDFLike.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/udf/UDFLike.java
@@ -62,6 +62,13 @@ public class UDFLike extends UDF {
}
public static String likePatternToRegExp(String likePattern) {
+ return likePatternToRegExp(likePattern, true, false);
+ }
+
+ public static String likePatternToRegExp(String likePattern, boolean
literalize, boolean greedyMatch) {
+ if (likePattern == null) {
+ return null;
+ }
StringBuilder sb = new StringBuilder();
for (int i = 0; i < likePattern.length(); i++) {
// Make a special case for "\\_" and "\\%"
@@ -77,9 +84,9 @@ public class UDFLike extends UDF {
if (n == '_') {
sb.append(".");
} else if (n == '%') {
- sb.append(".*?");
+ sb.append(greedyMatch ? ".*" : ".*?");
} else {
- sb.append(Pattern.quote(Character.toString(n)));
+ sb.append(literalize ? Pattern.quote(Character.toString(n)) : n);
}
}
return sb.toString();
diff --git a/ql/src/test/org/apache/hadoop/hive/ql/metadata/TestHive.java
b/ql/src/test/org/apache/hadoop/hive/ql/metadata/TestHive.java
index 8f03685cf86..2d42ea880ba 100755
--- a/ql/src/test/org/apache/hadoop/hive/ql/metadata/TestHive.java
+++ b/ql/src/test/org/apache/hadoop/hive/ql/metadata/TestHive.java
@@ -23,6 +23,7 @@ import static
org.apache.hadoop.hive.metastore.Warehouse.DEFAULT_DATABASE_NAME;
import java.io.OutputStream;
import java.util.ArrayList;
import java.util.Arrays;
+import java.util.Collections;
import java.util.HashMap;
import java.util.HashSet;
import java.util.LinkedList;
@@ -442,6 +443,7 @@ public class TestHive {
hm.createTable(tbl2);
List<String> fts = hm.getTablesForDb(dbName, ".*");
+ Collections.sort(fts);
assertEquals(ts, fts);
assertEquals(2, fts.size());
diff --git
a/ql/src/test/results/clientpositive/llap/authorization_privilege_objects.q.out
b/ql/src/test/results/clientpositive/llap/authorization_privilege_objects.q.out
index e5e86fadd61..148d92970c2 100644
---
a/ql/src/test/results/clientpositive/llap/authorization_privilege_objects.q.out
+++
b/ql/src/test/results/clientpositive/llap/authorization_privilege_objects.q.out
@@ -42,8 +42,8 @@ PREHOOK: query: SHOW TABLES IN test_auth_obj_db
PREHOOK: type: SHOWTABLES
PREHOOK: Input: database:test_auth_obj_db
filterListCmdObjects
-HIVE PRIVILEGE OBJECT { objectName: test_privs type: TABLE_OR_VIEW actionType:
OTHER dbName: test_auth_obj_db OWNER: testuser OWNERTYPE: USER}
-HIVE PRIVILEGE OBJECT { objectName: test_privs2 type: TABLE_OR_VIEW
actionType: OTHER dbName: test_auth_obj_db OWNER: testuser2 OWNERTYPE: USER}
+HIVE PRIVILEGE OBJECT { objectName: test_privs type: TABLE_OR_VIEW actionType:
OTHER dbName: test_auth_obj_db}
+HIVE PRIVILEGE OBJECT { objectName: test_privs2 type: TABLE_OR_VIEW
actionType: OTHER dbName: test_auth_obj_db}
POSTHOOK: query: SHOW TABLES IN test_auth_obj_db
POSTHOOK: type: SHOWTABLES
POSTHOOK: Input: database:test_auth_obj_db
diff --git
a/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/HiveMetaStoreClient.java
b/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/HiveMetaStoreClient.java
index 0125768033e..780b0c025e0 100644
---
a/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/HiveMetaStoreClient.java
+++
b/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/HiveMetaStoreClient.java
@@ -3015,26 +3015,44 @@ public class HiveMetaStoreClient implements
IMetaStoreClient, AutoCloseable {
@Override
public List<String> getTables(String catName, String dbName, String
tablePattern)
throws TException {
- List<String> tables = new ArrayList<>();
- GetProjectionsSpec projectionsSpec = new GetProjectionsSpec();
- projectionsSpec.setFieldList(Arrays.asList("dbName", "tableName", "owner",
"ownerType"));
- GetTablesRequest req = new GetTablesRequest(dbName);
- req.setCatName(catName);
- req.setCapabilities(version);
- req.setTblNames(null);
- if(tablePattern == null){
+ if (tablePattern == null) {
tablePattern = ".*";
}
- req.setTablesPattern(tablePattern);
- if (processorCapabilities != null)
- req.setProcessorCapabilities(new
ArrayList<String>(Arrays.asList(processorCapabilities)));
- if (processorIdentifier != null)
- req.setProcessorIdentifier(processorIdentifier);
- req.setProjectionSpec(projectionsSpec);
- List<Table> tableObjects =
client.get_table_objects_by_name_req(req).getTables();
- tableObjects =
deepCopyTables(FilterUtils.filterTablesIfEnabled(isClientFilterEnabled,
filterHook, tableObjects));
- for (Table tbl : tableObjects) {
- tables.add(tbl.getTableName());
+ List<String> tables = new ArrayList<>();
+ Database db = null;
+ try {
+ db = getDatabase(catName, dbName);
+ } catch (NoSuchObjectException e) { /* appears exception is not thrown
currently if db doesnt exist */ }
+
+ if (MetaStoreUtils.isDatabaseRemote(db)) {
+ // TODO: remote database does not support list table names by pattern
yet.
+ // This branch can be removed once it's supported.
+ GetProjectionsSpec projectionsSpec = new GetProjectionsSpec();
+ projectionsSpec.setFieldList(Arrays.asList("dbName", "tableName",
"owner", "ownerType"));
+ GetTablesRequest req = new GetTablesRequest(dbName);
+ req.setCatName(catName);
+ req.setCapabilities(version);
+ req.setTblNames(null);
+ req.setTablesPattern(tablePattern);
+ if (processorCapabilities != null)
+ req.setProcessorCapabilities(Arrays.asList(processorCapabilities));
+ if (processorIdentifier != null)
+ req.setProcessorIdentifier(processorIdentifier);
+ req.setProjectionSpec(projectionsSpec);
+ List<Table> tableObjects =
client.get_table_objects_by_name_req(req).getTables();
+ tableObjects =
deepCopyTables(FilterUtils.filterTablesIfEnabled(isClientFilterEnabled,
filterHook, tableObjects));
+ for (Table tbl : tableObjects) {
+ tables.add(tbl.getTableName());
+ }
+ } else {
+ // This trick handles pattern for both string regex and wildcards ('*'
and '|').
+ // We need unify the pattern definition, see HIVE-28297 for details.
+ String[] patterns = tablePattern.split("\\|");
+ for (String pattern : patterns) {
+ pattern = "(?i)" + pattern.replaceAll("(?<!\\.)\\*", ".*");
+ String filter = String.format("%s like \"%s\"",
hive_metastoreConstants.HIVE_FILTER_FIELD_TABLE_NAME, pattern);
+ tables.addAll(listTableNamesByFilter(catName, dbName, filter, (short)
-1));
+ }
}
return tables;
}
diff --git
a/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/utils/MetaStoreUtils.java
b/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/utils/MetaStoreUtils.java
index bbd36befcd7..5f3c78dfd9c 100644
---
a/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/utils/MetaStoreUtils.java
+++
b/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/utils/MetaStoreUtils.java
@@ -67,6 +67,7 @@ import
org.apache.hadoop.hive.metastore.api.PartitionsSpecByExprResult;
import org.apache.hadoop.hive.metastore.api.StorageDescriptor;
import org.apache.hadoop.hive.metastore.api.Table;
import org.apache.hadoop.hive.metastore.api.Database;
+import org.apache.hadoop.hive.metastore.api.DatabaseType;
import org.apache.hadoop.hive.metastore.api.WMPoolSchedulingPolicy;
import org.apache.hadoop.hive.metastore.api.hive_metastoreConstants;
import org.apache.hadoop.hive.metastore.conf.MetastoreConf;
@@ -1335,4 +1336,8 @@ public class MetaStoreUtils {
}
return httpPath;
}
+
+ public static boolean isDatabaseRemote(Database db) {
+ return db != null && db.getType() == DatabaseType.REMOTE;
+ }
}
diff --git
a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/HMSHandler.java
b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/HMSHandler.java
index 4ca45d56f07..37450f68c1a 100644
---
a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/HMSHandler.java
+++
b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/HMSHandler.java
@@ -1595,7 +1595,7 @@ public class HMSHandler extends FacebookBase implements
IHMSHandler {
try {
ms.openTransaction();
db = ms.getDatabase(req.getCatalogName(), req.getName());
- if (db.getType() == DatabaseType.REMOTE) {
+ if (MetaStoreUtils.isDatabaseRemote(db)) {
success = drop_remote_database_core(ms, db);
return;
}
@@ -2334,7 +2334,7 @@ public class HMSHandler extends FacebookBase implements
IHMSHandler {
}
Database db = get_database_core(tbl.getCatName(), tbl.getDbName());
- if (db != null && db.getType().equals(DatabaseType.REMOTE)) {
+ if (MetaStoreUtils.isDatabaseRemote(db)) {
// HIVE-24425: Create table in REMOTE db should fail
throw new MetaException("Create table in REMOTE database " +
db.getName() + " is not allowed");
}
@@ -2985,7 +2985,7 @@ public class HMSHandler extends FacebookBase implements
IHMSHandler {
// HIVE-25282: Drop/Alter table in REMOTE db should fail
db = ms.getDatabase(catName, dbname);
- if (db.getType() == DatabaseType.REMOTE) {
+ if (MetaStoreUtils.isDatabaseRemote(db)) {
throw new MetaException("Drop table in REMOTE database " +
db.getName() + " is not allowed");
}
isReplicated = isDbReplicationTarget(db);
@@ -3758,15 +3758,13 @@ public class HMSHandler extends FacebookBase implements
IHMSHandler {
db = get_database_core(catName, dbName);
} catch (Exception e) { /* appears exception is not thrown currently if db
doesnt exist */ }
- if (db != null) {
- if (db.getType().equals(DatabaseType.REMOTE)) {
- t =
DataConnectorProviderFactory.getDataConnectorProvider(db).getTable(tblName);
- if (t == null) {
- throw new NoSuchObjectException(TableName.getQualified(catName,
dbName, tblName) + " table not found");
- }
- t.setDbName(dbName);
- return t;
+ if (MetaStoreUtils.isDatabaseRemote(db)) {
+ t =
DataConnectorProviderFactory.getDataConnectorProvider(db).getTable(tblName);
+ if (t == null) {
+ throw new NoSuchObjectException(TableName.getQualified(catName,
dbName, tblName) + " table not found");
}
+ t.setDbName(dbName);
+ return t;
}
try {
@@ -5210,13 +5208,10 @@ public class HMSHandler extends FacebookBase implements
IHMSHandler {
try {
String[] parsedDbName = parseDbName(name, conf);
Database db = get_database_core(parsedDbName[CAT_NAME],
parsedDbName[DB_NAME]);
- if (db != null && db.getType() != null && db.getType() ==
DatabaseType.REMOTE) {
- return true;
- }
+ return MetaStoreUtils.isDatabaseRemote(db);
} catch (Exception e) {
return false;
}
- return false;
}
private void deleteParentRecursive(Path parent, int depth, boolean
mustPurge, boolean needRecycle)
@@ -6261,7 +6256,7 @@ public class HMSHandler extends FacebookBase implements
IHMSHandler {
// HIVE-25282: Drop/Alter table in REMOTE db should fail
try {
Database db = get_database_core(catName, dbname);
- if (db != null && db.getType().equals(DatabaseType.REMOTE)) {
+ if (MetaStoreUtils.isDatabaseRemote(db)) {
throw new MetaException("Alter table in REMOTE database " +
db.getName() + " is not allowed");
}
} catch (NoSuchObjectException e) {
@@ -6379,10 +6374,8 @@ public class HMSHandler extends FacebookBase implements
IHMSHandler {
Database db = null;
try {
db = get_database_core(catName, dbname);
- if (db != null) {
- if (db.getType().equals(DatabaseType.REMOTE)) {
- return
DataConnectorProviderFactory.getDataConnectorProvider(db).getTableNames();
- }
+ if (MetaStoreUtils.isDatabaseRemote(db)) {
+ return
DataConnectorProviderFactory.getDataConnectorProvider(db).getTableNames();
}
} catch (Exception e) { /* ignore */ }
@@ -8732,7 +8725,7 @@ public class HMSHandler extends FacebookBase implements
IHMSHandler {
throw new NoSuchObjectException("The database " + func.getDbName() + "
does not exist");
}
- if (db.getType() == DatabaseType.REMOTE) {
+ if (MetaStoreUtils.isDatabaseRemote(db)) {
throw new MetaException("Operation create_function not support for
REMOTE database");
}
diff --git
a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/ObjectStore.java
b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/ObjectStore.java
index 1e651161d83..c41d4d0087e 100644
---
a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/ObjectStore.java
+++
b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/ObjectStore.java
@@ -2247,19 +2247,7 @@ public class ObjectStore implements RawStore,
Configurable {
}
if (mtables == null || mtables.isEmpty()) {
- Database tempDB = null;
- NoSuchObjectException ex = null;
- try {
- tempDB = getDatabase(catName, db);
- } catch(NoSuchObjectException nsoe) {
- ex = nsoe;
- }
-
- if (tempDB == null) {
- final String errorMessage = (ex == null ? "" : (": " +
ex.getMessage()));
- throw new UnknownDBException("Could not find database " +
DatabaseName.getQualified(catName, db) +
- errorMessage);
- }
+ ensureGetDatabase(catName, db);
} else {
for (Iterator iter = mtables.iterator(); iter.hasNext(); ) {
Table tbl = convertToTable((MTable) iter.next());
@@ -4944,6 +4932,14 @@ public class ObjectStore implements RawStore,
Configurable {
return convertToTable(ensureGetMTable(catName, dbName, tblName));
}
+ private Database ensureGetDatabase(String catName, String dbName) throws
UnknownDBException {
+ try {
+ return getDatabase(catName, dbName);
+ } catch (NoSuchObjectException nsoe) {
+ throw new UnknownDBException("Could not find database " +
DatabaseName.getQualified(catName, dbName));
+ }
+ }
+
/**
* Makes a JDO query filter string.
* Makes a JDO query filter string for tables or partitions.
@@ -5044,7 +5040,7 @@ public class ObjectStore implements RawStore,
Configurable {
@Override
public List<String> listTableNamesByFilter(String catName, String dbName,
String filter,
- short maxTables) throws
MetaException {
+ short maxTables) throws
MetaException, UnknownDBException {
boolean success = false;
Query query = null;
List<String> tableNames = new ArrayList<>();
@@ -5053,6 +5049,9 @@ public class ObjectStore implements RawStore,
Configurable {
LOG.debug("Executing listTableNamesByFilter");
catName = normalizeIdentifier(catName);
dbName = normalizeIdentifier(dbName);
+
+ ensureGetDatabase(catName, dbName);
+
Map<String, Object> params = new HashMap<>();
String queryFilterString = makeQueryFilterString(catName, dbName, null,
filter, params);
query = pm.newQuery(MTable.class);
diff --git
a/standalone-metastore/metastore-server/src/main/sql/derby/hive-schema-4.1.0.derby.sql
b/standalone-metastore/metastore-server/src/main/sql/derby/hive-schema-4.1.0.derby.sql
index ea1f90aff89..8fd6c26f1f3 100644
---
a/standalone-metastore/metastore-server/src/main/sql/derby/hive-schema-4.1.0.derby.sql
+++
b/standalone-metastore/metastore-server/src/main/sql/derby/hive-schema-4.1.0.derby.sql
@@ -269,7 +269,7 @@ CREATE UNIQUE INDEX "APP"."ROLEENTITYINDEX" ON
"APP"."ROLES" ("ROLE_NAME");
CREATE INDEX "APP"."TABLEPRIVILEGEINDEX" ON "APP"."TBL_PRIVS" ("AUTHORIZER",
"TBL_ID", "PRINCIPAL_NAME", "PRINCIPAL_TYPE", "TBL_PRIV", "GRANTOR",
"GRANTOR_TYPE");
-CREATE UNIQUE INDEX "APP"."UNIQUETABLE" ON "APP"."TBLS" ("TBL_NAME", "DB_ID");
+CREATE UNIQUE INDEX "APP"."UNIQUETABLE" ON "APP"."TBLS" ("DB_ID", "TBL_NAME");
CREATE UNIQUE INDEX "APP"."UNIQUE_DATABASE" ON "APP"."DBS" ("NAME",
"CTLG_NAME");
diff --git
a/standalone-metastore/metastore-server/src/main/sql/derby/upgrade-4.0.0-to-4.1.0.derby.sql
b/standalone-metastore/metastore-server/src/main/sql/derby/upgrade-4.0.0-to-4.1.0.derby.sql
index fd56b427ae7..4dddb514fdf 100644
---
a/standalone-metastore/metastore-server/src/main/sql/derby/upgrade-4.0.0-to-4.1.0.derby.sql
+++
b/standalone-metastore/metastore-server/src/main/sql/derby/upgrade-4.0.0-to-4.1.0.derby.sql
@@ -12,5 +12,9 @@ ALTER TABLE "APP"."PART_COL_STATS" DROP COLUMN "DB_NAME";
ALTER TABLE "APP"."PART_COL_STATS" DROP COLUMN "TABLE_NAME";
ALTER TABLE "APP"."PART_COL_STATS" DROP COLUMN "PARTITION_NAME";
+-- HIVE-28292
+DROP INDEX "APP"."UNIQUETABLE";
+CREATE UNIQUE INDEX "APP"."UNIQUETABLE" ON "APP"."TBLS" ("DB_ID", "TBL_NAME");
+
-- This needs to be the last thing done. Insert any changes above this line.
UPDATE "APP".VERSION SET SCHEMA_VERSION='4.1.0', VERSION_COMMENT='Hive release
version 4.1.0' where VER_ID=1;
\ No newline at end of file
diff --git
a/standalone-metastore/metastore-server/src/main/sql/mssql/hive-schema-4.1.0.mssql.sql
b/standalone-metastore/metastore-server/src/main/sql/mssql/hive-schema-4.1.0.mssql.sql
index c0cfeafc370..72430f4a2b7 100644
---
a/standalone-metastore/metastore-server/src/main/sql/mssql/hive-schema-4.1.0.mssql.sql
+++
b/standalone-metastore/metastore-server/src/main/sql/mssql/hive-schema-4.1.0.mssql.sql
@@ -818,9 +818,7 @@ ALTER TABLE TBLS ADD CONSTRAINT TBLS_FK1 FOREIGN KEY
(DB_ID) REFERENCES DBS (DB_
CREATE INDEX TBLS_N50 ON TBLS (SD_ID);
-CREATE UNIQUE INDEX UNIQUETABLE ON TBLS (TBL_NAME,DB_ID);
-
-CREATE INDEX TBLS_N49 ON TBLS (DB_ID);
+CREATE UNIQUE INDEX UNIQUETABLE ON TBLS (DB_ID,TBL_NAME);
-- Constraints for table SDS for class(es)
[org.apache.hadoop.hive.metastore.model.MStorageDescriptor]
diff --git
a/standalone-metastore/metastore-server/src/main/sql/mssql/upgrade-4.0.0-to-4.1.0.mssql.sql
b/standalone-metastore/metastore-server/src/main/sql/mssql/upgrade-4.0.0-to-4.1.0.mssql.sql
index 9fbf56fa5d7..02cdd43829e 100644
---
a/standalone-metastore/metastore-server/src/main/sql/mssql/upgrade-4.0.0-to-4.1.0.mssql.sql
+++
b/standalone-metastore/metastore-server/src/main/sql/mssql/upgrade-4.0.0-to-4.1.0.mssql.sql
@@ -11,6 +11,11 @@ CREATE INDEX PCS_STATS_IDX ON PART_COL_STATS
(PART_ID,COLUMN_NAME);
DROP INDEX PART_COL_STATS_N49 on PART_COL_STATS;
ALTER TABLE PART_COL_STATS DROP COLUMN CAT_NAME, DB_NAME, TABLE_NAME,
PARTITION_NAME;
+-- HIVE-28292
+DROP INDEX UNIQUETABLE ON TBLS;
+CREATE UNIQUE INDEX UNIQUETABLE ON TBLS (DB_ID,TBL_NAME);
+DROP INDEX TBLS_N49 ON TBLS;
+
-- These lines need to be last. Insert any changes above.
UPDATE VERSION SET SCHEMA_VERSION='4.1.0', VERSION_COMMENT='Hive release
version 4.1.0' where VER_ID=1;
SELECT 'Finished upgrading MetaStore schema from 4.0.0 to 4.1.0' AS MESSAGE;
diff --git
a/standalone-metastore/metastore-server/src/main/sql/mysql/hive-schema-4.1.0.mysql.sql
b/standalone-metastore/metastore-server/src/main/sql/mysql/hive-schema-4.1.0.mysql.sql
index 078f81a097a..ab1468aedcc 100644
---
a/standalone-metastore/metastore-server/src/main/sql/mysql/hive-schema-4.1.0.mysql.sql
+++
b/standalone-metastore/metastore-server/src/main/sql/mysql/hive-schema-4.1.0.mysql.sql
@@ -597,9 +597,8 @@ CREATE TABLE IF NOT EXISTS `TBLS` (
`IS_REWRITE_ENABLED` bit(1) NOT NULL DEFAULT 0,
`WRITE_ID` bigint(20) DEFAULT 0,
PRIMARY KEY (`TBL_ID`),
- UNIQUE KEY `UNIQUETABLE` (`TBL_NAME`,`DB_ID`),
+ UNIQUE KEY `UNIQUETABLE` (`DB_ID`,`TBL_NAME`),
KEY `TBLS_N50` (`SD_ID`),
- KEY `TBLS_N49` (`DB_ID`),
CONSTRAINT `TBLS_FK1` FOREIGN KEY (`SD_ID`) REFERENCES `SDS` (`SD_ID`),
CONSTRAINT `TBLS_FK2` FOREIGN KEY (`DB_ID`) REFERENCES `DBS` (`DB_ID`)
) ENGINE=InnoDB DEFAULT CHARSET=latin1;
diff --git
a/standalone-metastore/metastore-server/src/main/sql/mysql/upgrade-4.0.0-to-4.1.0.mysql.sql
b/standalone-metastore/metastore-server/src/main/sql/mysql/upgrade-4.0.0-to-4.1.0.mysql.sql
index 5d535289dff..03ffcac15fb 100644
---
a/standalone-metastore/metastore-server/src/main/sql/mysql/upgrade-4.0.0-to-4.1.0.mysql.sql
+++
b/standalone-metastore/metastore-server/src/main/sql/mysql/upgrade-4.0.0-to-4.1.0.mysql.sql
@@ -9,6 +9,11 @@ DROP INDEX PCS_STATS_IDX ON PART_COL_STATS;
CREATE INDEX PCS_STATS_IDX ON PART_COL_STATS (PART_ID, COLUMN_NAME) USING
BTREE;
ALTER TABLE PART_COL_STATS DROP COLUMN CAT_NAME, DROP COLUMN DB_NAME, DROP
COLUMN TABLE_NAME, DROP COLUMN PARTITION_NAME;
+-- HIVE-28292
+DROP INDEX UNIQUETABLE ON TBLS;
+CREATE UNIQUE INDEX UNIQUETABLE ON TBLS (DB_ID, TBL_NAME);
+DROP INDEX TBLS_N49 ON TBLS;
+
-- These lines need to be last. Insert any changes above.
UPDATE VERSION SET SCHEMA_VERSION='4.1.0', VERSION_COMMENT='Hive release
version 4.1.0' where VER_ID=1;
SELECT 'Finished upgrading MetaStore schema from 4.0.0 to 4.1.0' AS MESSAGE;
diff --git
a/standalone-metastore/metastore-server/src/main/sql/oracle/hive-schema-4.1.0.oracle.sql
b/standalone-metastore/metastore-server/src/main/sql/oracle/hive-schema-4.1.0.oracle.sql
index b878606fe6e..a24a154973f 100644
---
a/standalone-metastore/metastore-server/src/main/sql/oracle/hive-schema-4.1.0.oracle.sql
+++
b/standalone-metastore/metastore-server/src/main/sql/oracle/hive-schema-4.1.0.oracle.sql
@@ -852,9 +852,7 @@ ALTER TABLE TBLS ADD CONSTRAINT TBLS_FK2 FOREIGN KEY
(DB_ID) REFERENCES DBS (DB_
ALTER TABLE TBLS ADD CONSTRAINT TBLS_FK1 FOREIGN KEY (SD_ID) REFERENCES SDS
(SD_ID) INITIALLY DEFERRED ;
-CREATE INDEX TBLS_N49 ON TBLS (DB_ID);
-
-CREATE UNIQUE INDEX UNIQUETABLE ON TBLS (TBL_NAME,DB_ID);
+CREATE UNIQUE INDEX UNIQUETABLE ON TBLS (DB_ID,TBL_NAME);
CREATE INDEX TBLS_N50 ON TBLS (SD_ID);
diff --git
a/standalone-metastore/metastore-server/src/main/sql/oracle/upgrade-4.0.0-to-4.1.0.oracle.sql
b/standalone-metastore/metastore-server/src/main/sql/oracle/upgrade-4.0.0-to-4.1.0.oracle.sql
index fbc3de39e28..9c75d4b1249 100644
---
a/standalone-metastore/metastore-server/src/main/sql/oracle/upgrade-4.0.0-to-4.1.0.oracle.sql
+++
b/standalone-metastore/metastore-server/src/main/sql/oracle/upgrade-4.0.0-to-4.1.0.oracle.sql
@@ -11,6 +11,11 @@ CREATE INDEX PCS_STATS_IDX ON PART_COL_STATS (PART_ID,
COLUMN_NAME);
DROP INDEX PART_COL_STATS_N49;
ALTER TABLE PART_COL_STATS DROP (CAT_NAME, DB_NAME, TABLE_NAME,
PARTITION_NAME);
+-- HIVE-28292
+DROP INDEX UNIQUETABLE;
+CREATE UNIQUE INDEX UNIQUETABLE ON TBLS (DB_ID, TBL_NAME);
+DROP INDEX TBLS_N49;
+
-- These lines need to be last. Insert any changes above.
UPDATE VERSION SET SCHEMA_VERSION='4.1.0', VERSION_COMMENT='Hive release
version 4.1.0' where VER_ID=1;
SELECT 'Finished upgrading MetaStore schema from 4.0.0 to 4.1.0' AS Status
from dual;
diff --git
a/standalone-metastore/metastore-server/src/main/sql/postgres/hive-schema-4.1.0.postgres.sql
b/standalone-metastore/metastore-server/src/main/sql/postgres/hive-schema-4.1.0.postgres.sql
index 2b3a4cd17db..970ca993a93 100644
---
a/standalone-metastore/metastore-server/src/main/sql/postgres/hive-schema-4.1.0.postgres.sql
+++
b/standalone-metastore/metastore-server/src/main/sql/postgres/hive-schema-4.1.0.postgres.sql
@@ -1028,7 +1028,7 @@ ALTER TABLE ONLY "PARTITIONS"
--
ALTER TABLE ONLY "TBLS"
- ADD CONSTRAINT "UNIQUETABLE" UNIQUE ("TBL_NAME", "DB_ID");
+ ADD CONSTRAINT "UNIQUETABLE" UNIQUE ("DB_ID", "TBL_NAME");
--
@@ -1245,13 +1245,6 @@ CREATE INDEX "TABLEPRIVILEGEINDEX" ON "TBL_PRIVS" USING
btree ("AUTHORIZER", "TB
CREATE INDEX "TABLE_PARAMS_N49" ON "TABLE_PARAMS" USING btree ("TBL_ID");
---
--- Name: TBLS_N49; Type: INDEX; Schema: public; Owner: hiveuser; Tablespace:
---
-
-CREATE INDEX "TBLS_N49" ON "TBLS" USING btree ("DB_ID");
-
-
--
-- Name: TBLS_N50; Type: INDEX; Schema: public; Owner: hiveuser; Tablespace:
--
diff --git
a/standalone-metastore/metastore-server/src/main/sql/postgres/upgrade-4.0.0-to-4.1.0.postgres.sql
b/standalone-metastore/metastore-server/src/main/sql/postgres/upgrade-4.0.0-to-4.1.0.postgres.sql
index cae50ce4100..51ea1d078f2 100644
---
a/standalone-metastore/metastore-server/src/main/sql/postgres/upgrade-4.0.0-to-4.1.0.postgres.sql
+++
b/standalone-metastore/metastore-server/src/main/sql/postgres/upgrade-4.0.0-to-4.1.0.postgres.sql
@@ -11,6 +11,11 @@ CREATE INDEX "PCS_STATS_IDX" ON "PART_COL_STATS" USING btree
("PART_ID","COLUMN_
DROP INDEX "PART_COL_STATS_N49";
ALTER TABLE "PART_COL_STATS" DROP COLUMN "CAT_NAME", DROP COLUMN "DB_NAME",
DROP COLUMN "TABLE_NAME", DROP COLUMN "PARTITION_NAME";
+-- HIVE-28292
+ALTER TABLE ONLY "TBLS" DROP CONSTRAINT "UNIQUETABLE";
+ALTER TABLE ONLY "TBLS" ADD CONSTRAINT "UNIQUETABLE" UNIQUE ("DB_ID",
"TBL_NAME");
+DROP INDEX "TBLS_N49";
+
-- These lines need to be last. Insert any changes above.
UPDATE "VERSION" SET "SCHEMA_VERSION"='4.1.0', "VERSION_COMMENT"='Hive release
version 4.1.0' where "VER_ID"=1;
SELECT 'Finished upgrading MetaStore schema from 4.0.0 to 4.1.0';
diff --git
a/standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/client/TestTablesGetExists.java
b/standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/client/TestTablesGetExists.java
index 84ac0f87210..cd51865da88 100644
---
a/standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/client/TestTablesGetExists.java
+++
b/standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/client/TestTablesGetExists.java
@@ -286,13 +286,6 @@ public class TestTablesGetExists extends
MetaStoreClientTest {
Assert.assertTrue("Comparing tablenames",
tables.contains(testTables[5].getTableName()));
Assert.assertTrue("Comparing tablenames",
tables.contains(testTables[6].getTableName()));
- tables = client.getTables("*", "*");
- Assert.assertEquals("All tables size", 7, tables.size());
- tables = client.getTables("d*", "*");
- Assert.assertEquals("All tables size", 7, tables.size());
- tables = client.getTables("def*", "*");
- Assert.assertEquals("All tables size", 5, tables.size());
-
// Look for tables but do not find any
tables = client.getTables(DEFAULT_DATABASE, "*_not_such_function_*");
Assert.assertEquals("No such table size", 0, tables.size());
diff --git
a/standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/client/TestTablesList.java
b/standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/client/TestTablesList.java
index 20c3af0b3cf..387232cd7b0 100644
---
a/standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/client/TestTablesList.java
+++
b/standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/client/TestTablesList.java
@@ -253,12 +253,11 @@ public class TestTablesList extends MetaStoreClientTest {
||
tableNames.contains(testTables[3].getTableName()));
}
- @Test
+ @Test(expected = UnknownDBException.class)
public void testListTableNamesByFilterCheckNoSuchDatabase() throws Exception
{
// No such database
- List<String> tableNames = client.listTableNamesByFilter("no_such_database",
+ client.listTableNamesByFilter("no_such_database",
hive_metastoreConstants.HIVE_FILTER_FIELD_LAST_ACCESS + ">2000",
(short)-1);
- Assert.assertEquals("Found tables", 0, tableNames.size());
}
@Test(expected = UnknownDBException.class)