wecharyu commented on code in PR #6427:
URL: https://github.com/apache/hive/pull/6427#discussion_r3081008985
##########
standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/HMSHandler.java:
##########
@@ -1420,149 +1416,33 @@ public TruncateTableResponse
truncate_table_req(TruncateTableRequest req)
}
@Override
- public List<ExtendedTableInfo> get_tables_ext(final GetTablesExtRequest req)
throws MetaException {
- List<String> tables = new ArrayList<String>();
- List<ExtendedTableInfo> ret = new ArrayList<ExtendedTableInfo>();
- String catalog = req.getCatalog();
- String database = req.getDatabase();
- String pattern = req.getTableNamePattern();
- List<String> processorCapabilities = req.getProcessorCapabilities();
- int limit = req.getLimit();
- String processorId = req.getProcessorIdentifier();
- List<Table> tObjects = new ArrayList<>();
-
- startTableFunction("get_tables_ext", catalog, database, pattern);
- Exception ex = null;
- try {
- tables = getMS().getTables(catalog, database, pattern, null, limit);
- LOG.debug("get_tables_ext:getTables() returned " + tables.size());
- tables = FilterUtils.filterTableNamesIfEnabled(isServerFilterEnabled,
filterHook,
- catalog, database, tables);
-
- if (tables.size() > 0) {
- tObjects = getMS().getTableObjectsByName(catalog, database, tables);
- LOG.debug("get_tables_ext:getTableObjectsByName() returned " +
tObjects.size());
- if (processorCapabilities == null || processorCapabilities.size() == 0
||
- processorCapabilities.contains("MANAGERAWMETADATA")) {
- LOG.info("Skipping translation for processor with " + processorId);
- } else {
- if (transformer != null) {
- Map<Table, List<String>> retMap = transformer.transform(tObjects,
processorCapabilities, processorId);
-
- for (Map.Entry<Table, List<String>> entry : retMap.entrySet()) {
- LOG.debug("Table " + entry.getKey().getTableName() + " requires
" + Arrays.toString((entry.getValue()).toArray()));
- ret.add(convertTableToExtendedTable(entry.getKey(),
entry.getValue(), req.getRequestedFields()));
- }
- } else {
- for (Table table : tObjects) {
- ret.add(convertTableToExtendedTable(table,
processorCapabilities, req.getRequestedFields()));
- }
- }
- }
- }
- } catch (Exception e) {
- ex = e;
- throw newMetaException(e);
- } finally {
- endFunction("get_tables_ext", ret != null, ex);
- }
- return ret;
- }
-
- private ExtendedTableInfo convertTableToExtendedTable (Table table,
- List<String>
processorCapabilities, int mask) {
- ExtendedTableInfo extTable = new ExtendedTableInfo(table.getTableName());
- if ((mask & GetTablesExtRequestFields.ACCESS_TYPE.getValue()) ==
GetTablesExtRequestFields.ACCESS_TYPE.getValue()) {
- extTable.setAccessType(table.getAccessType());
- }
-
- if ((mask & GetTablesExtRequestFields.PROCESSOR_CAPABILITIES.getValue())
- == GetTablesExtRequestFields.PROCESSOR_CAPABILITIES.getValue()) {
-
extTable.setRequiredReadCapabilities(table.getRequiredReadCapabilities());
-
extTable.setRequiredWriteCapabilities(table.getRequiredWriteCapabilities());
- }
-
- return extTable;
+ public List<ExtendedTableInfo> get_tables_ext(final GetTablesExtRequest req)
throws TException {
+ req.setCatalog(req.isSetCatalog() ? req.getCatalog() :
getDefaultCatalog(conf));
+ return GetTableHandler.getTables(
+ () -> startTableFunction("get_tables_ext", req.getCatalog(),
req.getDatabase(), req.getTableNamePattern()),
+ this, req,
+ t -> endFunction("get_tables_ext", t.getLeft() != null, t.getRight()));
}
@Override
public GetTableResult get_table_req(GetTableRequest req) throws
MetaException,
NoSuchObjectException {
req.setCatName(req.isSetCatName() ? req.getCatName() :
getDefaultCatalog(conf));
- return new GetTableResult(getTableInternal(req));
- }
-
- /**
- * This function retrieves table from metastore. If getColumnStats flag is
true,
- * then engine should be specified so the table is retrieve with the column
stats
- * for that engine.
- */
- private Table getTableInternal(GetTableRequest getTableRequest) throws
MetaException, NoSuchObjectException {
-
- Preconditions.checkArgument(!getTableRequest.isGetColumnStats() ||
getTableRequest.getEngine() != null,
- "To retrieve column statistics with a table, engine parameter cannot
be null");
-
- if (isInTest) {
- assertClientHasCapability(getTableRequest.getCapabilities(),
ClientCapability.TEST_CAPABILITY, "Hive tests",
- "get_table_req");
- }
-
- Table t = null;
- startTableFunction("get_table", getTableRequest.getCatName(),
getTableRequest.getDbName(),
- getTableRequest.getTblName());
- Exception ex = null;
- try {
- t = get_table_core(getTableRequest);
- if (MetaStoreUtils.isInsertOnlyTableParam(t.getParameters())) {
- assertClientHasCapability(getTableRequest.getCapabilities(),
ClientCapability.INSERT_ONLY_TABLES,
- "insert-only tables", "get_table_req");
- }
-
- if (CollectionUtils.isEmpty(getTableRequest.getProcessorCapabilities())
|| getTableRequest
- .getProcessorCapabilities().contains("MANAGERAWMETADATA")) {
- LOG.info("Skipping translation for processor with " +
getTableRequest.getProcessorIdentifier());
- } else {
- if (transformer != null) {
- List<Table> tList = new ArrayList<>();
- tList.add(t);
- Map<Table, List<String>> ret = transformer
- .transform(tList, getTableRequest.getProcessorCapabilities(),
getTableRequest.getProcessorIdentifier());
- if (ret.size() > 1) {
- LOG.warn("Unexpected resultset size:" + ret.size());
- throw new MetaException("Unexpected result from metadata
transformer:return list size is " + ret.size());
- }
- t = ret.keySet().iterator().next();
- }
- }
-
- firePreEvent(new PreReadTableEvent(t, this));
- } catch (MetaException | NoSuchObjectException e) {
- ex = e;
- throw e;
- } finally {
- endFunction("get_table", t != null, ex, getTableRequest.getTblName());
- }
- return t;
+ return new GetTableResult(GetTableHandler.getTable(
+ () -> startTableFunction("get_table", req.getCatName(),
req.getDbName(), req.getTblName()),
+ this, req, false,
+ t -> endFunction("get_table", t.getLeft() != null, t.getRight(),
req.getTblName())));
}
@Override
public List<TableMeta> get_table_meta(String dbnames, String tblNames,
List<String> tblTypes)
- throws MetaException, NoSuchObjectException {
- List<TableMeta> t = null;
+ throws TException {
String[] parsedDbName = parseDbName(dbnames, conf);
- startTableFunction("get_table_metas", parsedDbName[CAT_NAME],
parsedDbName[DB_NAME], tblNames);
- Exception ex = null;
- try {
- t = getMS().getTableMeta(parsedDbName[CAT_NAME], parsedDbName[DB_NAME],
tblNames, tblTypes);
- t = FilterUtils.filterTableMetasIfEnabled(isServerFilterEnabled,
filterHook, t);
- t = filterReadableTables(parsedDbName[CAT_NAME], t);
- } catch (Exception e) {
- ex = e;
- throw newMetaException(e);
- } finally {
- endFunction("get_table_metas", t != null, ex);
- }
- return t;
+ return GetTableHandler.getTables(
+ () -> startTableFunction("get_table_metas", parsedDbName[CAT_NAME],
parsedDbName[DB_NAME], tblNames),
Review Comment:
nit: get_table_metas => get_table_meta
##########
standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/HMSHandler.java:
##########
@@ -1622,189 +1464,30 @@ public Table get_table_core(GetTableRequest
getTableRequest) throws MetaExceptio
* are retrievable from the database specified by "dbnames."
* There is no guarantee of the order of the returned tables.
* If there are duplicate names, only one instance of the table will
be returned.
- * @throws MetaException
- * @throws InvalidOperationException
- * @throws UnknownDBException
+ * @throws TException
*/
-
@Override
public GetTablesResult get_table_objects_by_name_req(GetTablesRequest req)
throws TException {
- String catName = req.isSetCatName() ? req.getCatName() :
getDefaultCatalog(conf);
- if (isDatabaseRemote(req.getDbName())) {
- return new
GetTablesResult(getRemoteTableObjectsInternal(req.getDbName(),
req.getTblNames(), req.getTablesPattern()));
- }
- return new GetTablesResult(getTableObjectsInternal(catName,
req.getDbName(),
- req.getTblNames(), req.getCapabilities(), req.getProjectionSpec(),
req.getTablesPattern()));
- }
-
- private List<Table> filterTablesByName(List<Table> tables, List<String>
tableNames) {
- List<Table> filteredTables = new ArrayList<>();
- for (Table table : tables) {
- if (tableNames.contains(table.getTableName())) {
- filteredTables.add(table);
- }
- }
- return filteredTables;
- }
-
- private List<Table> getRemoteTableObjectsInternal(String dbname,
List<String> tableNames, String pattern) throws MetaException {
- String[] parsedDbName = parseDbName(dbname, conf);
- try {
- // retrieve tables from remote database
- Database db = get_database_core(parsedDbName[CAT_NAME],
parsedDbName[DB_NAME]);
- List<Table> tables =
DataConnectorProviderFactory.getDataConnectorProvider(db).getTables(null);
-
- // filtered out undesired tables
- if (tableNames != null) {
- tables = filterTablesByName(tables, tableNames);
- }
-
- // set remote tables' local hive database reference
- for (Table table : tables) {
- table.setDbName(dbname);
- }
-
- return FilterUtils.filterTablesIfEnabled(isServerFilterEnabled,
filterHook, tables);
- } catch (Exception e) {
- LOG.warn("Unexpected exception while getting table(s) in remote database
" + dbname , e);
- if (isInTest) {
- // ignore the exception
- return new ArrayList<Table>();
- } else {
- throw newMetaException(e);
- }
- }
- }
-
- private List<Table> getTableObjectsInternal(String catName, String dbName,
- List<String> tableNames,
- ClientCapabilities capabilities,
- GetProjectionsSpec
projectionsSpec, String tablePattern)
- throws MetaException, InvalidOperationException, UnknownDBException {
- if (isInTest) {
- assertClientHasCapability(capabilities, ClientCapability.TEST_CAPABILITY,
- "Hive tests", "get_table_objects_by_name_req");
- }
-
- if (projectionsSpec != null) {
- if (!projectionsSpec.isSetFieldList() &&
(projectionsSpec.isSetIncludeParamKeyPattern() ||
- projectionsSpec.isSetExcludeParamKeyPattern())) {
- throw new InvalidOperationException("Include and Exclude Param key are
not supported.");
- }
- }
-
- List<Table> tables = new ArrayList<>();
- startMultiTableFunction("get_multi_table", dbName, tableNames);
- Exception ex = null;
- int tableBatchSize = MetastoreConf.getIntVar(conf,
- ConfVars.BATCH_RETRIEVE_MAX);
-
- try {
- if (dbName == null || dbName.isEmpty()) {
- throw new UnknownDBException("DB name is null or empty");
- }
- RawStore ms = getMS();
- if(tablePattern != null){
- tables = ms.getTableObjectsByName(catName, dbName, tableNames,
projectionsSpec, tablePattern);
- }else {
- if (tableNames == null) {
- throw new InvalidOperationException(dbName + " cannot find null
tables");
- }
-
- // The list of table names could contain duplicates.
RawStore.getTableObjectsByName()
- // only guarantees returning no duplicate table objects in one batch.
If we need
- // to break into multiple batches, remove duplicates first.
- List<String> distinctTableNames = tableNames;
- if (distinctTableNames.size() > tableBatchSize) {
- List<String> lowercaseTableNames = new ArrayList<>();
- for (String tableName : tableNames) {
- lowercaseTableNames.add(normalizeIdentifier(tableName));
- }
- distinctTableNames = new ArrayList<>(new
HashSet<>(lowercaseTableNames));
- }
-
- int startIndex = 0;
- // Retrieve the tables from the metastore in batches. Some databases
like
- // Oracle cannot have over 1000 expressions in a in-list
- while (startIndex < distinctTableNames.size()) {
- int endIndex = Math.min(startIndex + tableBatchSize,
distinctTableNames.size());
- tables.addAll(ms.getTableObjectsByName(catName, dbName,
distinctTableNames.subList(
- startIndex, endIndex), projectionsSpec, tablePattern));
- startIndex = endIndex;
- }
- }
- for (Table t : tables) {
- if (t.getParameters() != null &&
MetaStoreUtils.isInsertOnlyTableParam(t.getParameters())) {
- assertClientHasCapability(capabilities,
ClientCapability.INSERT_ONLY_TABLES,
- "insert-only tables", "get_table_req");
- }
- }
-
- tables = FilterUtils.filterTablesIfEnabled(isServerFilterEnabled,
filterHook, tables);
- } catch (Exception e) {
- ex = e;
- throw handleException(e)
- .throwIfInstance(MetaException.class,
InvalidOperationException.class, UnknownDBException.class)
- .defaultMetaException();
- } finally {
- endFunction("get_multi_table", tables != null, ex, join(tableNames,
","));
- }
- return tables;
+ List<Table> tables = GetTableHandler.getTables(
+ () -> startMultiTableFunction("get_multi_table", req.getDbName(),
req.getTblNames()),
Review Comment:
nit: get_multi_table => get_table_objects
##########
standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/handler/GetTableHandler.java:
##########
@@ -0,0 +1,608 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hive.metastore.handler;
+
+import com.google.common.base.Preconditions;
+import com.google.common.base.Supplier;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.function.Consumer;
+import java.util.stream.Collectors;
+
+import org.apache.commons.collections4.CollectionUtils;
+import org.apache.commons.lang3.tuple.Pair;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hive.common.DatabaseName;
+import org.apache.hadoop.hive.common.StatsSetupConst;
+import org.apache.hadoop.hive.common.TableName;
+import org.apache.hadoop.hive.metastore.HMSHandler;
+import org.apache.hadoop.hive.metastore.IHMSHandler;
+import org.apache.hadoop.hive.metastore.IMetaStoreMetadataTransformer;
+import org.apache.hadoop.hive.metastore.MetaStoreFilterHook;
+import org.apache.hadoop.hive.metastore.RawStore;
+import org.apache.hadoop.hive.metastore.TableType;
+import org.apache.hadoop.hive.metastore.api.ClientCapabilities;
+import org.apache.hadoop.hive.metastore.api.ClientCapability;
+import org.apache.hadoop.hive.metastore.api.ColumnStatistics;
+import org.apache.hadoop.hive.metastore.api.Database;
+import org.apache.hadoop.hive.metastore.api.ExtendedTableInfo;
+import org.apache.hadoop.hive.metastore.api.GetProjectionsSpec;
+import org.apache.hadoop.hive.metastore.api.GetTableRequest;
+import org.apache.hadoop.hive.metastore.api.GetTablesExtRequest;
+import org.apache.hadoop.hive.metastore.api.GetTablesExtRequestFields;
+import org.apache.hadoop.hive.metastore.api.GetTablesRequest;
+import org.apache.hadoop.hive.metastore.api.InvalidOperationException;
+import org.apache.hadoop.hive.metastore.api.MetaException;
+import org.apache.hadoop.hive.metastore.api.NoSuchObjectException;
+import org.apache.hadoop.hive.metastore.api.Table;
+import org.apache.hadoop.hive.metastore.api.TableMeta;
+import org.apache.hadoop.hive.metastore.api.UnknownDBException;
+import org.apache.hadoop.hive.metastore.conf.MetastoreConf;
+import
org.apache.hadoop.hive.metastore.dataconnector.DataConnectorProviderFactory;
+import org.apache.hadoop.hive.metastore.events.PreEventContext;
+import org.apache.hadoop.hive.metastore.events.PreReadDatabaseEvent;
+import org.apache.hadoop.hive.metastore.events.PreReadTableEvent;
+import org.apache.hadoop.hive.metastore.utils.FilterUtils;
+import org.apache.hadoop.hive.metastore.utils.MetaStoreUtils;
+import org.apache.thrift.TException;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import static
org.apache.hadoop.hive.metastore.ExceptionHandler.handleException;
+import static
org.apache.hadoop.hive.metastore.ExceptionHandler.newMetaException;
+import static
org.apache.hadoop.hive.metastore.conf.MetastoreConf.ConfVars.HIVE_IN_TEST;
+import static org.apache.hadoop.hive.metastore.utils.MetaStoreUtils.CAT_NAME;
+import static org.apache.hadoop.hive.metastore.utils.MetaStoreUtils.DB_NAME;
+import static
org.apache.hadoop.hive.metastore.utils.MetaStoreUtils.getDefaultCatalog;
+import static
org.apache.hadoop.hive.metastore.utils.MetaStoreUtils.isDatabaseRemote;
+import static
org.apache.hadoop.hive.metastore.utils.MetaStoreUtils.parseDbName;
+import static
org.apache.hadoop.hive.metastore.utils.MetaStoreUtils.prependCatalogToDbName;
+import static
org.apache.hadoop.hive.metastore.utils.StringUtils.normalizeIdentifier;
+
+@SuppressWarnings({"unchecked", "rawtypes"})
+@RequestHandler(requestBody = GetTableHandler.GetTableReq.class)
+public class GetTableHandler<R, T> extends
+ AbstractRequestHandler<GetTableHandler.GetTableReq<R>,
GetTableHandler.GetTableResult<T>> {
+ private static final Logger LOG =
LoggerFactory.getLogger(GetTableHandler.class);
+ private RawStore ms;
+ private IMetaStoreMetadataTransformer transformer;
+ private MetaStoreFilterHook filterHook;
+ private Configuration conf;
+ private boolean isInTest;
+ GetTableHandler(IHMSHandler handler, GetTableReq request) {
+ super(handler, false, request);
+ }
+
+ @Override
+ protected void beforeExecute() throws TException, IOException {
+ this.ms = handler.getMS();
+ this.transformer = handler.getMetadataTransformer();
+ this.filterHook = handler.getMetaFilterHook();
+ this.conf = handler.getConf();
+
+ this.isInTest = MetastoreConf.getBoolVar(handler.getConf(), HIVE_IN_TEST);
+ }
+
+ @Override
+ protected GetTableResult execute() throws TException, IOException {
+ R req = request.getRequest();
+ if (req instanceof GetTablesExtRequest getTablesExtRequest) {
+ List<ExtendedTableInfo> extendedTableInfos =
getTablesExt(getTablesExtRequest);
+ return new GetTableResult(extendedTableInfos, true);
+ } else if (req instanceof GetTableRequest getTableRequest) {
+ Table table = request.rawTable ? getTableCore(getTableRequest) :
getTable(getTableRequest);
+ return new GetTableResult(List.of(table), true);
+ } else if (req instanceof GetTablesRequest getTablesRequest) {
+ List<Table> tables = getTableObjects(getTablesRequest);
+ return new GetTableResult(tables, true);
+ } else if (req instanceof GetTableNamesRequest getTableNames) {
+ return getTableNames.forTableMeta ? new
GetTableResult(getTableMeta(getTableNames), true) :
+ new GetTableResult(getTableNames(getTableNames), true);
+ }
+ throw new UnsupportedOperationException(req + " not yet implemented");
+ }
+
+ private List<ExtendedTableInfo> getTablesExt(GetTablesExtRequest req) throws
MetaException, UnknownDBException {
+ List<ExtendedTableInfo> ret = new ArrayList<ExtendedTableInfo>();
+ String pattern = req.getTableNamePattern();
+ List<String> processorCapabilities = req.getProcessorCapabilities();
+ int limit = req.getLimit();
+ String catalog = req.isSetCatalog() ? req.getCatalog() :
getDefaultCatalog(conf);
+ String database = req.getDatabase();
+ String processorId = req.getProcessorIdentifier();
+ List<String> tables = ms.getTables(catalog, database, pattern, null,
limit);
+ LOG.debug("get_tables_ext:getTables() returned {}", tables.size());
+ tables = FilterUtils.filterTableNamesIfEnabled(filterHook != null,
filterHook,
+ catalog, database, tables);
+ if (tables.isEmpty()) {
+ return ret;
+ }
+ List<Table> tObjects = ms.getTableObjectsByName(catalog, database, tables);
+ LOG.debug("get_tables_ext:getTableObjectsByName() returned {}",
tObjects.size());
+ if (processorCapabilities == null || processorCapabilities.isEmpty() ||
+ processorCapabilities.contains("MANAGERAWMETADATA")) {
+ LOG.info("Skipping translation for processor with {}", processorId);
+ } else {
+ if (transformer != null) {
+ Map<Table, List<String>> retMap = transformer.transform(tObjects,
processorCapabilities, processorId);
+ for (Map.Entry<Table, List<String>> entry : retMap.entrySet()) {
+ LOG.debug("Table " + entry.getKey().getTableName() + " requires " +
Arrays.toString((entry.getValue()).toArray()));
+ ret.add(convertTableToExtendedTable(entry.getKey(),
entry.getValue(), req.getRequestedFields()));
+ }
+ } else {
+ for (Table table : tObjects) {
+ ret.add(convertTableToExtendedTable(table, processorCapabilities,
req.getRequestedFields()));
+ }
+ }
+ }
+ return ret;
+ }
+
+ private ExtendedTableInfo convertTableToExtendedTable(Table table,
+ List<String> processorCapabilities, int mask) {
+ ExtendedTableInfo extTable = new ExtendedTableInfo(table.getTableName());
+ if ((mask & GetTablesExtRequestFields.ACCESS_TYPE.getValue()) ==
GetTablesExtRequestFields.ACCESS_TYPE.getValue()) {
+ extTable.setAccessType(table.getAccessType());
+ }
+
+ if ((mask & GetTablesExtRequestFields.PROCESSOR_CAPABILITIES.getValue())
+ == GetTablesExtRequestFields.PROCESSOR_CAPABILITIES.getValue()) {
+
extTable.setRequiredReadCapabilities(table.getRequiredReadCapabilities());
+
extTable.setRequiredWriteCapabilities(table.getRequiredWriteCapabilities());
+ }
+
+ return extTable;
+ }
+
+ /**
+ * This function retrieves table from metastore. If getColumnStats flag is
true,
+ * then engine should be specified so the table is retrieve with the column
stats
+ * for that engine.
+ */
+ private Table getTable(GetTableRequest getTableRequest) throws
MetaException, NoSuchObjectException {
Review Comment:
Could we add comments to highlight the differences from `getTableCore()`?
##########
standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/handler/GetTableHandler.java:
##########
@@ -0,0 +1,608 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hive.metastore.handler;
+
+import com.google.common.base.Preconditions;
+import com.google.common.base.Supplier;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.function.Consumer;
+import java.util.stream.Collectors;
+
+import org.apache.commons.collections4.CollectionUtils;
+import org.apache.commons.lang3.tuple.Pair;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hive.common.DatabaseName;
+import org.apache.hadoop.hive.common.StatsSetupConst;
+import org.apache.hadoop.hive.common.TableName;
+import org.apache.hadoop.hive.metastore.HMSHandler;
+import org.apache.hadoop.hive.metastore.IHMSHandler;
+import org.apache.hadoop.hive.metastore.IMetaStoreMetadataTransformer;
+import org.apache.hadoop.hive.metastore.MetaStoreFilterHook;
+import org.apache.hadoop.hive.metastore.RawStore;
+import org.apache.hadoop.hive.metastore.TableType;
+import org.apache.hadoop.hive.metastore.api.ClientCapabilities;
+import org.apache.hadoop.hive.metastore.api.ClientCapability;
+import org.apache.hadoop.hive.metastore.api.ColumnStatistics;
+import org.apache.hadoop.hive.metastore.api.Database;
+import org.apache.hadoop.hive.metastore.api.ExtendedTableInfo;
+import org.apache.hadoop.hive.metastore.api.GetProjectionsSpec;
+import org.apache.hadoop.hive.metastore.api.GetTableRequest;
+import org.apache.hadoop.hive.metastore.api.GetTablesExtRequest;
+import org.apache.hadoop.hive.metastore.api.GetTablesExtRequestFields;
+import org.apache.hadoop.hive.metastore.api.GetTablesRequest;
+import org.apache.hadoop.hive.metastore.api.InvalidOperationException;
+import org.apache.hadoop.hive.metastore.api.MetaException;
+import org.apache.hadoop.hive.metastore.api.NoSuchObjectException;
+import org.apache.hadoop.hive.metastore.api.Table;
+import org.apache.hadoop.hive.metastore.api.TableMeta;
+import org.apache.hadoop.hive.metastore.api.UnknownDBException;
+import org.apache.hadoop.hive.metastore.conf.MetastoreConf;
+import
org.apache.hadoop.hive.metastore.dataconnector.DataConnectorProviderFactory;
+import org.apache.hadoop.hive.metastore.events.PreEventContext;
+import org.apache.hadoop.hive.metastore.events.PreReadDatabaseEvent;
+import org.apache.hadoop.hive.metastore.events.PreReadTableEvent;
+import org.apache.hadoop.hive.metastore.utils.FilterUtils;
+import org.apache.hadoop.hive.metastore.utils.MetaStoreUtils;
+import org.apache.thrift.TException;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import static
org.apache.hadoop.hive.metastore.ExceptionHandler.handleException;
+import static
org.apache.hadoop.hive.metastore.ExceptionHandler.newMetaException;
+import static
org.apache.hadoop.hive.metastore.conf.MetastoreConf.ConfVars.HIVE_IN_TEST;
+import static org.apache.hadoop.hive.metastore.utils.MetaStoreUtils.CAT_NAME;
+import static org.apache.hadoop.hive.metastore.utils.MetaStoreUtils.DB_NAME;
+import static
org.apache.hadoop.hive.metastore.utils.MetaStoreUtils.getDefaultCatalog;
+import static
org.apache.hadoop.hive.metastore.utils.MetaStoreUtils.isDatabaseRemote;
+import static
org.apache.hadoop.hive.metastore.utils.MetaStoreUtils.parseDbName;
+import static
org.apache.hadoop.hive.metastore.utils.MetaStoreUtils.prependCatalogToDbName;
+import static
org.apache.hadoop.hive.metastore.utils.StringUtils.normalizeIdentifier;
+
+@SuppressWarnings({"unchecked", "rawtypes"})
+@RequestHandler(requestBody = GetTableHandler.GetTableReq.class)
+public class GetTableHandler<R, T> extends
+ AbstractRequestHandler<GetTableHandler.GetTableReq<R>,
GetTableHandler.GetTableResult<T>> {
+ private static final Logger LOG =
LoggerFactory.getLogger(GetTableHandler.class);
+ private RawStore ms;
+ private IMetaStoreMetadataTransformer transformer;
+ private MetaStoreFilterHook filterHook;
+ private Configuration conf;
+ private boolean isInTest;
+ GetTableHandler(IHMSHandler handler, GetTableReq request) {
+ super(handler, false, request);
+ }
+
+ @Override
+ protected void beforeExecute() throws TException, IOException {
+ this.ms = handler.getMS();
+ this.transformer = handler.getMetadataTransformer();
+ this.filterHook = handler.getMetaFilterHook();
+ this.conf = handler.getConf();
+
+ this.isInTest = MetastoreConf.getBoolVar(handler.getConf(), HIVE_IN_TEST);
+ }
+
+ @Override
+ protected GetTableResult execute() throws TException, IOException {
+ R req = request.getRequest();
+ if (req instanceof GetTablesExtRequest getTablesExtRequest) {
+ List<ExtendedTableInfo> extendedTableInfos =
getTablesExt(getTablesExtRequest);
+ return new GetTableResult(extendedTableInfos, true);
+ } else if (req instanceof GetTableRequest getTableRequest) {
+ Table table = request.rawTable ? getTableCore(getTableRequest) :
getTable(getTableRequest);
+ return new GetTableResult(List.of(table), true);
+ } else if (req instanceof GetTablesRequest getTablesRequest) {
+ List<Table> tables = getTableObjects(getTablesRequest);
+ return new GetTableResult(tables, true);
+ } else if (req instanceof GetTableNamesRequest getTableNames) {
+ return getTableNames.forTableMeta ? new
GetTableResult(getTableMeta(getTableNames), true) :
+ new GetTableResult(getTableNames(getTableNames), true);
+ }
+ throw new UnsupportedOperationException(req + " not yet implemented");
+ }
+
+ private List<ExtendedTableInfo> getTablesExt(GetTablesExtRequest req) throws
MetaException, UnknownDBException {
+ List<ExtendedTableInfo> ret = new ArrayList<ExtendedTableInfo>();
+ String pattern = req.getTableNamePattern();
+ List<String> processorCapabilities = req.getProcessorCapabilities();
+ int limit = req.getLimit();
+ String catalog = req.isSetCatalog() ? req.getCatalog() :
getDefaultCatalog(conf);
+ String database = req.getDatabase();
+ String processorId = req.getProcessorIdentifier();
+ List<String> tables = ms.getTables(catalog, database, pattern, null,
limit);
+ LOG.debug("get_tables_ext:getTables() returned {}", tables.size());
+ tables = FilterUtils.filterTableNamesIfEnabled(filterHook != null,
filterHook,
+ catalog, database, tables);
+ if (tables.isEmpty()) {
+ return ret;
+ }
+ List<Table> tObjects = ms.getTableObjectsByName(catalog, database, tables);
+ LOG.debug("get_tables_ext:getTableObjectsByName() returned {}",
tObjects.size());
+ if (processorCapabilities == null || processorCapabilities.isEmpty() ||
+ processorCapabilities.contains("MANAGERAWMETADATA")) {
+ LOG.info("Skipping translation for processor with {}", processorId);
+ } else {
+ if (transformer != null) {
+ Map<Table, List<String>> retMap = transformer.transform(tObjects,
processorCapabilities, processorId);
+ for (Map.Entry<Table, List<String>> entry : retMap.entrySet()) {
+ LOG.debug("Table " + entry.getKey().getTableName() + " requires " +
Arrays.toString((entry.getValue()).toArray()));
+ ret.add(convertTableToExtendedTable(entry.getKey(),
entry.getValue(), req.getRequestedFields()));
+ }
+ } else {
+ for (Table table : tObjects) {
+ ret.add(convertTableToExtendedTable(table, processorCapabilities,
req.getRequestedFields()));
+ }
+ }
+ }
+ return ret;
+ }
+
+ private ExtendedTableInfo convertTableToExtendedTable(Table table,
+ List<String> processorCapabilities, int mask) {
+ ExtendedTableInfo extTable = new ExtendedTableInfo(table.getTableName());
+ if ((mask & GetTablesExtRequestFields.ACCESS_TYPE.getValue()) ==
GetTablesExtRequestFields.ACCESS_TYPE.getValue()) {
+ extTable.setAccessType(table.getAccessType());
+ }
+
+ if ((mask & GetTablesExtRequestFields.PROCESSOR_CAPABILITIES.getValue())
+ == GetTablesExtRequestFields.PROCESSOR_CAPABILITIES.getValue()) {
+
extTable.setRequiredReadCapabilities(table.getRequiredReadCapabilities());
+
extTable.setRequiredWriteCapabilities(table.getRequiredWriteCapabilities());
+ }
+
+ return extTable;
+ }
+
+ /**
+ * This function retrieves table from metastore. If getColumnStats flag is
true,
+ * then engine should be specified so the table is retrieve with the column
stats
+ * for that engine.
+ */
+ private Table getTable(GetTableRequest getTableRequest) throws
MetaException, NoSuchObjectException {
+
+ Preconditions.checkArgument(!getTableRequest.isGetColumnStats() ||
getTableRequest.getEngine() != null,
+ "To retrieve column statistics with a table, engine parameter cannot
be null");
+
+ if (isInTest) {
+ assertClientHasCapability(getTableRequest.getCapabilities(),
ClientCapability.TEST_CAPABILITY, "Hive tests",
+ "get_table_req");
+ }
+
+ Table t = getTableCore(getTableRequest);
+ if (MetaStoreUtils.isInsertOnlyTableParam(t.getParameters())) {
+ assertClientHasCapability(getTableRequest.getCapabilities(),
ClientCapability.INSERT_ONLY_TABLES,
+ "insert-only tables", "get_table_req");
+ }
+
+ if (CollectionUtils.isEmpty(getTableRequest.getProcessorCapabilities()) ||
getTableRequest
+ .getProcessorCapabilities().contains("MANAGERAWMETADATA")) {
+ LOG.info("Skipping translation for processor with " +
getTableRequest.getProcessorIdentifier());
+ } else {
+ if (transformer != null) {
+ List<Table> tList = new ArrayList<>();
+ tList.add(t);
+ Map<Table, List<String>> ret = transformer
+ .transform(tList, getTableRequest.getProcessorCapabilities(),
getTableRequest.getProcessorIdentifier());
+ if (ret.size() > 1) {
+ LOG.warn("Unexpected resultset size:{}", ret.size());
+ throw new MetaException("Unexpected result from metadata
transformer:return list size is " + ret.size());
+ }
+ t = ret.keySet().iterator().next();
+ }
+ }
+
+ ((HMSHandler) handler).firePreEvent(new PreReadTableEvent(t, handler));
+ return t;
+ }
+
+ /**
+ * This function retrieves table from metastore. If getColumnStats flag is
true,
+ * then engine should be specified so the table is retrieve with the column
stats
+ * for that engine.
+ */
+ private Table getTableCore(GetTableRequest getTableRequest) throws
MetaException, NoSuchObjectException {
+ Preconditions.checkArgument(!getTableRequest.isGetColumnStats() ||
getTableRequest.getEngine() != null,
+ "To retrieve column statistics with a table, engine parameter cannot
be null");
+ String catName = getTableRequest.getCatName();
+ String dbName = getTableRequest.getDbName();
+ String tblName = getTableRequest.getTblName();
+ Database db = null;
+ Table t;
+ try {
+ db = handler.get_database_core(catName, dbName);
+ } catch (Exception e) { /* appears exception is not thrown currently if db
doesnt exist */ }
+
+ if (MetaStoreUtils.isDatabaseRemote(db)) {
+ t =
DataConnectorProviderFactory.getDataConnectorProvider(db).getTable(tblName);
+ if (t == null) {
+ throw new NoSuchObjectException(TableName.getQualified(catName,
dbName, tblName) + " table not found");
+ }
+ t.setDbName(dbName);
+ return t;
+ }
+
+ t = ms.getTable(catName, dbName, tblName,
getTableRequest.getValidWriteIdList(), getTableRequest.getId());
+ if (t == null) {
+ throw new NoSuchObjectException(TableName.getQualified(catName, dbName,
tblName) + " table not found");
+ }
+
+ // If column statistics was requested and is valid fetch it.
+ if (getTableRequest.isGetColumnStats()) {
+ ColumnStatistics colStats = ms.getTableColumnStatistics(catName, dbName,
tblName,
+ StatsSetupConst.getColumnsHavingStats(t.getParameters()),
getTableRequest.getEngine(),
+ getTableRequest.getValidWriteIdList());
+ if (colStats != null) {
+ t.setColStats(colStats);
+ }
+ }
+ return t;
+ }
+
+ private List<Table> getTableObjects(GetTablesRequest req) throws TException {
+ String catName = req.isSetCatName() ? req.getCatName() :
getDefaultCatalog(conf);
+ String dbName = req.getDbName();
+ if (dbName == null || dbName.isEmpty()) {
+ throw new UnknownDBException("DB name is null or empty");
+ }
+ try {
+ Database database = handler.get_database_core(catName, dbName);
+ if (isDatabaseRemote(database)) {
+ return getRemoteTableObjectsInternal(database, req.getTblNames(),
req.getTablesPattern());
+ }
+ } catch (NoSuchObjectException nse) {
+ // The caller consumes UnknownDBException other than
NoSuchObjectException
+ // in case database doesn't exist
+ throw new UnknownDBException("Could not find database " +
DatabaseName.getQualified(catName, dbName));
+ }
+ return getTableObjectsInternal(req);
+ }
+
+ private List<Table> filterTablesByName(List<Table> tables, List<String>
tableNames) {
+ List<Table> filteredTables = new ArrayList<>();
+ for (Table table : tables) {
+ if (tableNames.contains(table.getTableName())) {
+ filteredTables.add(table);
+ }
+ }
+ return filteredTables;
+ }
+
+ private List<Table> getRemoteTableObjectsInternal(Database db, List<String>
tableNames, String pattern) throws MetaException {
+ try {
+ List<Table> tables =
DataConnectorProviderFactory.getDataConnectorProvider(db).getTables(null);
+ // filtered out undesired tables
+ if (tableNames != null) {
+ tables = filterTablesByName(tables, tableNames);
+ }
+ // set remote tables' local hive database reference
+ for (Table table : tables) {
+ table.setDbName(db.getName());
+ }
+ return FilterUtils.filterTablesIfEnabled(filterHook != null, filterHook,
tables);
+ } catch (Exception e) {
+ LOG.warn("Unexpected exception while getting table(s) in remote database
" + db.getName() , e);
+ if (isInTest) {
+ // ignore the exception
+ return new ArrayList<Table>();
+ } else {
+ throw newMetaException(e);
+ }
+ }
+ }
+
+ private List<Table> getTableObjectsInternal(GetTablesRequest req)
+ throws MetaException, InvalidOperationException, UnknownDBException {
+ if (isInTest) {
+ assertClientHasCapability(req.getCapabilities(),
ClientCapability.TEST_CAPABILITY,
+ "Hive tests", "get_table_objects_by_name_req");
+ }
+
+ GetProjectionsSpec projectionsSpec = req.getProjectionSpec();
+ if (projectionsSpec != null) {
+ if (!projectionsSpec.isSetFieldList() &&
(projectionsSpec.isSetIncludeParamKeyPattern() ||
+ projectionsSpec.isSetExcludeParamKeyPattern())) {
+ throw new InvalidOperationException("Include and Exclude Param key are
not supported.");
+ }
+ }
+
+ String dbName = req.getDbName();
+ String catName = req.isSetCatName() ? req.getCatName() :
getDefaultCatalog(conf);
+ List<Table> tables = new ArrayList<>();
+ int tableBatchSize = MetastoreConf.getIntVar(conf,
MetastoreConf.ConfVars.BATCH_RETRIEVE_MAX);
+ List<String> tableNames = req.getTblNames();
+ if(req.getTablesPattern() != null) {
+ tables = ms.getTableObjectsByName(catName, dbName, tableNames,
projectionsSpec, req.getTablesPattern());
+ } else {
+ if (tableNames == null) {
+ throw new InvalidOperationException(dbName + " cannot find null
tables");
+ }
+
+ // The list of table names could contain duplicates.
RawStore.getTableObjectsByName()
+ // only guarantees returning no duplicate table objects in one batch. If
we need
+ // to break into multiple batches, remove duplicates first.
+ List<String> distinctTableNames = tableNames;
+ if (distinctTableNames.size() > tableBatchSize) {
+ List<String> lowercaseTableNames = new ArrayList<>();
+ for (String tableName : tableNames) {
+ lowercaseTableNames.add(normalizeIdentifier(tableName));
+ }
+ distinctTableNames = new ArrayList<>(new
HashSet<>(lowercaseTableNames));
+ }
+
+ int startIndex = 0;
+ // Retrieve the tables from the metastore in batches. Some databases like
+ // Oracle cannot have over 1000 expressions in a in-list
+ while (startIndex < distinctTableNames.size()) {
+ int endIndex = Math.min(startIndex + tableBatchSize,
distinctTableNames.size());
+ tables.addAll(ms.getTableObjectsByName(catName, dbName,
distinctTableNames.subList(
+ startIndex, endIndex), projectionsSpec, null));
+ startIndex = endIndex;
+ }
+ }
+ for (Table t : tables) {
+ if (t.getParameters() != null &&
MetaStoreUtils.isInsertOnlyTableParam(t.getParameters())) {
+ assertClientHasCapability(req.getCapabilities(),
ClientCapability.INSERT_ONLY_TABLES,
+ "insert-only tables", "get_table_req");
+ }
+ }
+
+ tables = FilterUtils.filterTablesIfEnabled(filterHook != null, filterHook,
tables);
+ return tables;
+ }
+
+ private void assertClientHasCapability(ClientCapabilities client,
+ ClientCapability value, String what, String call) throws MetaException {
+ if (!doesClientHaveCapability(client, value)) {
+ throw new MetaException("Your client does not appear to support " + what
+ ". To skip"
+ + " capability checks, please set " +
MetastoreConf.ConfVars.CAPABILITY_CHECK.toString()
+ + " to false. This setting can be set globally, or on the client for
the current"
+ + " metastore session. Note that this may lead to incorrect results,
data loss,"
+ + " undefined behavior, etc. if your client is actually
incompatible. You can also"
+ + " specify custom client capabilities via " + call + " API.");
+ }
+ }
+
+ private boolean doesClientHaveCapability(ClientCapabilities client,
ClientCapability value) {
+ if (!MetastoreConf.getBoolVar(conf,
MetastoreConf.ConfVars.CAPABILITY_CHECK)) {
+ return true;
+ }
+ return (client != null && client.isSetValues() &&
client.getValues().contains(value));
+ }
+
+ private List<TableMeta> getTableMeta(GetTableNamesRequest getNamesReq)
throws TException {
+ String catName = getNamesReq.catName;
+ String dbname = getNamesReq.dbName;
+ List<String> tblTypes = null;
+ if (getNamesReq.tableType != null) {
+ tblTypes = Arrays.asList(getNamesReq.tableType.split(","));
+ }
+ List<TableMeta> t = ms.getTableMeta(catName, dbname, getNamesReq.pattern,
tblTypes);
+ t = FilterUtils.filterTableMetasIfEnabled(filterHook != null, filterHook,
t);
+ return filterReadableTables(catName, t);
+ }
+
+ /**
+ * filters out the table meta for which read database access is not granted
+ * @param catName catalog name
+ * @param tableMetas list of table metas
+ * @return filtered list of table metas
+ * @throws RuntimeException
+ * @throws NoSuchObjectException
+ */
+ private List<TableMeta> filterReadableTables(String catName, List<TableMeta>
tableMetas)
+ throws RuntimeException, NoSuchObjectException {
+ List<TableMeta> finalT = new ArrayList<>();
+ Map<String, Boolean> databaseNames = new HashMap();
+ for (TableMeta tableMeta : tableMetas) {
+ String fullDbName = prependCatalogToDbName(catName,
tableMeta.getDbName(), conf);
+ if (databaseNames.get(fullDbName) == null) {
+ boolean isExecptionThrown = false;
+ try {
+ fireReadDatabasePreEvent(fullDbName);
+ } catch (MetaException e) {
+ isExecptionThrown = true;
+ }
+ databaseNames.put(fullDbName, isExecptionThrown);
+ }
+ if (!databaseNames.get(fullDbName)) {
+ finalT.add(tableMeta);
+ }
+ }
+ return finalT;
+ }
+
+ /**
+ * Fire a pre-event for read database operation, if there are any
+ * pre-event listeners registered
+ */
+ private void fireReadDatabasePreEvent(final String name)
+ throws MetaException, RuntimeException, NoSuchObjectException {
+ Supplier<PreEventContext> supplier = () -> {
+ String[] parsedDbName = parseDbName(name, conf);
+ Database db = null;
+ try {
+ db = handler.get_database_core(parsedDbName[CAT_NAME],
parsedDbName[DB_NAME]);
+ if (db == null) {
+ throw new NoSuchObjectException("Database: " + name + " not found");
+ }
+ } catch(MetaException | NoSuchObjectException e) {
+ throw new RuntimeException(e);
+ }
+ return new PreReadDatabaseEvent(db, handler);
+ };
+ ((HMSHandler) handler).firePreEvent(supplier);
+ }
+
+
+ private List<String> getTableNames(GetTableNamesRequest getNamesReq) throws
TException {
+ String catName = getNamesReq.catName;
+ String dbname = getNamesReq.dbName;
+ try {
+ Database database = handler.get_database_core(catName, dbname);
+ if (isDatabaseRemote(database)) {
+ return
DataConnectorProviderFactory.getDataConnectorProvider(database).getTableNames();
Review Comment:
It does not filter out the result by possible filter or pattern.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
