HIVE-10319: Hive CLI startup takes a long time with a large number of databases (Nezih Yigitbasi, reviewed by Jason Dere)
Project: http://git-wip-us.apache.org/repos/asf/hive/repo Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/cb648903 Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/cb648903 Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/cb648903 Branch: refs/heads/master Commit: cb648903c7afa9a17420248f7956d85c653f6987 Parents: 607f23d Author: Jason Dere <jd...@hortonworks.com> Authored: Mon Aug 3 17:51:35 2015 -0700 Committer: Jason Dere <jd...@hortonworks.com> Committed: Mon Aug 3 17:51:35 2015 -0700 ---------------------------------------------------------------------- .../hive/metastore/TestHiveMetaStore.java | 30 +- metastore/if/hive_metastore.thrift | 5 + .../gen/thrift/gen-cpp/ThriftHiveMetastore.cpp | 2046 +++++++----- .../gen/thrift/gen-cpp/ThriftHiveMetastore.h | 135 + .../ThriftHiveMetastore_server.skeleton.cpp | 5 + .../gen/thrift/gen-cpp/hive_metastore_types.cpp | 304 +- .../gen/thrift/gen-cpp/hive_metastore_types.h | 47 + .../hive/metastore/api/AbortTxnRequest.java | 2 +- .../metastore/api/AddDynamicPartitions.java | 2 +- .../metastore/api/AddPartitionsRequest.java | 2 +- .../hive/metastore/api/AddPartitionsResult.java | 2 +- .../hadoop/hive/metastore/api/AggrStats.java | 2 +- .../metastore/api/AlreadyExistsException.java | 2 +- .../metastore/api/BinaryColumnStatsData.java | 2 +- .../metastore/api/BooleanColumnStatsData.java | 2 +- .../hive/metastore/api/CheckLockRequest.java | 2 +- .../hive/metastore/api/ColumnStatistics.java | 2 +- .../metastore/api/ColumnStatisticsDesc.java | 2 +- .../hive/metastore/api/ColumnStatisticsObj.java | 2 +- .../hive/metastore/api/CommitTxnRequest.java | 2 +- .../hive/metastore/api/CompactionRequest.java | 2 +- .../api/ConfigValSecurityException.java | 2 +- .../api/CurrentNotificationEventId.java | 2 +- .../hadoop/hive/metastore/api/Database.java | 2 +- .../apache/hadoop/hive/metastore/api/Date.java | 2 +- .../hive/metastore/api/DateColumnStatsData.java | 2 +- .../hadoop/hive/metastore/api/Decimal.java | 2 +- .../metastore/api/DecimalColumnStatsData.java | 2 +- .../metastore/api/DoubleColumnStatsData.java | 2 +- .../hive/metastore/api/DropPartitionsExpr.java | 2 +- .../metastore/api/DropPartitionsRequest.java | 2 +- .../metastore/api/DropPartitionsResult.java | 2 +- .../hive/metastore/api/EnvironmentContext.java | 2 +- .../hadoop/hive/metastore/api/FieldSchema.java | 2 +- .../hive/metastore/api/FireEventRequest.java | 2 +- .../hive/metastore/api/FireEventResponse.java | 2 +- .../hadoop/hive/metastore/api/Function.java | 2 +- .../metastore/api/GetAllFunctionsResponse.java | 447 +++ .../metastore/api/GetOpenTxnsInfoResponse.java | 2 +- .../hive/metastore/api/GetOpenTxnsResponse.java | 2 +- .../api/GetPrincipalsInRoleRequest.java | 2 +- .../api/GetPrincipalsInRoleResponse.java | 2 +- .../api/GetRoleGrantsForPrincipalRequest.java | 2 +- .../api/GetRoleGrantsForPrincipalResponse.java | 2 +- .../api/GrantRevokePrivilegeRequest.java | 2 +- .../api/GrantRevokePrivilegeResponse.java | 2 +- .../metastore/api/GrantRevokeRoleRequest.java | 2 +- .../metastore/api/GrantRevokeRoleResponse.java | 2 +- .../hive/metastore/api/HeartbeatRequest.java | 2 +- .../metastore/api/HeartbeatTxnRangeRequest.java | 2 +- .../api/HeartbeatTxnRangeResponse.java | 2 +- .../hive/metastore/api/HiveObjectPrivilege.java | 2 +- .../hive/metastore/api/HiveObjectRef.java | 2 +- .../apache/hadoop/hive/metastore/api/Index.java | 2 +- .../api/IndexAlreadyExistsException.java | 2 +- .../metastore/api/InsertEventRequestData.java | 2 +- .../metastore/api/InvalidInputException.java | 2 +- .../metastore/api/InvalidObjectException.java | 2 +- .../api/InvalidOperationException.java | 2 +- .../api/InvalidPartitionException.java | 2 +- .../hive/metastore/api/LockComponent.java | 2 +- .../hadoop/hive/metastore/api/LockRequest.java | 2 +- .../hadoop/hive/metastore/api/LockResponse.java | 2 +- .../hive/metastore/api/LongColumnStatsData.java | 2 +- .../hive/metastore/api/MetaException.java | 2 +- .../hive/metastore/api/NoSuchLockException.java | 2 +- .../metastore/api/NoSuchObjectException.java | 2 +- .../hive/metastore/api/NoSuchTxnException.java | 2 +- .../hive/metastore/api/NotificationEvent.java | 2 +- .../metastore/api/NotificationEventRequest.java | 2 +- .../api/NotificationEventResponse.java | 2 +- .../hive/metastore/api/OpenTxnRequest.java | 2 +- .../hive/metastore/api/OpenTxnsResponse.java | 2 +- .../apache/hadoop/hive/metastore/api/Order.java | 2 +- .../hadoop/hive/metastore/api/Partition.java | 2 +- .../api/PartitionListComposingSpec.java | 2 +- .../hive/metastore/api/PartitionSpec.java | 2 +- .../api/PartitionSpecWithSharedSD.java | 2 +- .../hive/metastore/api/PartitionWithoutSD.java | 2 +- .../metastore/api/PartitionsByExprRequest.java | 2 +- .../metastore/api/PartitionsByExprResult.java | 2 +- .../metastore/api/PartitionsStatsRequest.java | 2 +- .../metastore/api/PartitionsStatsResult.java | 2 +- .../metastore/api/PrincipalPrivilegeSet.java | 2 +- .../hadoop/hive/metastore/api/PrivilegeBag.java | 2 +- .../hive/metastore/api/PrivilegeGrantInfo.java | 2 +- .../hadoop/hive/metastore/api/ResourceUri.java | 2 +- .../apache/hadoop/hive/metastore/api/Role.java | 2 +- .../hive/metastore/api/RolePrincipalGrant.java | 2 +- .../hadoop/hive/metastore/api/Schema.java | 2 +- .../hadoop/hive/metastore/api/SerDeInfo.java | 2 +- .../api/SetPartitionsStatsRequest.java | 2 +- .../hive/metastore/api/ShowCompactRequest.java | 2 +- .../hive/metastore/api/ShowCompactResponse.java | 2 +- .../api/ShowCompactResponseElement.java | 2 +- .../hive/metastore/api/ShowLocksRequest.java | 2 +- .../hive/metastore/api/ShowLocksResponse.java | 2 +- .../metastore/api/ShowLocksResponseElement.java | 2 +- .../hadoop/hive/metastore/api/SkewedInfo.java | 2 +- .../hive/metastore/api/StorageDescriptor.java | 2 +- .../metastore/api/StringColumnStatsData.java | 2 +- .../apache/hadoop/hive/metastore/api/Table.java | 2 +- .../hive/metastore/api/TableStatsRequest.java | 2 +- .../hive/metastore/api/TableStatsResult.java | 2 +- .../hive/metastore/api/ThriftHiveMetastore.java | 3140 +++++++++++------- .../hive/metastore/api/TxnAbortedException.java | 2 +- .../hadoop/hive/metastore/api/TxnInfo.java | 2 +- .../hive/metastore/api/TxnOpenException.java | 2 +- .../apache/hadoop/hive/metastore/api/Type.java | 2 +- .../hive/metastore/api/UnknownDBException.java | 2 +- .../api/UnknownPartitionException.java | 2 +- .../metastore/api/UnknownTableException.java | 2 +- .../hive/metastore/api/UnlockRequest.java | 2 +- .../hadoop/hive/metastore/api/Version.java | 2 +- .../gen-php/metastore/ThriftHiveMetastore.php | 1265 ++++--- .../src/gen/thrift/gen-php/metastore/Types.php | 103 + .../hive_metastore/ThriftHiveMetastore-remote | 21 +- .../hive_metastore/ThriftHiveMetastore.py | 879 +++-- .../gen/thrift/gen-py/hive_metastore/ttypes.py | 74 + .../gen/thrift/gen-rb/hive_metastore_types.rb | 16 + .../gen/thrift/gen-rb/thrift_hive_metastore.rb | 60 + .../hadoop/hive/metastore/HiveMetaStore.java | 21 + .../hive/metastore/HiveMetaStoreClient.java | 7 + .../hadoop/hive/metastore/IMetaStoreClient.java | 4 + .../hadoop/hive/metastore/ObjectStore.java | 28 + .../apache/hadoop/hive/metastore/RawStore.java | 7 + .../DummyRawStoreControlledCommit.java | 7 + .../DummyRawStoreForJdoConnection.java | 6 + .../hadoop/hive/ql/plan/api/Adjacency.java | 2 +- .../apache/hadoop/hive/ql/plan/api/Graph.java | 2 +- .../hadoop/hive/ql/plan/api/Operator.java | 2 +- .../apache/hadoop/hive/ql/plan/api/Query.java | 2 +- .../hadoop/hive/ql/plan/api/QueryPlan.java | 2 +- .../apache/hadoop/hive/ql/plan/api/Stage.java | 2 +- .../apache/hadoop/hive/ql/plan/api/Task.java | 2 +- .../apache/hadoop/hive/ql/metadata/Hive.java | 30 +- .../hadoop/hive/serde/test/InnerStruct.java | 2 +- .../hadoop/hive/serde/test/ThriftTestObj.java | 2 +- .../hadoop/hive/serde2/thrift/test/Complex.java | 2 +- .../hive/serde2/thrift/test/IntString.java | 2 +- .../hive/serde2/thrift/test/MegaStruct.java | 2 +- .../hive/serde2/thrift/test/MiniStruct.java | 2 +- .../hive/serde2/thrift/test/SetIntString.java | 2 +- .../hadoop/hive/service/HiveClusterStatus.java | 2 +- .../hive/service/HiveServerException.java | 2 +- .../apache/hadoop/hive/service/ThriftHive.java | 2 +- .../service/cli/thrift/TArrayTypeEntry.java | 2 +- .../hive/service/cli/thrift/TBinaryColumn.java | 2 +- .../hive/service/cli/thrift/TBoolColumn.java | 2 +- .../hive/service/cli/thrift/TBoolValue.java | 2 +- .../hive/service/cli/thrift/TByteColumn.java | 2 +- .../hive/service/cli/thrift/TByteValue.java | 2 +- .../hive/service/cli/thrift/TCLIService.java | 2 +- .../cli/thrift/TCancelDelegationTokenReq.java | 2 +- .../cli/thrift/TCancelDelegationTokenResp.java | 2 +- .../service/cli/thrift/TCancelOperationReq.java | 2 +- .../cli/thrift/TCancelOperationResp.java | 2 +- .../service/cli/thrift/TCloseOperationReq.java | 2 +- .../service/cli/thrift/TCloseOperationResp.java | 2 +- .../service/cli/thrift/TCloseSessionReq.java | 2 +- .../service/cli/thrift/TCloseSessionResp.java | 2 +- .../hive/service/cli/thrift/TColumnDesc.java | 2 +- .../hive/service/cli/thrift/TDoubleColumn.java | 2 +- .../hive/service/cli/thrift/TDoubleValue.java | 2 +- .../cli/thrift/TExecuteStatementReq.java | 2 +- .../cli/thrift/TExecuteStatementResp.java | 2 +- .../service/cli/thrift/TFetchResultsReq.java | 2 +- .../service/cli/thrift/TFetchResultsResp.java | 2 +- .../service/cli/thrift/TGetCatalogsReq.java | 2 +- .../service/cli/thrift/TGetCatalogsResp.java | 2 +- .../hive/service/cli/thrift/TGetColumnsReq.java | 2 +- .../service/cli/thrift/TGetColumnsResp.java | 2 +- .../cli/thrift/TGetDelegationTokenReq.java | 2 +- .../cli/thrift/TGetDelegationTokenResp.java | 2 +- .../service/cli/thrift/TGetFunctionsReq.java | 2 +- .../service/cli/thrift/TGetFunctionsResp.java | 2 +- .../hive/service/cli/thrift/TGetInfoReq.java | 2 +- .../hive/service/cli/thrift/TGetInfoResp.java | 2 +- .../cli/thrift/TGetOperationStatusReq.java | 2 +- .../cli/thrift/TGetOperationStatusResp.java | 2 +- .../cli/thrift/TGetResultSetMetadataReq.java | 2 +- .../cli/thrift/TGetResultSetMetadataResp.java | 2 +- .../hive/service/cli/thrift/TGetSchemasReq.java | 2 +- .../service/cli/thrift/TGetSchemasResp.java | 2 +- .../service/cli/thrift/TGetTableTypesReq.java | 2 +- .../service/cli/thrift/TGetTableTypesResp.java | 2 +- .../hive/service/cli/thrift/TGetTablesReq.java | 2 +- .../hive/service/cli/thrift/TGetTablesResp.java | 2 +- .../service/cli/thrift/TGetTypeInfoReq.java | 2 +- .../service/cli/thrift/TGetTypeInfoResp.java | 2 +- .../service/cli/thrift/THandleIdentifier.java | 2 +- .../hive/service/cli/thrift/TI16Column.java | 2 +- .../hive/service/cli/thrift/TI16Value.java | 2 +- .../hive/service/cli/thrift/TI32Column.java | 2 +- .../hive/service/cli/thrift/TI32Value.java | 2 +- .../hive/service/cli/thrift/TI64Column.java | 2 +- .../hive/service/cli/thrift/TI64Value.java | 2 +- .../hive/service/cli/thrift/TMapTypeEntry.java | 2 +- .../service/cli/thrift/TOpenSessionReq.java | 2 +- .../service/cli/thrift/TOpenSessionResp.java | 2 +- .../service/cli/thrift/TOperationHandle.java | 2 +- .../service/cli/thrift/TPrimitiveTypeEntry.java | 2 +- .../cli/thrift/TRenewDelegationTokenReq.java | 2 +- .../cli/thrift/TRenewDelegationTokenResp.java | 2 +- .../apache/hive/service/cli/thrift/TRow.java | 2 +- .../apache/hive/service/cli/thrift/TRowSet.java | 2 +- .../hive/service/cli/thrift/TSessionHandle.java | 2 +- .../apache/hive/service/cli/thrift/TStatus.java | 2 +- .../hive/service/cli/thrift/TStringColumn.java | 2 +- .../hive/service/cli/thrift/TStringValue.java | 2 +- .../service/cli/thrift/TStructTypeEntry.java | 2 +- .../hive/service/cli/thrift/TTableSchema.java | 2 +- .../hive/service/cli/thrift/TTypeDesc.java | 2 +- .../service/cli/thrift/TTypeQualifiers.java | 2 +- .../service/cli/thrift/TUnionTypeEntry.java | 2 +- .../cli/thrift/TUserDefinedTypeEntry.java | 2 +- .../gen-py/hive_service/ThriftHive-remote | 21 +- 217 files changed, 5877 insertions(+), 3215 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hive/blob/cb648903/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestHiveMetaStore.java ---------------------------------------------------------------------- diff --git a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestHiveMetaStore.java b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestHiveMetaStore.java index 06c6b76..5a344bb 100644 --- a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestHiveMetaStore.java +++ b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestHiveMetaStore.java @@ -51,6 +51,7 @@ import org.apache.hadoop.hive.metastore.api.DoubleColumnStatsData; import org.apache.hadoop.hive.metastore.api.FieldSchema; import org.apache.hadoop.hive.metastore.api.Function; import org.apache.hadoop.hive.metastore.api.FunctionType; +import org.apache.hadoop.hive.metastore.api.GetAllFunctionsResponse; import org.apache.hadoop.hive.metastore.api.InvalidObjectException; import org.apache.hadoop.hive.metastore.api.InvalidOperationException; import org.apache.hadoop.hive.metastore.api.MetaException; @@ -2533,6 +2534,7 @@ public abstract class TestHiveMetaStore extends TestCase { String funcName = "test_func"; String className = "org.apache.hadoop.hive.ql.udf.generic.GenericUDFUpper"; String owner = "test_owner"; + final int N_FUNCTIONS = 5; PrincipalType ownerType = PrincipalType.USER; int createTime = (int) (System.currentTimeMillis() / 1000); FunctionType funcType = FunctionType.JAVA; @@ -2542,14 +2544,16 @@ public abstract class TestHiveMetaStore extends TestCase { createDb(dbName); - createFunction(dbName, funcName, className, owner, ownerType, createTime, funcType, null); + for (int i = 0; i < N_FUNCTIONS; i++) { + createFunction(dbName, funcName + "_" + i, className, owner, ownerType, createTime, funcType, null); + } // Try the different getters // getFunction() - Function func = client.getFunction(dbName, funcName); + Function func = client.getFunction(dbName, funcName + "_0"); assertEquals("function db name", dbName, func.getDbName()); - assertEquals("function name", funcName, func.getFunctionName()); + assertEquals("function name", funcName + "_0", func.getFunctionName()); assertEquals("function class name", className, func.getClassName()); assertEquals("function owner name", owner, func.getOwnerName()); assertEquals("function owner type", PrincipalType.USER, func.getOwnerType()); @@ -2566,21 +2570,31 @@ public abstract class TestHiveMetaStore extends TestCase { } assertEquals(true, gotException); + // getAllFunctions() + GetAllFunctionsResponse response = client.getAllFunctions(); + List<Function> allFunctions = response.getFunctions(); + assertEquals(N_FUNCTIONS, allFunctions.size()); + assertEquals(funcName + "_3", allFunctions.get(3).getFunctionName()); + // getFunctions() - List<String> funcs = client.getFunctions(dbName, "*_func"); - assertEquals(1, funcs.size()); - assertEquals(funcName, funcs.get(0)); + List<String> funcs = client.getFunctions(dbName, "*_func_*"); + assertEquals(N_FUNCTIONS, funcs.size()); + assertEquals(funcName + "_0", funcs.get(0)); funcs = client.getFunctions(dbName, "nonexistent_func"); assertEquals(0, funcs.size()); // dropFunction() - client.dropFunction(dbName, funcName); + for (int i = 0; i < N_FUNCTIONS; i++) { + client.dropFunction(dbName, funcName + "_" + i); + } // Confirm that the function is now gone funcs = client.getFunctions(dbName, funcName); assertEquals(0, funcs.size()); - + response = client.getAllFunctions(); + allFunctions = response.getFunctions(); + assertEquals(0, allFunctions.size()); } catch (Exception e) { System.err.println(StringUtils.stringifyException(e)); System.err.println("testConcurrentMetastores() failed."); http://git-wip-us.apache.org/repos/asf/hive/blob/cb648903/metastore/if/hive_metastore.thrift ---------------------------------------------------------------------- diff --git a/metastore/if/hive_metastore.thrift b/metastore/if/hive_metastore.thrift index adf4830..d651195 100755 --- a/metastore/if/hive_metastore.thrift +++ b/metastore/if/hive_metastore.thrift @@ -715,6 +715,9 @@ struct FireEventResponse { } +struct GetAllFunctionsResponse { + 1: optional list<Function> functions +} exception MetaException { 1: string message @@ -1114,6 +1117,8 @@ service ThriftHiveMetastore extends fb303.FacebookService Function get_function(1:string dbName, 2:string funcName) throws (1:MetaException o1, 2:NoSuchObjectException o2) + GetAllFunctionsResponse get_all_functions() throws (1:MetaException o1) + //authorization privileges bool create_role(1:Role role) throws(1:MetaException o1)
