HIVE-19416 : merge master into branch (Sergey Shelukhin) 0703
Project: http://git-wip-us.apache.org/repos/asf/hive/repo Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/4db8b1c6 Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/4db8b1c6 Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/4db8b1c6 Branch: refs/heads/master-txnstats Commit: 4db8b1c66c459d6c0487565c3c176055a910da2a Parents: d7bbc20 2ca70b9 Author: sergey <ser...@apache.org> Authored: Tue Jul 3 10:47:09 2018 -0700 Committer: sergey <ser...@apache.org> Committed: Tue Jul 3 10:47:09 2018 -0700 ---------------------------------------------------------------------- .../hive/accumulo/serde/AccumuloSerDe.java | 6 +- .../java/org/apache/hive/beeline/BeeLine.java | 2 +- .../org/apache/hive/beeline/BeeLineOpts.java | 2 +- .../org/apache/hive/beeline/HiveSchemaTool.java | 1563 ------ .../hive/beeline/schematool/HiveSchemaTool.java | 415 ++ .../schematool/HiveSchemaToolCommandLine.java | 286 ++ .../beeline/schematool/HiveSchemaToolTask.java | 32 + .../HiveSchemaToolTaskAlterCatalog.java | 90 + .../HiveSchemaToolTaskCreateCatalog.java | 132 + .../schematool/HiveSchemaToolTaskInfo.java | 43 + .../schematool/HiveSchemaToolTaskInit.java | 73 + .../HiveSchemaToolTaskMoveDatabase.java | 96 + .../schematool/HiveSchemaToolTaskMoveTable.java | 142 + .../schematool/HiveSchemaToolTaskUpgrade.java | 116 + .../schematool/HiveSchemaToolTaskValidate.java | 631 +++ .../apache/hive/beeline/TestHiveSchemaTool.java | 92 - .../beeline/schematool/TestHiveSchemaTool.java | 92 + bin/ext/schemaTool.sh | 4 +- .../org/apache/hadoop/hive/conf/HiveConf.java | 11 +- .../listener/DbNotificationListener.java | 209 +- .../listener/DummyRawStoreFailEvent.java | 17 +- .../listener/TestDbNotificationListener.java | 5 + .../hadoop/hive/ql/parse/TestExportImport.java | 55 + .../hive/ql/parse/TestReplicationScenarios.java | 72 - .../TestReplicationScenariosAcidTables.java | 602 ++- ...TestReplicationScenariosAcrossInstances.java | 92 +- .../hadoop/hive/ql/parse/WarehouseInstance.java | 5 + .../TestHiveAuthorizerCheckInvocation.java | 94 +- .../org/apache/hive/beeline/TestSchemaTool.java | 801 --- .../hive/beeline/TestSchemaToolCatalogOps.java | 417 -- .../hive/beeline/schematool/TestSchemaTool.java | 826 ++++ .../schematool/TestSchemaToolCatalogOps.java | 478 ++ .../hive/jdbc/TestTriggersWorkloadManager.java | 2 + .../test/resources/testconfiguration.properties | 6 +- .../apache/hadoop/hive/ql/QOutProcessor.java | 2 + .../org/apache/hadoop/hive/ql/QTestUtil.java | 3 +- .../org/apache/hive/jdbc/HiveStatement.java | 4 +- .../hive/llap/io/api/impl/LlapRecordReader.java | 64 + pom.xml | 2 +- .../metastore/SynchronizedMetaStoreClient.java | 5 + .../java/org/apache/hadoop/hive/ql/Driver.java | 10 +- .../apache/hadoop/hive/ql/DriverContext.java | 29 +- .../org/apache/hadoop/hive/ql/ErrorMsg.java | 4 +- .../apache/hadoop/hive/ql/exec/ExportTask.java | 2 +- .../hadoop/hive/ql/exec/FileSinkOperator.java | 19 +- .../apache/hadoop/hive/ql/exec/MoveTask.java | 6 +- .../apache/hadoop/hive/ql/exec/Operator.java | 11 +- .../hadoop/hive/ql/exec/ReplCopyTask.java | 5 +- .../apache/hadoop/hive/ql/exec/ReplTxnTask.java | 31 +- .../hive/ql/exec/SerializationUtilities.java | 22 - .../hadoop/hive/ql/exec/TableScanOperator.java | 1 + .../hadoop/hive/ql/exec/repl/ReplDumpTask.java | 4 +- .../IncrementalLoadTasksBuilder.java | 73 +- .../hadoop/hive/ql/exec/spark/SparkTask.java | 15 +- .../ql/exec/spark/session/SparkSessionImpl.java | 10 +- .../exec/spark/status/LocalSparkJobMonitor.java | 4 +- .../spark/status/RemoteSparkJobMonitor.java | 5 +- .../ql/exec/spark/status/RenderStrategy.java | 246 + .../ql/exec/spark/status/SparkJobMonitor.java | 157 +- .../org/apache/hadoop/hive/ql/io/AcidUtils.java | 37 +- .../hadoop/hive/ql/io/HiveInputFormat.java | 24 +- .../hadoop/hive/ql/lockmgr/DbTxnManager.java | 18 +- .../hadoop/hive/ql/lockmgr/DummyTxnManager.java | 7 +- .../hadoop/hive/ql/lockmgr/HiveTxnManager.java | 10 +- .../apache/hadoop/hive/ql/metadata/Hive.java | 166 +- .../hadoop/hive/ql/metadata/HiveUtils.java | 11 +- .../apache/hadoop/hive/ql/metadata/Table.java | 7 +- .../hive/ql/optimizer/GenMapRedUtils.java | 4 +- .../hive/ql/parse/BaseSemanticAnalyzer.java | 15 +- .../hive/ql/parse/DDLSemanticAnalyzer.java | 28 +- .../hive/ql/parse/ExplainSemanticAnalyzer.java | 5 +- .../hadoop/hive/ql/parse/GenTezUtils.java | 2 +- .../hive/ql/parse/ImportSemanticAnalyzer.java | 83 +- .../hive/ql/parse/LoadSemanticAnalyzer.java | 28 +- .../hadoop/hive/ql/parse/SemanticAnalyzer.java | 128 +- .../ql/parse/UpdateDeleteSemanticAnalyzer.java | 32 +- .../hadoop/hive/ql/parse/repl/CopyUtils.java | 2 +- .../hadoop/hive/ql/parse/repl/dump/Utils.java | 8 +- .../repl/dump/events/CommitTxnHandler.java | 125 +- .../ql/parse/repl/dump/events/EventHandler.java | 23 +- .../parse/repl/dump/events/InsertHandler.java | 4 + .../parse/repl/load/UpdatedMetaDataTracker.java | 124 +- .../repl/load/message/AbortTxnHandler.java | 7 +- .../repl/load/message/AllocWriteIdHandler.java | 2 +- .../repl/load/message/CommitTxnHandler.java | 78 +- .../parse/repl/load/message/MessageHandler.java | 8 +- .../parse/repl/load/message/OpenTxnHandler.java | 7 +- .../apache/hadoop/hive/ql/plan/ExplainWork.java | 18 +- .../apache/hadoop/hive/ql/plan/MoveWork.java | 12 +- .../apache/hadoop/hive/ql/plan/ReplTxnWork.java | 15 + .../hadoop/hive/ql/reexec/ReExecDriver.java | 2 +- .../hive/ql/stats/StatsCollectionContext.java | 12 +- .../hive/ql/stats/fs/FSStatsPublisher.java | 15 +- .../apache/hadoop/hive/ql/TestTxnNoBuckets.java | 41 + .../exec/spark/TestSparkInvalidFileFormat.java | 81 + .../hive/ql/exec/spark/TestSparkTask.java | 1 + .../exec/spark/status/TestSparkJobMonitor.java | 29 +- .../ql/parse/TestMacroSemanticAnalyzer.java | 9 +- .../hadoop/hive/ql/parse/TestQBCompact.java | 15 +- .../authorization/AuthorizationTestUtil.java | 8 +- .../spark_submit_negative_executor_cores.q | 5 + .../spark_submit_negative_executor_memory.q | 5 + .../queries/clientpositive/autoColumnStats_9.q | 3 + .../clientpositive/orc_schema_evolution_float.q | 2 - .../queries/clientpositive/repl_2_exim_basic.q | 1 + .../schema_evol_orc_nonvec_part_all_primitive.q | 2 - ...evol_orc_nonvec_part_all_primitive_llap_io.q | 2 - .../schema_evol_orc_vec_part_all_primitive.q | 2 - ...ma_evol_orc_vec_part_all_primitive_llap_io.q | 2 - .../clientpositive/type_change_test_int.q | 3 - .../type_change_test_int_vectorized.q | 2 - .../test/queries/clientpositive/unicode_data.q | 15 + .../queries/clientpositive/union_fast_stats.q | 6 + .../queries/clientpositive/union_rowcounts.q | 51 + .../test/queries/clientpositive/union_stats.q | 44 +- .../vector_llap_io_data_conversion.q | 19 + .../exim_03_nonpart_noncompat_colschema.q.out | 2 +- .../exim_04_nonpart_noncompat_colnumber.q.out | 2 +- .../exim_05_nonpart_noncompat_coltype.q.out | 2 +- .../exim_06_nonpart_noncompat_storage.q.out | 2 +- .../exim_07_nonpart_noncompat_ifof.q.out | 2 +- .../exim_08_nonpart_noncompat_serde.q.out | 2 +- .../exim_09_nonpart_noncompat_serdeparam.q.out | 2 +- .../exim_10_nonpart_noncompat_bucketing.q.out | 2 +- .../exim_11_nonpart_noncompat_sorting.q.out | 2 +- .../exim_12_nonnative_export.q.out | 2 +- .../exim_13_nonnative_import.q.out | 2 +- .../clientnegative/exim_14_nonpart_part.q.out | 2 +- .../clientnegative/exim_15_part_nonpart.q.out | 2 +- .../exim_16_part_noncompat_schema.q.out | 2 +- .../exim_19_external_over_existing.q.out | 2 +- .../exim_21_part_managed_external.q.out | 2 +- .../spark_submit_negative_executor_cores.q.out | 5 + .../spark_submit_negative_executor_memory.q.out | 5 + .../clientpositive/autoColumnStats_9.q.out | 13 +- .../clientpositive/llap/multiMapJoin1.q.out | 150 +- .../clientpositive/llap/orc_merge11.q.out | 54 +- ...schema_evol_orc_vec_part_all_primitive.q.out | 19 +- ...vol_orc_vec_part_all_primitive_llap_io.q.out | 19 +- .../clientpositive/llap/union_fast_stats.q.out | 58 +- .../clientpositive/llap/union_rowcounts.q.out | 180 + .../clientpositive/llap/union_stats.q.out | 145 +- .../llap/vector_llap_io_data_conversion.q.out | 187 + .../results/clientpositive/orc_file_dump.q.out | 24 +- .../results/clientpositive/orc_merge11.q.out | 54 +- .../results/clientpositive/orc_merge12.q.out | 84 +- .../clientpositive/tez/orc_merge12.q.out | 84 +- .../results/clientpositive/unicode_data.q.out | 123 + .../results/clientpositive/union_stats.q.out | 243 + .../apache/hadoop/hive/serde2/Deserializer.java | 2 + .../org/apache/hive/service/ServiceUtils.java | 5 +- .../cli/SparkProgressMonitorStatusMapper.java | 52 + .../service/cli/thrift/ThriftCLIService.java | 5 +- .../apache/hadoop/hive/shims/Hadoop23Shims.java | 3 + .../hive/spark/client/AbstractSparkClient.java | 25 +- .../spark/client/SparkSubmitSparkClient.java | 22 +- .../apache/hive/spark/client/rpc/RpcServer.java | 21 +- .../gen/thrift/gen-cpp/ThriftHiveMetastore.cpp | 2649 +++++----- .../gen/thrift/gen-cpp/ThriftHiveMetastore.h | 126 + .../ThriftHiveMetastore_server.skeleton.cpp | 5 + .../gen/thrift/gen-cpp/hive_metastore_types.cpp | 4003 ++++++++------- .../gen/thrift/gen-cpp/hive_metastore_types.h | 218 +- .../metastore/api/AddDynamicPartitions.java | 32 +- .../api/AllocateTableWriteIdsRequest.java | 68 +- .../api/AllocateTableWriteIdsResponse.java | 36 +- .../metastore/api/AlterPartitionsRequest.java | 36 +- .../metastore/api/ClearFileMetadataRequest.java | 32 +- .../hive/metastore/api/ClientCapabilities.java | 32 +- .../hive/metastore/api/CommitTxnRequest.java | 168 +- .../hive/metastore/api/CompactionRequest.java | 44 +- .../hive/metastore/api/CreationMetadata.java | 32 +- .../metastore/api/FindSchemasByColsResp.java | 36 +- .../hive/metastore/api/FireEventRequest.java | 32 +- .../metastore/api/GetAllFunctionsResponse.java | 36 +- .../api/GetFileMetadataByExprRequest.java | 32 +- .../api/GetFileMetadataByExprResult.java | 48 +- .../metastore/api/GetFileMetadataRequest.java | 32 +- .../metastore/api/GetFileMetadataResult.java | 44 +- .../hive/metastore/api/GetTablesRequest.java | 32 +- .../hive/metastore/api/GetTablesResult.java | 36 +- .../metastore/api/GetValidWriteIdsRequest.java | 32 +- .../metastore/api/GetValidWriteIdsResponse.java | 36 +- .../api/HeartbeatTxnRangeResponse.java | 64 +- .../metastore/api/InsertEventRequestData.java | 227 +- .../hadoop/hive/metastore/api/LockRequest.java | 36 +- .../hive/metastore/api/Materialization.java | 32 +- .../api/NotificationEventResponse.java | 36 +- .../metastore/api/PutFileMetadataRequest.java | 64 +- .../api/ReplTblWriteIdStateRequest.java | 32 +- .../hive/metastore/api/SchemaVersion.java | 36 +- .../hive/metastore/api/ShowCompactResponse.java | 36 +- .../hive/metastore/api/ShowLocksResponse.java | 36 +- .../hive/metastore/api/TableValidWriteIds.java | 32 +- .../hive/metastore/api/ThriftHiveMetastore.java | 4564 +++++++++++------- .../hive/metastore/api/WMFullResourcePlan.java | 144 +- .../api/WMGetAllResourcePlanResponse.java | 36 +- .../WMGetTriggersForResourePlanResponse.java | 36 +- .../api/WMValidateResourcePlanResponse.java | 64 +- .../hive/metastore/api/WriteEventInfo.java | 1012 ++++ .../api/WriteNotificationLogRequest.java | 949 ++++ .../api/WriteNotificationLogResponse.java | 283 ++ .../gen-php/metastore/ThriftHiveMetastore.php | 1610 +++--- .../src/gen/thrift/gen-php/metastore/Types.php | 1628 +++++-- .../hive_metastore/ThriftHiveMetastore-remote | 7 + .../hive_metastore/ThriftHiveMetastore.py | 1125 +++-- .../gen/thrift/gen-py/hive_metastore/ttypes.py | 947 ++-- .../gen/thrift/gen-rb/hive_metastore_types.rb | 86 +- .../gen/thrift/gen-rb/thrift_hive_metastore.rb | 54 + .../hadoop/hive/metastore/HiveMetaStore.java | 86 + .../hive/metastore/HiveMetaStoreClient.java | 10 +- .../hadoop/hive/metastore/IMetaStoreClient.java | 16 +- .../hive/metastore/MetaStoreEventListener.java | 12 + .../metastore/MetaStoreListenerNotifier.java | 6 + .../hadoop/hive/metastore/ObjectStore.java | 65 +- .../apache/hadoop/hive/metastore/RawStore.java | 16 +- .../hive/metastore/ReplChangeManager.java | 10 +- .../hive/metastore/cache/CachedStore.java | 13 +- .../hive/metastore/events/AcidWriteEvent.java | 91 + .../metastore/messaging/AcidWriteMessage.java | 50 + .../metastore/messaging/CommitTxnMessage.java | 23 + .../hive/metastore/messaging/EventMessage.java | 3 +- .../messaging/MessageDeserializer.java | 9 + .../metastore/messaging/MessageFactory.java | 12 + .../messaging/json/JSONAcidWriteMessage.java | 150 + .../messaging/json/JSONCommitTxnMessage.java | 95 + .../messaging/json/JSONMessageDeserializer.java | 9 + .../messaging/json/JSONMessageFactory.java | 8 + .../model/MTxnWriteNotificationLog.java | 123 + .../hive/metastore/tools/SQLGenerator.java | 9 + .../hadoop/hive/metastore/txn/TxnDbUtil.java | 28 + .../hadoop/hive/metastore/txn/TxnHandler.java | 187 +- .../hadoop/hive/metastore/txn/TxnStore.java | 11 + .../hadoop/hive/metastore/utils/FileUtils.java | 12 +- .../src/main/resources/package.jdo | 35 + .../main/sql/derby/hive-schema-3.1.0.derby.sql | 15 + .../main/sql/derby/hive-schema-4.0.0.derby.sql | 15 + .../sql/derby/upgrade-2.3.0-to-3.0.0.derby.sql | 1 - .../sql/derby/upgrade-3.0.0-to-3.1.0.derby.sql | 16 + .../main/sql/mssql/hive-schema-3.1.0.mssql.sql | 17 + .../main/sql/mssql/hive-schema-4.0.0.mssql.sql | 17 + .../sql/mssql/upgrade-3.0.0-to-3.1.0.mssql.sql | 16 + .../main/sql/mysql/hive-schema-3.0.0.mysql.sql | 1 - .../main/sql/mysql/hive-schema-3.1.0.mysql.sql | 16 + .../main/sql/mysql/hive-schema-4.0.0.mysql.sql | 16 + .../sql/mysql/upgrade-2.3.0-to-3.0.0.mysql.sql | 4 +- .../sql/mysql/upgrade-3.0.0-to-3.1.0.mysql.sql | 16 + .../sql/oracle/hive-schema-3.0.0.oracle.sql | 1 - .../sql/oracle/hive-schema-3.1.0.oracle.sql | 15 + .../sql/oracle/hive-schema-4.0.0.oracle.sql | 15 + .../oracle/upgrade-2.3.0-to-3.0.0.oracle.sql | 3 +- .../oracle/upgrade-3.0.0-to-3.1.0.oracle.sql | 16 + .../sql/postgres/hive-schema-3.0.0.postgres.sql | 2 - .../sql/postgres/hive-schema-3.1.0.postgres.sql | 15 + .../sql/postgres/hive-schema-4.0.0.postgres.sql | 15 + .../upgrade-3.0.0-to-3.1.0.postgres.sql | 16 + .../src/main/thrift/hive_metastore.thrift | 30 +- .../DummyRawStoreControlledCommit.java | 13 +- .../DummyRawStoreForJdoConnection.java | 12 +- .../HiveMetaStoreClientPreCatalog.java | 10 +- .../hadoop/hive/metastore/TestObjectStore.java | 8 +- .../hive/streaming/AbstractRecordWriter.java | 2 +- testutils/ptest2/pom.xml | 2 +- 262 files changed, 22305 insertions(+), 11434 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hive/blob/4db8b1c6/itests/hcatalog-unit/src/test/java/org/apache/hive/hcatalog/listener/DummyRawStoreFailEvent.java ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hive/blob/4db8b1c6/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/parse/TestReplicationScenarios.java ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hive/blob/4db8b1c6/pom.xml ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hive/blob/4db8b1c6/ql/src/java/org/apache/hadoop/hive/ql/io/AcidUtils.java ---------------------------------------------------------------------- diff --cc ql/src/java/org/apache/hadoop/hive/ql/io/AcidUtils.java index f126550,16ba82e..19dbc64 --- a/ql/src/java/org/apache/hadoop/hive/ql/io/AcidUtils.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/io/AcidUtils.java @@@ -2032,9 -1892,31 +2017,31 @@@ public class AcidUtils return null; } + //Get the first level acid directory (if any) from a given path + public static String getFirstLevelAcidDirPath(Path dataPath, FileSystem fileSystem) throws IOException { + if (dataPath == null) { + return null; + } + String firstLevelAcidDir = getAcidSubDir(dataPath); + if (firstLevelAcidDir != null) { + return firstLevelAcidDir; + } + + String acidDirPath = getFirstLevelAcidDirPath(dataPath.getParent(), fileSystem); + if (acidDirPath == null) { + return null; + } + + // We need the path for directory so no need to append file name + if (fileSystem.isDirectory(dataPath)) { + return acidDirPath + Path.SEPARATOR + dataPath.getName(); + } + return acidDirPath; + } + public static boolean isAcidEnabled(HiveConf hiveConf) { - String txnMgr = hiveConf.getVar(HiveConf.ConfVars.HIVE_TXN_MANAGER); - boolean concurrency = hiveConf.getBoolVar(HiveConf.ConfVars.HIVE_SUPPORT_CONCURRENCY); + String txnMgr = hiveConf.getVar(ConfVars.HIVE_TXN_MANAGER); + boolean concurrency = hiveConf.getBoolVar(ConfVars.HIVE_SUPPORT_CONCURRENCY); String dbTxnMgr = "org.apache.hadoop.hive.ql.lockmgr.DbTxnManager"; if (txnMgr.equals(dbTxnMgr) && concurrency) { return true; http://git-wip-us.apache.org/repos/asf/hive/blob/4db8b1c6/ql/src/java/org/apache/hadoop/hive/ql/lockmgr/DbTxnManager.java ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hive/blob/4db8b1c6/ql/src/java/org/apache/hadoop/hive/ql/lockmgr/DummyTxnManager.java ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hive/blob/4db8b1c6/ql/src/java/org/apache/hadoop/hive/ql/lockmgr/HiveTxnManager.java ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hive/blob/4db8b1c6/ql/src/java/org/apache/hadoop/hive/ql/metadata/Hive.java ----------------------------------------------------------------------
