Merge branch 'master' into beeline-cli
Project: http://git-wip-us.apache.org/repos/asf/hive/repo Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/753b2b30 Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/753b2b30 Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/753b2b30 Branch: refs/heads/beeline-cli Commit: 753b2b308afeb3808a3c10de5474f863d975a46f Parents: 2ddd86d 3e713bc Author: Xuefu Zhang <xzh...@cloudera.com> Authored: Thu May 14 21:31:01 2015 -0700 Committer: Xuefu Zhang <xzh...@cloudera.com> Committed: Thu May 14 21:31:01 2015 -0700 ---------------------------------------------------------------------- .../src/test/templates/TestAccumuloCliDriver.vm | 19 +- .../apache/hadoop/hive/ant/QTestGenTask.java | 25 + .../java/org/apache/hive/beeline/BeeLine.java | 10 +- bin/beeline.cmd | 11 +- bin/ext/hiveserver2.cmd | 2 +- bin/ext/orcfiledump.sh | 9 +- bin/hive | 2 +- .../org/apache/hadoop/hive/conf/HiveConf.java | 27 +- data/files/tjoin1.txt | 3 + data/files/tjoin2.txt | 4 + dev-support/jenkins-execute-build.sh | 2 +- dev-support/jenkins-execute-hms-test.sh | 4 +- .../src/test/templates/TestHBaseCliDriver.vm | 18 +- .../templates/TestHBaseNegativeCliDriver.vm | 19 +- .../apache/hive/hcatalog/common/HCatUtil.java | 37 +- .../hive/hcatalog/common/HiveClientCache.java | 94 +- .../DefaultOutputCommitterContainer.java | 6 +- .../mapreduce/FileOutputCommitterContainer.java | 22 +- .../mapreduce/FileOutputFormatContainer.java | 8 +- .../hcatalog/mapreduce/HCatBaseInputFormat.java | 50 +- .../hcatalog/mapreduce/HCatOutputFormat.java | 6 +- .../hcatalog/mapreduce/HCatRecordReader.java | 4 +- .../hive/hcatalog/mapreduce/HCatSplit.java | 21 +- .../hive/hcatalog/mapreduce/HCatTableInfo.java | 12 + .../hcatalog/mapreduce/InitializeInput.java | 6 +- .../hive/hcatalog/mapreduce/InputJobInfo.java | 5 + .../hive/hcatalog/mapreduce/PartInfo.java | 117 +- .../hive/hcatalog/mapreduce/Security.java | 10 +- .../mapreduce/TaskCommitContextRegistry.java | 6 +- .../hcatalog/common/TestHiveClientCache.java | 37 +- .../hcatalog/mapreduce/HCatMapReduceTest.java | 3 +- .../mapreduce/TestHCatOutputFormat.java | 5 +- .../hcatalog/mapreduce/TestHCatPartitioned.java | 32 +- .../hcatalog/mapreduce/TestPassProperties.java | 5 +- .../apache/hive/hcatalog/pig/PigHCatUtil.java | 10 +- .../deployers/config/webhcat/webhcat-site.xml | 9 +- .../templeton/deployers/deploy_e2e_artifacts.sh | 3 + .../src/test/e2e/templeton/deployers/env.sh | 1 + .../streaming/AbstractRecordWriter.java | 11 +- .../hive/hcatalog/streaming/HiveEndPoint.java | 9 +- .../hive/hcatalog/api/HCatClientHMSImpl.java | 17 +- .../hcatalog/api/repl/TestReplicationTask.java | 14 +- hcatalog/webhcat/svr/pom.xml | 13 +- .../svr/src/main/config/webhcat-default.xml | 10 +- .../hcatalog/templeton/CompleteDelegator.java | 6 +- .../hcatalog/templeton/SecureProxySupport.java | 9 +- .../hcatalog/templeton/tool/LaunchMapper.java | 2 +- .../templeton/tool/TempletonControllerJob.java | 15 +- .../hive/metastore/TestHiveMetaStore.java | 30 + .../hive/beeline/TestBeeLineWithArgs.java | 12 + .../test/resources/testconfiguration.properties | 9 +- .../org/apache/hadoop/hive/ql/QTestUtil.java | 46 + .../org/apache/hive/jdbc/HiveConnection.java | 3 +- metastore/dbs/derby/execute.sh | 37 + metastore/dbs/derby/prepare.sh | 63 + metastore/dbs/postgres/execute.sh | 29 + metastore/dbs/postgres/prepare.sh | 72 + .../upgrade/mssql/006-HIVE-9456.mssql.sql | 323 + .../upgrade/mssql/hive-schema-1.2.0.mssql.sql | 256 +- .../upgrade/mssql/hive-schema-1.3.0.mssql.sql | 256 +- .../mssql/upgrade-1.1.0-to-1.2.0.mssql.sql | 1 + .../upgrade/mysql/021-HIVE-7018.mysql.sql | 53 - .../upgrade/mysql/hive-schema-1.2.0.mysql.sql | 10 +- .../upgrade/mysql/hive-schema-1.3.0.mysql.sql | 10 +- .../mysql/upgrade-1.1.0-to-1.2.0.mysql.sql | 2 +- .../hive/metastore/AggregateStatsCache.java | 33 +- .../hadoop/hive/metastore/HiveMetaStore.java | 27 +- .../hive/metastore/HiveMetaStoreClient.java | 6 +- .../hadoop/hive/metastore/IMetaStoreClient.java | 3 + .../hive/metastore/MetaStoreDirectSql.java | 24 +- .../hive/metastore/RetryingMetaStoreClient.java | 89 +- .../hadoop/hive/metastore/txn/TxnHandler.java | 35 +- .../hive/metastore/TestHiveMetastoreCli.java | 63 + .../hive/metastore/txn/TestTxnHandler.java | 39 +- packaging/src/main/assembly/bin.xml | 1 + pom.xml | 5 + .../java/org/apache/hadoop/hive/ql/Context.java | 10 +- .../hive/ql/exec/CommonMergeJoinOperator.java | 54 +- .../hadoop/hive/ql/exec/FunctionRegistry.java | 7 +- .../hadoop/hive/ql/exec/HashTableLoader.java | 4 +- .../hadoop/hive/ql/exec/MapJoinOperator.java | 227 +- .../apache/hadoop/hive/ql/exec/ObjectCache.java | 7 + .../apache/hadoop/hive/ql/exec/Operator.java | 4 + .../hadoop/hive/ql/exec/ReduceSinkOperator.java | 3 +- .../apache/hadoop/hive/ql/exec/Registry.java | 29 +- .../apache/hadoop/hive/ql/exec/Utilities.java | 18 + .../hadoop/hive/ql/exec/mr/ExecDriver.java | 1 + .../hadoop/hive/ql/exec/mr/HashTableLoader.java | 2 +- .../hadoop/hive/ql/exec/mr/ObjectCache.java | 5 + .../persistence/BytesBytesMultiHashMap.java | 1 + .../exec/persistence/HybridHashTableConf.java | 86 + .../persistence/HybridHashTableContainer.java | 248 +- .../ql/exec/persistence/KeyValueContainer.java | 31 +- .../ql/exec/persistence/ObjectContainer.java | 31 +- .../hive/ql/exec/spark/HashTableLoader.java | 2 +- .../hadoop/hive/ql/exec/tez/DagUtils.java | 1 + .../hive/ql/exec/tez/HashTableLoader.java | 65 +- .../hadoop/hive/ql/exec/tez/ObjectCache.java | 6 + .../hive/ql/exec/tez/TezSessionState.java | 13 +- .../ql/exec/vector/VectorColumnSetInfo.java | 3 +- .../exec/vector/VectorMapJoinBaseOperator.java | 185 + .../ql/exec/vector/VectorMapJoinOperator.java | 127 +- .../VectorMapJoinOuterFilteredOperator.java | 120 + .../mapjoin/VectorMapJoinCommonOperator.java | 16 +- .../VectorMapJoinGenerateResultOperator.java | 34 +- .../VectorMapJoinInnerBigOnlyLongOperator.java | 5 + ...ctorMapJoinInnerBigOnlyMultiKeyOperator.java | 5 + ...VectorMapJoinInnerBigOnlyStringOperator.java | 5 + ...ectorMapJoinInnerGenerateResultOperator.java | 15 + .../mapjoin/VectorMapJoinInnerLongOperator.java | 4 + .../VectorMapJoinInnerMultiKeyOperator.java | 4 + .../VectorMapJoinInnerStringOperator.java | 4 + .../VectorMapJoinLeftSemiLongOperator.java | 5 + .../VectorMapJoinLeftSemiMultiKeyOperator.java | 5 + .../VectorMapJoinLeftSemiStringOperator.java | 5 + ...ectorMapJoinOuterGenerateResultOperator.java | 31 +- .../mapjoin/VectorMapJoinOuterLongOperator.java | 4 + .../VectorMapJoinOuterMultiKeyOperator.java | 4 + .../VectorMapJoinOuterStringOperator.java | 4 + .../mapjoin/VectorMapJoinRowBytesContainer.java | 2 +- .../fast/VectorMapJoinFastBytesHashMap.java | 4 +- .../VectorMapJoinFastBytesHashMultiSet.java | 4 +- .../fast/VectorMapJoinFastBytesHashSet.java | 4 +- .../fast/VectorMapJoinFastBytesHashTable.java | 4 +- .../mapjoin/fast/VectorMapJoinFastHashMap.java | 4 +- .../fast/VectorMapJoinFastHashMultiSet.java | 4 +- .../mapjoin/fast/VectorMapJoinFastHashSet.java | 4 +- .../fast/VectorMapJoinFastHashTable.java | 4 +- .../fast/VectorMapJoinFastHashTableLoader.java | 4 +- .../fast/VectorMapJoinFastLongHashMap.java | 4 +- .../fast/VectorMapJoinFastLongHashMultiSet.java | 4 +- .../fast/VectorMapJoinFastLongHashSet.java | 4 +- .../fast/VectorMapJoinFastLongHashTable.java | 4 +- .../fast/VectorMapJoinFastMultiKeyHashMap.java | 6 +- .../VectorMapJoinFastMultiKeyHashMultiSet.java | 4 +- .../fast/VectorMapJoinFastMultiKeyHashSet.java | 4 +- .../fast/VectorMapJoinFastStringHashMap.java | 4 +- .../VectorMapJoinFastStringHashMultiSet.java | 4 +- .../fast/VectorMapJoinFastStringHashSet.java | 4 +- .../fast/VectorMapJoinFastTableContainer.java | 23 +- .../hive/ql/io/orc/ColumnStatisticsImpl.java | 33 +- .../hive/ql/io/orc/DateColumnStatistics.java | 6 +- .../apache/hadoop/hive/ql/io/orc/FileDump.java | 91 +- .../hadoop/hive/ql/io/orc/JsonFileDump.java | 365 + .../hadoop/hive/ql/io/orc/RecordReaderImpl.java | 222 +- .../hive/ql/io/sarg/SearchArgumentImpl.java | 85 +- .../ql/lockmgr/zookeeper/ZooKeeperHiveLock.java | 22 + .../BucketingSortingReduceSinkOptimizer.java | 13 + .../optimizer/ConstantPropagateProcFactory.java | 83 +- .../ql/optimizer/IdentityProjectRemover.java | 12 + .../ql/optimizer/LimitPushdownOptimizer.java | 9 +- .../ql/optimizer/NonBlockingOpDeDupProc.java | 11 + .../hadoop/hive/ql/optimizer/Optimizer.java | 12 +- .../ql/optimizer/calcite/cost/HiveCost.java | 16 +- .../calcite/reloperators/HiveSortExchange.java | 37 +- .../rules/HiveInsertExchange4JoinRule.java | 6 +- .../calcite/translator/HiveOpConverter.java | 28 +- .../translator/HiveOpConverterPostProc.java | 10 + .../hive/ql/optimizer/physical/Vectorizer.java | 23 +- .../hadoop/hive/ql/parse/CalcitePlanner.java | 1 + .../apache/hadoop/hive/ql/parse/HiveParser.g | 19 +- .../org/apache/hadoop/hive/ql/parse/QB.java | 10 +- .../hadoop/hive/ql/parse/SemanticAnalyzer.java | 104 +- .../hadoop/hive/ql/parse/WindowingSpec.java | 7 +- .../apache/hadoop/hive/ql/plan/MapJoinDesc.java | 2 +- .../apache/hadoop/hive/ql/plan/PlanUtils.java | 46 +- .../MetaStoreAuthzAPIAuthorizerEmbedOnly.java | 3 +- .../hadoop/hive/ql/session/SessionState.java | 31 +- .../hadoop/hive/ql/txn/compactor/Cleaner.java | 38 +- .../hive/ql/txn/compactor/CompactorThread.java | 12 +- .../hadoop/hive/ql/txn/compactor/Initiator.java | 11 +- .../hadoop/hive/ql/txn/compactor/Worker.java | 42 +- .../org/apache/hadoop/hive/ql/udf/UDFMd5.java | 79 + .../hive/ql/udf/generic/GenericUDFQuarter.java | 85 + .../apache/hadoop/hive/ql/TestTxnCommands2.java | 18 +- .../fast/TestVectorMapJoinFastLongHashMap.java | 14 +- .../TestVectorMapJoinFastMultiKeyHashMap.java | 14 +- .../hive/ql/io/orc/TestColumnStatistics.java | 20 +- .../hadoop/hive/ql/io/orc/TestJsonFileDump.java | 138 + .../hive/ql/io/orc/TestRecordReaderImpl.java | 170 +- .../hive/ql/io/sarg/TestSearchArgumentImpl.java | 104 +- ...tedCharsInColumnNameCreateTableNegative.java | 87 + .../hadoop/hive/ql/plan/TestViewEntity.java | 108 + .../hive/ql/txn/compactor/TestCleaner.java | 56 +- .../hive/ql/txn/compactor/TestInitiator.java | 63 +- .../hive/ql/txn/compactor/TestWorker.java | 45 + .../hadoop/hive/ql/udf/TestGenericUDFDate.java | 92 - .../hive/ql/udf/TestGenericUDFDateAdd.java | 145 - .../hive/ql/udf/TestGenericUDFDateDiff.java | 116 - .../hive/ql/udf/TestGenericUDFDateSub.java | 143 - .../hadoop/hive/ql/udf/TestGenericUDFUtils.java | 58 - .../apache/hadoop/hive/ql/udf/TestUDFMd5.java | 57 + .../hive/ql/udf/generic/TestGenericUDFDate.java | 92 + .../ql/udf/generic/TestGenericUDFDateAdd.java | 143 + .../ql/udf/generic/TestGenericUDFDateDiff.java | 116 + .../ql/udf/generic/TestGenericUDFDateSub.java | 143 + .../ql/udf/generic/TestGenericUDFQuarter.java | 182 + .../ql/udf/generic/TestGenericUDFUtils.java | 57 + .../clientpositive/auto_sortmerge_join_13.q | 2 + .../test/queries/clientpositive/bucket_many.q | 16 + .../test/queries/clientpositive/explainuser_2.q | 1 + .../extrapolate_part_stats_partial.q | 2 + .../extrapolate_part_stats_partial_ndv.q | 2 + ql/src/test/queries/clientpositive/fold_case.q | 12 + ql/src/test/queries/clientpositive/fold_when.q | 31 + .../clientpositive/hybridgrace_hashjoin_1.q | 258 + .../clientpositive/hybridgrace_hashjoin_2.q | 152 + .../queries/clientpositive/hybridhashjoin.q | 250 - .../clientpositive/insert_overwrite_directory.q | 141 + .../queries/clientpositive/limit_pushdown.q | 4 + .../queries/clientpositive/mapjoin_mapjoin.q | 1 + ql/src/test/queries/clientpositive/mergejoin.q | 17 + .../test/queries/clientpositive/tez_join_hash.q | 2 + .../test/queries/clientpositive/tez_smb_main.q | 6 + ql/src/test/queries/clientpositive/udf_md5.q | 13 + .../test/queries/clientpositive/udf_quarter.q | 100 + .../clientpositive/vector_binary_join_groupby.q | 55 + .../clientpositive/vector_left_outer_join2.q | 62 + .../clientpositive/vector_leftsemi_mapjoin.q | 403 + .../clientpositive/windowing_windowspec.q | 2 + ql/src/test/resources/orc-file-dump.json | 1354 ++ .../annotate_stats_join_pkfk.q.out | 20 +- .../results/clientpositive/bucket_many.q.out | 230 + .../encryption_insert_partition_static.q.out | 14 +- .../test/results/clientpositive/fold_case.q.out | 301 + .../test/results/clientpositive/fold_when.q.out | 480 + .../insert_overwrite_directory.q.out | 1813 +++ ql/src/test/results/clientpositive/join32.q.out | 84 +- .../clientpositive/join32_lessSize.q.out | 423 +- ql/src/test/results/clientpositive/join33.q.out | 84 +- .../clientpositive/join_alt_syntax.q.out | 306 +- .../clientpositive/join_cond_pushdown_2.q.out | 150 +- .../clientpositive/join_cond_pushdown_4.q.out | 150 +- .../results/clientpositive/limit_pushdown.q.out | 88 + .../test/results/clientpositive/mergejoin.q.out | 844 +- .../ql_rewrite_gbtoidx_cbo_2.q.out | 14 +- .../results/clientpositive/show_functions.q.out | 3 + .../results/clientpositive/spark/cbo_gby.q.out | 4 +- .../clientpositive/spark/cbo_udf_udaf.q.out | 2 +- ...pby_complex_types_multi_single_reducer.q.out | 38 +- .../results/clientpositive/spark/join32.q.out | 88 +- .../clientpositive/spark/join32_lessSize.q.out | 286 +- .../results/clientpositive/spark/join33.q.out | 88 +- .../clientpositive/spark/join_alt_syntax.q.out | 210 +- .../spark/join_cond_pushdown_2.q.out | 98 +- .../spark/join_cond_pushdown_4.q.out | 98 +- .../spark/lateral_view_explode2.q.out | 4 +- .../clientpositive/spark/limit_pushdown.q.out | 94 + .../clientpositive/spark/union_remove_25.q.out | 2 +- .../clientpositive/spark/union_top_level.q.out | 16 +- .../spark/vector_cast_constant.q.java1.7.out | 16 +- .../spark/vector_cast_constant.q.java1.8.out | 16 +- .../spark/vectorized_timestamp_funcs.q.out | 4 +- .../clientpositive/tez/auto_join29.q.out | 500 + .../clientpositive/tez/explainuser_2.q.out | 1529 +- .../tez/hybridgrace_hashjoin_1.q.out | 1587 ++ .../tez/hybridgrace_hashjoin_2.q.out | 1417 ++ .../clientpositive/tez/hybridhashjoin.q.out | 1566 -- .../clientpositive/tez/limit_pushdown.q.out | 94 + .../results/clientpositive/tez/mergejoin.q.out | 844 +- .../tez/vector_binary_join_groupby.q.out | 303 + .../tez/vector_left_outer_join2.q.out | 553 + .../tez/vector_leftsemi_mapjoin.q.out | 13807 +++++++++++++++++ .../test/results/clientpositive/udf_md5.q.out | 61 + .../results/clientpositive/udf_quarter.q.out | 246 + .../vector_binary_join_groupby.q.out | 293 + .../vector_left_outer_join2.q.out | 568 + .../vector_leftsemi_mapjoin.q.out | 13572 ++++++++++++++++ .../clientpositive/windowing_windowspec.q.out | 108 + ql/src/test/templates/TestCliDriver.vm | 18 +- ql/src/test/templates/TestCompareCliDriver.vm | 21 +- ql/src/test/templates/TestNegativeCliDriver.vm | 18 +- ql/src/test/templates/TestParseNegative.vm | 17 +- .../hadoop/hive/ql/io/sarg/PredicateLeaf.java | 19 +- service/pom.xml | 5 + .../auth/LdapAuthenticationProviderImpl.java | 2 +- .../thrift/EmbeddedThriftBinaryCLIService.java | 5 + .../cli/thrift/ThriftBinaryCLIService.java | 1 - .../apache/hive/service/server/HiveServer2.java | 106 +- .../service/cli/session/TestSessionHooks.java | 3 +- testutils/metastore/execute-test-on-lxc.sh | 7 +- .../ptest2/src/main/resources/source-prep.vm | 11 + .../hive/ptest/execution/TestScripts.java | 23 + .../TestScripts.testPrepGit.approved.txt | 2 +- .../TestScripts.testPrepHadoop1.approved.txt | 111 + .../TestScripts.testPrepNone.approved.txt | 2 +- .../TestScripts.testPrepSvn.approved.txt | 2 +- 287 files changed, 47828 insertions(+), 6152 deletions(-) ----------------------------------------------------------------------
