Merge branch 'master' of https://git-wip-us.apache.org/repos/asf/hive
Project: http://git-wip-us.apache.org/repos/asf/hive/repo Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/51efcb80 Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/51efcb80 Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/51efcb80 Branch: refs/heads/llap Commit: 51efcb80e1d09e490dab644bcad2cab54c99e353 Parents: 39d66a4 8c1f055 Author: Dmitry Tolpeko <dmtolp...@gmail.com> Authored: Wed Mar 30 00:23:51 2016 -0700 Committer: Dmitry Tolpeko <dmtolp...@gmail.com> Committed: Wed Mar 30 00:23:51 2016 -0700 ---------------------------------------------------------------------- .../apache/hadoop/hive/ant/GenVectorCode.java | 531 ++++----- .../apache/hadoop/hive/common/FileUtils.java | 54 + .../hive/common/type/HiveIntervalDayTime.java | 245 ---- .../org/apache/hadoop/hive/conf/HiveConf.java | 15 +- .../org/apache/hive/common/util/DateUtils.java | 19 - .../hive/contrib/serde2/MultiDelimitSerDe.java | 23 +- data/files/alltypesorc3xcols | Bin 0 -> 1504592 bytes data/files/timestamps.txt | 50 + .../hive/hcatalog/streaming/HiveEndPoint.java | 11 + .../hcatalog/templeton/SecureProxySupport.java | 46 +- .../listener/TestDbNotificationListener.java | 18 + .../org/apache/hive/jdbc/miniHS2/MiniHS2.java | 56 +- .../hive/metastore/TestHiveMetaStore.java | 20 +- .../jdbc/TestJdbcWithLocalClusterSpark.java | 2 +- .../apache/hive/jdbc/TestJdbcWithMiniMr.java | 2 +- ...stMultiSessionsHS2WithLocalClusterSpark.java | 6 +- .../jdbc/TestServiceDiscoveryWithMiniHS2.java | 132 +++ .../jdbc/authorization/TestHS2AuthzContext.java | 4 +- .../authorization/TestJdbcMetadataApiAuth.java | 2 +- .../TestJdbcWithSQLAuthorization.java | 2 +- .../test/resources/testconfiguration.properties | 5 + .../hive/jdbc/ZooKeeperHiveClientHelper.java | 21 +- .../hadoop/hive/llap/io/api/LlapProxy.java | 2 + .../org/apache/hadoop/hive/llap/LlapUtil.java | 26 + .../apache/hadoop/hive/llap/tez/Converters.java | 1 + .../hadoop/hive/llap/tez/TestConverters.java | 190 +++ .../llap/IncrementalObjectSizeEstimator.java | 54 +- .../hadoop/hive/llap/cache/LlapDataBuffer.java | 12 +- .../hive/llap/cache/LowLevelCacheImpl.java | 35 +- .../llap/cache/LowLevelCacheMemoryManager.java | 6 +- .../llap/cache/LowLevelFifoCachePolicy.java | 4 +- .../llap/cache/LowLevelLrfuCachePolicy.java | 14 +- .../hive/llap/cache/SimpleBufferManager.java | 8 +- .../hive/llap/daemon/impl/LlapDaemon.java | 6 +- .../hive/llap/io/api/impl/LlapInputFormat.java | 32 +- .../hive/llap/io/api/impl/LlapIoImpl.java | 21 +- .../llap/io/decode/OrcColumnVectorProducer.java | 4 +- .../llap/io/encoded/OrcEncodedDataReader.java | 95 +- .../hadoop/hive/llap/old/BufferInProgress.java | 82 -- .../apache/hadoop/hive/llap/old/BufferPool.java | 225 ---- .../hadoop/hive/llap/old/CachePolicy.java | 34 - .../apache/hadoop/hive/llap/old/ChunkPool.java | 237 ---- .../resources/llap-daemon-log4j2.properties | 14 +- .../hive/metastore/MetaStoreDirectSql.java | 32 +- .../hadoop/hive/metastore/ObjectStore.java | 2 +- .../hive/metastore/StatObjectConverter.java | 2 +- .../hadoop/hive/metastore/hbase/HBaseUtils.java | 8 +- .../hadoop/hive/metastore/hbase/StatsCache.java | 20 +- .../stats/BinaryColumnStatsAggregator.java | 43 +- .../stats/BooleanColumnStatsAggregator.java | 42 +- .../hbase/stats/ColumnStatsAggregator.java | 12 +- .../stats/ColumnStatsAggregatorFactory.java | 8 +- .../stats/DecimalColumnStatsAggregator.java | 340 +++++- .../stats/DoubleColumnStatsAggregator.java | 307 ++++- .../hbase/stats/IExtrapolatePartStatus.java | 30 + .../hbase/stats/LongColumnStatsAggregator.java | 305 ++++- .../stats/StringColumnStatsAggregator.java | 85 +- ...stHBaseAggregateStatsCacheWithBitVector.java | 6 +- .../TestHBaseAggregateStatsExtrapolation.java | 717 ++++++++++++ .../TestHBaseAggregateStatsNDVUniformDist.java | 581 ++++++++++ orc/src/java/org/apache/orc/OrcFile.java | 21 +- .../java/org/apache/orc/impl/WriterImpl.java | 41 +- ...eColumnArithmeticIntervalYearMonthColumn.txt | 56 +- ...eColumnArithmeticIntervalYearMonthScalar.txt | 55 +- .../DateColumnArithmeticTimestampColumn.txt | 141 ++- .../DateColumnArithmeticTimestampColumnBase.txt | 171 --- .../DateColumnArithmeticTimestampScalar.txt | 113 +- .../DateColumnArithmeticTimestampScalarBase.txt | 137 --- ...eScalarArithmeticIntervalYearMonthColumn.txt | 53 +- .../DateScalarArithmeticTimestampColumn.txt | 108 +- .../DateScalarArithmeticTimestampColumnBase.txt | 147 --- ...ayTimeColumnCompareIntervalDayTimeColumn.txt | 52 - ...ayTimeColumnCompareIntervalDayTimeScalar.txt | 55 - ...ayTimeScalarCompareIntervalDayTimeColumn.txt | 55 - ...erLongDoubleColumnCompareTimestampColumn.txt | 2 +- ...erLongDoubleColumnCompareTimestampScalar.txt | 4 +- ...erLongDoubleScalarCompareTimestampColumn.txt | 4 + .../FilterTimestampColumnBetween.txt | 11 +- ...terTimestampColumnCompareTimestampColumn.txt | 417 ++++++- ...imestampColumnCompareTimestampColumnBase.txt | 429 ------- ...terTimestampColumnCompareTimestampScalar.txt | 128 ++- ...imestampColumnCompareTimestampScalarBase.txt | 145 --- ...erTimestampScalarCompareLongDoubleColumn.txt | 3 +- ...terTimestampScalarCompareTimestampColumn.txt | 132 ++- ...imestampScalarCompareTimestampColumnBase.txt | 147 --- ...ayTimeColumnCompareIntervalDayTimeColumn.txt | 54 - ...ayTimeColumnCompareIntervalDayTimeScalar.txt | 57 - ...ayTimeScalarCompareIntervalDayTimeColumn.txt | 57 - ...ervalYearMonthColumnArithmeticDateColumn.txt | 55 +- ...ervalYearMonthColumnArithmeticDateScalar.txt | 51 +- ...YearMonthColumnArithmeticTimestampColumn.txt | 63 +- ...YearMonthColumnArithmeticTimestampScalar.txt | 48 +- ...ervalYearMonthScalarArithmeticDateColumn.txt | 51 +- ...YearMonthScalarArithmeticTimestampColumn.txt | 55 +- .../LongDoubleColumnCompareTimestampColumn.txt | 1 - .../LongDoubleColumnCompareTimestampScalar.txt | 3 +- .../LongDoubleScalarCompareTimestampColumn.txt | 1 + .../TimestampColumnArithmeticDateColumn.txt | 138 ++- .../TimestampColumnArithmeticDateColumnBase.txt | 172 --- .../TimestampColumnArithmeticDateScalar.txt | 98 +- .../TimestampColumnArithmeticDateScalarBase.txt | 126 -- ...pColumnArithmeticIntervalYearMonthColumn.txt | 59 +- ...pColumnArithmeticIntervalYearMonthScalar.txt | 41 +- ...TimestampColumnArithmeticTimestampColumn.txt | 128 ++- ...stampColumnArithmeticTimestampColumnBase.txt | 152 --- ...TimestampColumnArithmeticTimestampScalar.txt | 96 +- ...stampColumnArithmeticTimestampScalarBase.txt | 125 -- .../TimestampColumnCompareLongDoubleScalar.txt | 1 + .../TimestampColumnCompareTimestampColumn.txt | 122 +- ...imestampColumnCompareTimestampColumnBase.txt | 140 --- .../TimestampColumnCompareTimestampScalar.txt | 114 +- ...imestampColumnCompareTimestampScalarBase.txt | 131 --- .../TimestampScalarArithmeticDateColumn.txt | 117 +- .../TimestampScalarArithmeticDateColumnBase.txt | 151 --- ...pScalarArithmeticIntervalYearMonthColumn.txt | 62 +- ...TimestampScalarArithmeticTimestampColumn.txt | 103 +- ...stampScalarArithmeticTimestampColumnBase.txt | 139 --- .../TimestampScalarCompareLongDoubleColumn.txt | 4 +- .../TimestampScalarCompareTimestampColumn.txt | 115 +- ...imestampScalarCompareTimestampColumnBase.txt | 132 --- .../VectorUDAFMinMaxIntervalDayTime.txt | 454 ++++++++ .../UDAFTemplates/VectorUDAFMinMaxTimestamp.txt | 31 +- .../org/apache/hadoop/hive/llap/DebugUtils.java | 43 - .../org/apache/hadoop/hive/llap/LogLevels.java | 53 - .../java/org/apache/hadoop/hive/ql/Driver.java | 20 + .../org/apache/hadoop/hive/ql/QueryDisplay.java | 11 +- .../hadoop/hive/ql/exec/FunctionRegistry.java | 7 + .../hadoop/hive/ql/exec/OperatorUtils.java | 45 +- .../hive/ql/exec/OrcFileMergeOperator.java | 4 +- .../persistence/HybridHashTableContainer.java | 40 +- .../ql/exec/persistence/KeyValueContainer.java | 25 +- .../ql/exec/persistence/ObjectContainer.java | 24 +- .../hive/ql/exec/persistence/RowContainer.java | 34 +- .../ql/exec/spark/HiveSparkClientFactory.java | 4 + .../ql/exec/spark/SparkReduceRecordHandler.java | 2 + .../hadoop/hive/ql/exec/tez/DagUtils.java | 1 + .../hive/ql/exec/vector/TimestampUtils.java | 8 + .../hive/ql/exec/vector/VectorAssignRow.java | 23 +- .../exec/vector/VectorColumnAssignFactory.java | 19 +- .../ql/exec/vector/VectorColumnSetInfo.java | 23 +- .../hive/ql/exec/vector/VectorCopyRow.java | 32 + .../ql/exec/vector/VectorDeserializeRow.java | 13 +- .../exec/vector/VectorExpressionDescriptor.java | 6 +- .../hive/ql/exec/vector/VectorExtractRow.java | 24 +- .../ql/exec/vector/VectorGroupByOperator.java | 2 +- .../ql/exec/vector/VectorGroupKeyHelper.java | 12 + .../ql/exec/vector/VectorHashKeyWrapper.java | 82 +- .../exec/vector/VectorHashKeyWrapperBatch.java | 112 +- .../hive/ql/exec/vector/VectorSerializeRow.java | 7 +- .../ql/exec/vector/VectorizationContext.java | 27 +- .../ql/exec/vector/VectorizedBatchUtil.java | 20 +- .../ql/exec/vector/VectorizedRowBatchCtx.java | 12 +- .../expressions/CastDecimalToTimestamp.java | 3 +- .../expressions/CastDoubleToTimestamp.java | 17 +- .../vector/expressions/CastLongToTimestamp.java | 8 +- .../CastMillisecondsLongToTimestamp.java | 22 +- .../CastStringToIntervalDayTime.java | 8 +- .../expressions/CastTimestampToBoolean.java | 4 +- .../vector/expressions/CastTimestampToDate.java | 2 +- .../expressions/CastTimestampToDecimal.java | 9 +- .../expressions/CastTimestampToDouble.java | 13 +- .../vector/expressions/CastTimestampToLong.java | 12 +- .../expressions/ConstantVectorExpression.java | 36 +- .../expressions/DateColSubtractDateColumn.java | 80 +- .../expressions/DateColSubtractDateScalar.java | 51 +- .../DateScalarSubtractDateColumn.java | 52 +- .../FilterTimestampColumnInList.java | 27 +- .../IfExprIntervalDayTimeColumnColumn.java | 103 +- .../IfExprIntervalDayTimeColumnScalar.java | 94 +- .../IfExprIntervalDayTimeScalarColumn.java | 96 +- .../IfExprIntervalDayTimeScalarScalar.java | 84 +- .../IfExprTimestampColumnColumnBase.java | 8 +- .../IfExprTimestampColumnScalar.java | 3 +- .../IfExprTimestampColumnScalarBase.java | 14 +- .../IfExprTimestampScalarColumn.java | 3 +- .../IfExprTimestampScalarColumnBase.java | 15 +- .../IfExprTimestampScalarScalar.java | 3 +- .../IfExprTimestampScalarScalarBase.java | 13 +- .../ql/exec/vector/expressions/NullUtil.java | 26 + .../expressions/TimestampColumnInList.java | 29 +- .../expressions/VectorExpressionWriter.java | 6 +- .../VectorExpressionWriterFactory.java | 124 +- .../expressions/VectorUDFDateAddColCol.java | 2 +- .../expressions/VectorUDFDateAddColScalar.java | 2 +- .../expressions/VectorUDFDateDiffColCol.java | 10 +- .../expressions/VectorUDFDateDiffColScalar.java | 2 +- .../expressions/VectorUDFDateDiffScalarCol.java | 2 +- .../expressions/VectorUDFDateTimestamp.java | 2 +- .../expressions/VectorUDFUnixTimeStampDate.java | 7 +- .../VectorUDFUnixTimeStampTimestamp.java | 5 +- .../aggregates/VectorUDAFAvgTimestamp.java | 40 +- .../aggregates/VectorUDAFStdPopTimestamp.java | 24 +- .../aggregates/VectorUDAFStdSampTimestamp.java | 27 +- .../aggregates/VectorUDAFVarPopTimestamp.java | 24 +- .../aggregates/VectorUDAFVarSampTimestamp.java | 24 +- .../mapjoin/VectorMapJoinCommonOperator.java | 2 +- .../mapjoin/VectorMapJoinRowBytesContainer.java | 24 +- .../ql/exec/vector/udf/VectorUDFAdaptor.java | 13 +- .../hadoop/hive/ql/io/orc/RecordReaderImpl.java | 15 +- .../hive/ql/io/orc/encoded/EncodedReader.java | 2 +- .../ql/io/orc/encoded/EncodedReaderImpl.java | 131 +-- .../io/parquet/convert/HiveSchemaConverter.java | 10 +- .../parquet/read/DataWritableReadSupport.java | 75 +- .../ql/io/parquet/serde/ParquetHiveSerDe.java | 11 +- .../apache/hadoop/hive/ql/metadata/Hive.java | 252 ++-- .../hadoop/hive/ql/optimizer/Optimizer.java | 4 +- .../ql/optimizer/ReduceSinkMapJoinProc.java | 24 +- .../hive/ql/optimizer/StatsOptimizer.java | 14 +- .../rules/HivePointLookupOptimizerRule.java | 381 ++++++ .../ql/optimizer/pcr/PcrExprProcFactory.java | 103 +- .../hive/ql/optimizer/physical/Vectorizer.java | 7 + .../stats/annotation/StatsRulesProcFactory.java | 3 +- .../hadoop/hive/ql/parse/CalcitePlanner.java | 40 +- .../hive/ql/parse/DDLSemanticAnalyzer.java | 15 +- .../hadoop/hive/ql/parse/GenTezUtils.java | 3 +- .../hadoop/hive/ql/parse/ParseContext.java | 5 + .../hadoop/hive/ql/parse/SemanticAnalyzer.java | 8 +- .../hadoop/hive/ql/txn/compactor/Cleaner.java | 6 + .../hive/ql/txn/compactor/CompactorThread.java | 5 + .../hadoop/hive/ql/txn/compactor/Initiator.java | 9 +- .../hadoop/hive/ql/txn/compactor/Worker.java | 9 +- .../hadoop/hive/ql/util/DateTimeMath.java | 214 +++- .../apache/hadoop/hive/ql/TestTxnCommands2.java | 47 + .../ql/exec/persistence/TestHashPartition.java | 3 +- .../TestTimestampWritableAndColumnVector.java | 68 ++ .../TestVectorExpressionWriters.java | 23 +- .../TestVectorFilterExpressions.java | 1 - .../expressions/TestVectorMathFunctions.java | 53 +- .../TestVectorTimestampExpressions.java | 26 +- .../vector/expressions/TestVectorTypeCasts.java | 216 ++-- .../TestVectorMapJoinRowBytesContainer.java | 3 +- .../FakeVectorRowBatchFromObjectIterables.java | 3 +- .../vector/util/VectorizedRowGroupGenUtil.java | 14 +- .../hive/ql/io/orc/TestColumnStatistics.java | 5 +- .../hive/ql/io/orc/TestInputOutputFormat.java | 2 +- .../hadoop/hive/ql/io/orc/TestOrcFile.java | 5 +- .../hive/ql/io/orc/TestVectorOrcFile.java | 22 +- .../ql/optimizer/physical/TestVectorizer.java | 5 + .../clientpositive/auto_join19_inclause.q | 18 + .../clientpositive/bucket_map_join_tez1.q | 27 + .../queries/clientpositive/filter_in_or_dup.q | 19 + ql/src/test/queries/clientpositive/llap_udf.q | 6 +- .../clientpositive/multi_insert_with_join.q | 29 + .../test/queries/clientpositive/orc_merge12.q | 51 + .../clientpositive/parquet_schema_evolution.q | 14 + .../sample_islocalmode_hook_use_metadata.q | 48 + .../clientpositive/vector_complex_join.q | 29 + .../clientpositive/vector_interval_arithmetic.q | 174 +++ .../clientnegative/dbtxnmgr_nodblock.q.out | 2 + .../clientnegative/dbtxnmgr_nodbunlock.q.out | 2 + .../lockneg_query_tbl_in_locked_db.q.out | 6 + .../lockneg_try_db_lock_conflict.q.out | 6 + .../lockneg_try_drop_locked_db.q.out | 4 + .../lockneg_try_lock_db_in_use.q.out | 6 + .../clientpositive/auto_join19_inclause.q.out | 130 +++ .../clientpositive/constprog_semijoin.q.out | 4 +- .../dynpart_sort_optimization_acid.q.out | 4 +- .../clientpositive/filter_in_or_dup.q.out | 96 ++ .../llap/bucket_map_join_tez1.q.out | 308 +++++ .../clientpositive/multi_insert_with_join.q.out | 128 +++ .../results/clientpositive/orc_merge12.q.out | 606 ++++++++++ .../parquet_map_null.q.java1.8.out | 1 + .../parquet_schema_evolution.q.out | 65 ++ .../clientpositive/parquet_type_promotion.q.out | 2 +- .../results/clientpositive/perf/query13.q.out | 14 +- .../results/clientpositive/perf/query27.q.out | 2 +- .../results/clientpositive/perf/query34.q.out | 2 +- .../results/clientpositive/perf/query48.q.out | 14 +- .../results/clientpositive/perf/query68.q.out | 2 +- .../results/clientpositive/perf/query73.q.out | 2 +- .../results/clientpositive/perf/query79.q.out | 2 +- .../results/clientpositive/perf/query82.q.out | 2 +- .../results/clientpositive/perf/query85.q.out | 26 +- .../results/clientpositive/pointlookup2.q.out | 38 +- .../results/clientpositive/pointlookup3.q.out | 50 +- .../results/clientpositive/pointlookup4.q.out | 2 +- .../sample_islocalmode_hook_use_metadata.q.out | 230 ++++ .../spark/bucket_map_join_tez1.q.out | 306 +++++ .../spark/constprog_semijoin.q.out | 4 +- .../spark/multi_insert_with_join.q.out | 128 +++ .../tez/bucket_map_join_tez1.q.out | 294 +++++ .../clientpositive/tez/bucketpruning1.q.out | 8 +- .../clientpositive/tez/constprog_semijoin.q.out | 4 +- .../clientpositive/tez/explainuser_1.q.out | 92 +- .../clientpositive/tez/orc_merge12.q.out | 606 ++++++++++ .../tez/vector_complex_join.q.out | 227 ++++ .../tez/vector_interval_arithmetic.q.out | 1086 ++++++++++++++++++ .../tez/vector_mr_diff_schema_alias.q.out | 2 +- .../clientpositive/tez/vectorized_casts.q.out | 18 +- .../tez/vectorized_timestamp.q.out | 157 +++ .../clientpositive/vector_complex_join.q.out | 225 ++++ .../vector_interval_arithmetic.q.out | 1027 +++++++++++++++++ .../vector_mr_diff_schema_alias.q.out | 2 +- .../clientpositive/vectorized_casts.q.out | 18 +- .../hive/serde2/io/TimestampWritable.java | 71 +- .../auth/LdapAuthenticationProviderImpl.java | 317 ++--- .../apache/hive/service/server/HiveServer2.java | 39 +- .../auth/TestLdapAtnProviderWithMiniDS.java | 200 +++- .../apache/hive/service/cli/CLIServiceTest.java | 8 + .../org/apache/hadoop/fs/ProxyFileSystem.java | 5 +- .../hive/common/type/HiveIntervalDayTime.java | 253 ++++ .../hadoop/hive/common/type/PisaTimestamp.java | 609 ---------- .../hadoop/hive/common/type/RandomTypeUtil.java | 70 +- .../hive/ql/exec/vector/ColumnVector.java | 2 +- .../vector/IntervalDayTimeColumnVector.java | 348 ++++++ .../ql/exec/vector/TimestampColumnVector.java | 341 ++---- .../hive/common/util/IntervalDayTimeUtils.java | 77 ++ .../hive/common/type/TestPisaTimestamp.java | 118 -- .../exec/vector/TestTimestampColumnVector.java | 117 ++ 309 files changed, 16607 insertions(+), 7075 deletions(-) ----------------------------------------------------------------------
