HIVE-10769: Merge master to parquet 05/20/2015 [Parquet branch]
Project: http://git-wip-us.apache.org/repos/asf/hive/repo Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/5a0ae3c2 Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/5a0ae3c2 Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/5a0ae3c2 Branch: refs/heads/parquet Commit: 5a0ae3c2a3a8ce0d332eae9b9d3f5a0ecca3e013 Parents: 4580944 1253a8d Author: Sergio Pena <sergio.p...@cloudera.com> Authored: Wed May 20 10:57:42 2015 -0500 Committer: Sergio Pena <sergio.p...@cloudera.com> Committed: Wed May 20 10:57:42 2015 -0500 ---------------------------------------------------------------------- .reviewboardrc | 2 +- README.txt | 36 +- RELEASE_NOTES.txt | 463 + accumulo-handler/pom.xml | 2 +- .../src/test/templates/TestAccumuloCliDriver.vm | 19 +- ant/pom.xml | 2 +- .../apache/hadoop/hive/ant/GenVectorCode.java | 2 + .../apache/hadoop/hive/ant/QTestGenTask.java | 25 + beeline/pom.xml | 2 +- .../java/org/apache/hive/beeline/BeeLine.java | 10 +- bin/beeline.cmd | 11 +- bin/ext/hiveserver2.cmd | 2 +- bin/ext/orcfiledump.sh | 9 +- bin/hive | 9 +- cli/pom.xml | 2 +- .../org/apache/hadoop/hive/cli/CliDriver.java | 2 - common/pom.xml | 2 +- .../apache/hadoop/hive/common/JavaUtils.java | 8 + .../hadoop/hive/common/StatsSetupConst.java | 5 + .../hive/common/jsonexplain/tez/Vertex.java | 2 +- .../hadoop/hive/common/type/HiveDecimal.java | 19 +- .../org/apache/hadoop/hive/conf/HiveConf.java | 146 +- .../apache/hive/common/util/BloomFilter.java | 291 + .../org/apache/hive/common/util/Murmur3.java | 334 + .../apache/hive/common/util/ReflectionUtil.java | 118 + .../hive/common/util/TestBloomFilter.java | 458 + .../apache/hive/common/util/TestMurmur3.java | 189 + contrib/pom.xml | 2 +- data/files/tjoin1.txt | 3 + data/files/tjoin2.txt | 4 + dev-support/jenkins-execute-build.sh | 6 +- dev-support/jenkins-execute-hms-test.sh | 20 +- hbase-handler/pom.xml | 2 +- .../hadoop/hive/hbase/HBaseStorageHandler.java | 8 + .../hive/hbase/HBaseTestStructSerializer.java | 20 +- .../src/test/queries/positive/hbase_timestamp.q | 22 +- .../test/results/positive/hbase_timestamp.q.out | 60 +- .../src/test/templates/TestHBaseCliDriver.vm | 18 +- .../templates/TestHBaseNegativeCliDriver.vm | 19 +- hcatalog/core/pom.xml | 2 +- .../apache/hive/hcatalog/common/HCatUtil.java | 37 +- .../hive/hcatalog/common/HiveClientCache.java | 94 +- .../data/HCatRecordObjectInspectorFactory.java | 2 +- .../hive/hcatalog/data/HCatRecordSerDe.java | 9 +- .../apache/hive/hcatalog/data/JsonSerDe.java | 7 +- .../hcatalog/data/schema/HCatFieldSchema.java | 6 +- .../DefaultOutputCommitterContainer.java | 6 +- .../mapreduce/FileOutputCommitterContainer.java | 22 +- .../mapreduce/FileOutputFormatContainer.java | 8 +- .../hcatalog/mapreduce/HCatBaseInputFormat.java | 50 +- .../hcatalog/mapreduce/HCatOutputFormat.java | 6 +- .../hcatalog/mapreduce/HCatRecordReader.java | 4 +- .../hive/hcatalog/mapreduce/HCatSplit.java | 21 +- .../hive/hcatalog/mapreduce/HCatTableInfo.java | 12 + .../hcatalog/mapreduce/InitializeInput.java | 6 +- .../hive/hcatalog/mapreduce/InputJobInfo.java | 5 + .../hive/hcatalog/mapreduce/InternalUtil.java | 2 + .../hive/hcatalog/mapreduce/OutputJobInfo.java | 2 +- .../hive/hcatalog/mapreduce/PartInfo.java | 117 +- .../hive/hcatalog/mapreduce/Security.java | 10 +- .../mapreduce/TaskCommitContextRegistry.java | 6 +- .../hcatalog/common/TestHiveClientCache.java | 37 +- .../hcatalog/mapreduce/HCatMapReduceTest.java | 3 +- .../mapreduce/TestHCatOutputFormat.java | 5 +- .../hcatalog/mapreduce/TestHCatPartitioned.java | 32 +- .../hcatalog/mapreduce/TestPassProperties.java | 5 +- hcatalog/hcatalog-pig-adapter/pom.xml | 2 +- .../apache/hive/hcatalog/pig/PigHCatUtil.java | 10 +- .../hcatalog/pig/TestHCatLoaderEncryption.java | 3 +- hcatalog/pom.xml | 2 +- hcatalog/server-extensions/pom.xml | 2 +- .../listener/DbNotificationListener.java | 6 +- .../hcatalog/listener/NotificationListener.java | 23 +- .../messaging/AlterPartitionMessage.java | 6 +- .../hive/hcatalog/messaging/InsertMessage.java | 7 +- .../hive/hcatalog/messaging/MessageFactory.java | 10 +- .../json/JSONAlterPartitionMessage.java | 14 +- .../messaging/json/JSONAlterTableMessage.java | 5 + .../messaging/json/JSONInsertMessage.java | 15 +- .../messaging/json/JSONMessageFactory.java | 14 +- .../listener/TestNotificationListener.java | 8 +- hcatalog/src/test/e2e/templeton/README.txt | 13 + hcatalog/src/test/e2e/templeton/build.xml | 29 + .../deployers/config/hive/hive-log4j.properties | 88 + .../deployers/config/hive/hive-site.mysql.xml | 77 + .../webhcat/webhcat-site.updateConfig.xml | 118 + .../deployers/config/webhcat/webhcat-site.xml | 9 +- .../templeton/deployers/deploy_e2e_artifacts.sh | 6 + .../src/test/e2e/templeton/deployers/env.sh | 7 + .../deployers/modify_webhcat_config.sh | 40 + .../deployers/restore_webhcat_config.sh | 36 + .../templeton/deployers/start_hive_services.sh | 7 + .../src/test/e2e/templeton/inpdir/xmlmapper.py | 15 + .../src/test/e2e/templeton/inpdir/xmlreducer.py | 16 + .../templeton/tests/modifyConfiguration.conf | 67 + hcatalog/streaming/pom.xml | 2 +- .../streaming/AbstractRecordWriter.java | 11 +- .../hive/hcatalog/streaming/HiveEndPoint.java | 9 +- hcatalog/webhcat/java-client/pom.xml | 2 +- .../apache/hive/hcatalog/api/HCatClient.java | 18 + .../hive/hcatalog/api/HCatClientHMSImpl.java | 22 +- .../hcatalog/api/HCatNotificationEvent.java | 2 +- .../apache/hive/hcatalog/api/HCatPartition.java | 12 + .../api/repl/HCatReplicationTaskIterator.java | 81 +- .../hcatalog/api/repl/NoopReplicationTask.java | 9 +- .../hive/hcatalog/api/repl/ReplicationTask.java | 103 +- .../hcatalog/api/repl/ReplicationUtils.java | 31 +- .../api/repl/commands/DropDatabaseCommand.java | 99 + .../api/repl/commands/DropPartitionCommand.java | 122 + .../api/repl/commands/DropTableCommand.java | 113 + .../api/repl/commands/ExportCommand.java | 131 + .../api/repl/commands/ImportCommand.java | 139 + .../hcatalog/api/repl/commands/NoopCommand.java | 22 +- .../repl/exim/AddPartitionReplicationTask.java | 111 + .../exim/AlterPartitionReplicationTask.java | 88 + .../repl/exim/AlterTableReplicationTask.java | 85 + .../exim/CreateDatabaseReplicationTask.java | 38 + .../repl/exim/CreateTableReplicationTask.java | 85 + .../repl/exim/DropDatabaseReplicationTask.java | 56 + .../repl/exim/DropPartitionReplicationTask.java | 77 + .../api/repl/exim/DropTableReplicationTask.java | 56 + .../repl/exim/EximReplicationTaskFactory.java | 63 + .../api/repl/exim/InsertReplicationTask.java | 99 + .../hive/hcatalog/api/TestHCatClient.java | 124 +- .../hcatalog/api/repl/CommandTestUtils.java | 107 + .../hcatalog/api/repl/TestReplicationTask.java | 92 + .../api/repl/commands/TestCommands.java | 590 + .../api/repl/commands/TestNoopCommand.java | 42 + .../api/repl/exim/TestEximReplicationTasks.java | 600 + hcatalog/webhcat/svr/pom.xml | 15 +- .../svr/src/main/config/webhcat-default.xml | 10 +- .../hcatalog/templeton/CompleteDelegator.java | 6 +- .../hcatalog/templeton/SecureProxySupport.java | 9 +- .../hcatalog/templeton/tool/LaunchMapper.java | 2 +- .../templeton/tool/TempletonControllerJob.java | 15 +- hwi/pom.xml | 2 +- itests/custom-serde/pom.xml | 2 +- itests/hcatalog-unit/pom.xml | 2 +- .../listener/TestDbNotificationListener.java | 75 +- itests/hive-jmh/pom.xml | 2 +- .../vectorization/VectorizationBench.java | 146 +- itests/hive-minikdc/pom.xml | 2 +- .../hive/minikdc/TestJdbcWithMiniKdcCookie.java | 102 + itests/hive-unit-hadoop2/pom.xml | 2 +- .../apache/hive/jdbc/TestSchedulerQueue.java | 3 +- itests/hive-unit/pom.xml | 135 +- .../org/apache/hive/jdbc/miniHS2/MiniHS2.java | 57 +- .../hive/metastore/TestHiveMetaStore.java | 30 + .../hadoop/hive/metastore/TestHiveMetaTool.java | 17 +- .../metastore/TestMetaStoreEventListener.java | 14 +- .../hive/beeline/TestBeeLineWithArgs.java | 12 + .../jdbc/TestJdbcWithLocalClusterSpark.java | 3 +- .../apache/hive/jdbc/TestJdbcWithMiniMr.java | 3 +- ...stMultiSessionsHS2WithLocalClusterSpark.java | 3 +- .../TestJdbcWithSQLAuthorization.java | 25 + .../operation/OperationLoggingAPITestBase.java | 235 + .../cli/operation/TestOperationLoggingAPI.java | 379 - .../TestOperationLoggingAPIWithMr.java | 168 + .../TestOperationLoggingAPIWithTez.java | 54 + .../cli/thrift/TestThriftHttpCLIService.java | 74 +- itests/pom.xml | 102 +- itests/qtest-spark/pom.xml | 2 +- itests/qtest/pom.xml | 60 +- .../test/resources/testconfiguration.properties | 59 +- itests/test-serde/pom.xml | 2 +- itests/util/pom.xml | 2 +- .../org/apache/hadoop/hive/ql/QTestUtil.java | 78 +- .../generic/GenericUDFTestGetJavaBoolean.java | 8 +- jdbc/pom.xml | 2 +- .../org/apache/hive/jdbc/HiveConnection.java | 101 +- .../apache/hive/jdbc/HiveQueryResultSet.java | 5 + .../org/apache/hive/jdbc/HiveStatement.java | 8 +- .../hive/jdbc/HttpBasicAuthInterceptor.java | 49 +- .../jdbc/HttpKerberosRequestInterceptor.java | 62 +- .../hive/jdbc/HttpRequestInterceptorBase.java | 89 + jdbc/src/java/org/apache/hive/jdbc/Utils.java | 22 +- .../hive/jdbc/ZooKeeperHiveClientHelper.java | 2 +- metastore/bin/.gitignore | 3 +- metastore/pom.xml | 2 +- .../upgrade/derby/hive-schema-1.3.0.derby.sql | 336 + .../derby/upgrade-1.2.0-to-1.3.0.derby.sql | 3 + .../scripts/upgrade/derby/upgrade.order.derby | 1 + .../upgrade/mssql/006-HIVE-9456.mssql.sql | 323 + .../upgrade/mssql/hive-schema-1.2.0.mssql.sql | 256 +- .../upgrade/mssql/hive-schema-1.3.0.mssql.sql | 947 ++ .../mssql/upgrade-1.1.0-to-1.2.0.mssql.sql | 1 + .../mssql/upgrade-1.2.0-to-1.3.0.mssql.sql | 5 + .../scripts/upgrade/mssql/upgrade.order.mssql | 1 + .../upgrade/mysql/021-HIVE-7018.mysql.sql | 53 - .../upgrade/mysql/hive-schema-1.2.0.mysql.sql | 10 +- .../upgrade/mysql/hive-schema-1.3.0.mysql.sql | 840 + .../mysql/upgrade-1.1.0-to-1.2.0.mysql.sql | 2 +- .../mysql/upgrade-1.2.0-to-1.3.0.mysql.sql | 4 + .../scripts/upgrade/mysql/upgrade.order.mysql | 1 + .../upgrade/oracle/hive-schema-1.3.0.oracle.sql | 788 + .../oracle/upgrade-1.2.0-to-1.3.0.oracle.sql | 4 + .../scripts/upgrade/oracle/upgrade.order.oracle | 1 + .../postgres/hive-schema-1.3.0.postgres.sql | 1493 ++ .../upgrade-1.2.0-to-1.3.0.postgres.sql | 12 + .../upgrade/postgres/upgrade.order.postgres | 1 + .../hive/metastore/AggregateStatsCache.java | 572 + .../hadoop/hive/metastore/HiveAlterHandler.java | 32 +- .../hadoop/hive/metastore/HiveMetaStore.java | 56 +- .../hive/metastore/HiveMetaStoreClient.java | 26 +- .../hadoop/hive/metastore/IMetaStoreClient.java | 11 + .../hive/metastore/MetaStoreDirectSql.java | 81 +- .../hadoop/hive/metastore/MetaStoreUtils.java | 27 +- .../hadoop/hive/metastore/ObjectStore.java | 107 +- .../hive/metastore/RetryingMetaStoreClient.java | 89 +- .../metastore/events/DropPartitionEvent.java | 14 +- .../hive/metastore/events/InsertEvent.java | 31 +- .../metastore/events/PreDropPartitionEvent.java | 23 +- .../hive/metastore/tools/HiveMetaTool.java | 35 +- .../hadoop/hive/metastore/txn/TxnHandler.java | 221 +- .../hive/metastore/TestAggregateStatsCache.java | 266 + .../hive/metastore/TestHiveMetastoreCli.java | 63 + .../hive/metastore/txn/TestTxnHandler.java | 40 +- odbc/pom.xml | 2 +- packaging/pom.xml | 2 +- packaging/src/main/assembly/bin.xml | 1 + pom.xml | 12 +- ql/.gitignore | 2 + ql/pom.xml | 18 +- .../ExpressionTemplates/ColumnDivideColumn.txt | 26 +- .../java/org/apache/hadoop/hive/ql/Context.java | 10 +- .../java/org/apache/hadoop/hive/ql/Driver.java | 6 +- .../org/apache/hadoop/hive/ql/ErrorMsg.java | 1 + .../hive/ql/exec/CommonMergeJoinOperator.java | 104 +- .../org/apache/hadoop/hive/ql/exec/DDLTask.java | 100 +- .../hive/ql/exec/DefaultFetchFormatter.java | 5 +- .../hadoop/hive/ql/exec/DemuxOperator.java | 6 +- .../hive/ql/exec/ExprNodeEvaluatorFactory.java | 12 +- .../ql/exec/ExprNodeGenericFuncEvaluator.java | 3 +- .../hive/ql/exec/ExprNodeNullEvaluator.java | 47 - .../hadoop/hive/ql/exec/FetchOperator.java | 15 +- .../apache/hadoop/hive/ql/exec/FetchTask.java | 2 + .../hadoop/hive/ql/exec/FileSinkOperator.java | 47 +- .../hadoop/hive/ql/exec/FunctionRegistry.java | 27 +- .../hadoop/hive/ql/exec/HashTableLoader.java | 4 +- .../hadoop/hive/ql/exec/JoinOperator.java | 3 +- .../apache/hadoop/hive/ql/exec/JoinUtil.java | 4 +- .../hadoop/hive/ql/exec/MapJoinOperator.java | 293 +- .../apache/hadoop/hive/ql/exec/ObjectCache.java | 7 + .../apache/hadoop/hive/ql/exec/Operator.java | 4 + .../hadoop/hive/ql/exec/OperatorFactory.java | 22 +- .../hadoop/hive/ql/exec/ReduceSinkOperator.java | 3 +- .../apache/hadoop/hive/ql/exec/Registry.java | 29 +- .../apache/hadoop/hive/ql/exec/RowSchema.java | 9 + .../hadoop/hive/ql/exec/SMBMapJoinOperator.java | 4 +- .../hadoop/hive/ql/exec/SecureCmdDoAs.java | 5 +- .../ql/exec/SparkHashTableSinkOperator.java | 35 +- .../hadoop/hive/ql/exec/StatsNoJobTask.java | 6 +- .../apache/hadoop/hive/ql/exec/TaskRunner.java | 1 + .../apache/hadoop/hive/ql/exec/Utilities.java | 71 +- .../hadoop/hive/ql/exec/mr/ExecDriver.java | 1 + .../hadoop/hive/ql/exec/mr/HashTableLoader.java | 2 +- .../hadoop/hive/ql/exec/mr/ObjectCache.java | 5 + .../persistence/BytesBytesMultiHashMap.java | 1 + .../exec/persistence/HybridHashTableConf.java | 86 + .../persistence/HybridHashTableContainer.java | 348 +- .../ql/exec/persistence/KeyValueContainer.java | 31 +- .../persistence/MapJoinBytesTableContainer.java | 88 +- .../hive/ql/exec/persistence/MapJoinKey.java | 14 + .../MapJoinTableContainerDirectAccess.java | 31 + .../ql/exec/persistence/ObjectContainer.java | 31 +- .../ReusableGetAdaptorDirectAccess.java | 30 + .../hive/ql/exec/persistence/RowContainer.java | 4 +- .../hive/ql/exec/spark/HashTableLoader.java | 2 +- .../ql/exec/spark/HiveSparkClientFactory.java | 28 +- .../hive/ql/exec/spark/KryoSerializer.java | 4 +- .../ql/exec/spark/RemoteHiveSparkClient.java | 28 +- .../ql/exec/spark/SparkMapRecordHandler.java | 1 + .../hive/ql/exec/spark/SparkPlanGenerator.java | 35 +- .../ql/exec/spark/SparkReduceRecordHandler.java | 1 + .../hadoop/hive/ql/exec/spark/SparkTask.java | 16 + .../hive/ql/exec/spark/SparkUtilities.java | 27 +- .../spark/status/impl/JobMetricsListener.java | 12 + .../hadoop/hive/ql/exec/tez/DagUtils.java | 1 + .../hive/ql/exec/tez/HashTableLoader.java | 73 +- .../hadoop/hive/ql/exec/tez/ObjectCache.java | 6 + .../hive/ql/exec/tez/ReduceRecordSource.java | 148 +- .../hadoop/hive/ql/exec/tez/TezJobMonitor.java | 4 +- .../hive/ql/exec/tez/TezSessionState.java | 13 +- .../apache/hadoop/hive/ql/exec/tez/TezTask.java | 3 +- .../vector/VectorAppMasterEventOperator.java | 123 +- .../ql/exec/vector/VectorColumnMapping.java | 6 +- .../ql/exec/vector/VectorColumnOrderedMap.java | 17 +- .../exec/vector/VectorColumnOutputMapping.java | 4 + .../ql/exec/vector/VectorColumnSetInfo.java | 3 +- .../exec/vector/VectorColumnSourceMapping.java | 4 + .../hive/ql/exec/vector/VectorCopyRow.java | 2 +- .../ql/exec/vector/VectorDeserializeRow.java | 13 +- .../ql/exec/vector/VectorFileSinkOperator.java | 78 +- .../ql/exec/vector/VectorGroupByOperator.java | 26 +- .../exec/vector/VectorMapJoinBaseOperator.java | 185 + .../ql/exec/vector/VectorMapJoinOperator.java | 171 +- .../VectorMapJoinOuterFilteredOperator.java | 122 + .../exec/vector/VectorReduceSinkOperator.java | 83 +- .../exec/vector/VectorSMBMapJoinOperator.java | 21 +- .../hive/ql/exec/vector/VectorSerializeRow.java | 17 + .../exec/vector/VectorSerializeRowNoNulls.java | 17 + .../ql/exec/vector/VectorizationContext.java | 89 +- .../ql/exec/vector/VectorizedBatchUtil.java | 54 +- .../ql/exec/vector/VectorizedRowBatchCtx.java | 4 +- .../expressions/ConstantVectorExpression.java | 3 - .../vector/expressions/IdentityExpression.java | 9 + .../ql/exec/vector/expressions/MathExpr.java | 9 +- .../VectorExpressionWriterFactory.java | 29 +- .../mapjoin/VectorMapJoinCommonOperator.java | 775 + .../VectorMapJoinGenerateResultOperator.java | 852 + ...pJoinInnerBigOnlyGenerateResultOperator.java | 328 + .../VectorMapJoinInnerBigOnlyLongOperator.java | 374 + ...ctorMapJoinInnerBigOnlyMultiKeyOperator.java | 387 + ...VectorMapJoinInnerBigOnlyStringOperator.java | 366 + ...ectorMapJoinInnerGenerateResultOperator.java | 228 + .../mapjoin/VectorMapJoinInnerLongOperator.java | 371 + .../VectorMapJoinInnerMultiKeyOperator.java | 383 + .../VectorMapJoinInnerStringOperator.java | 360 + ...orMapJoinLeftSemiGenerateResultOperator.java | 225 + .../VectorMapJoinLeftSemiLongOperator.java | 364 + .../VectorMapJoinLeftSemiMultiKeyOperator.java | 380 + .../VectorMapJoinLeftSemiStringOperator.java | 353 + ...ectorMapJoinOuterGenerateResultOperator.java | 745 + .../mapjoin/VectorMapJoinOuterLongOperator.java | 447 + .../VectorMapJoinOuterMultiKeyOperator.java | 464 + .../VectorMapJoinOuterStringOperator.java | 433 + .../mapjoin/VectorMapJoinRowBytesContainer.java | 318 + .../fast/VectorMapJoinFastBytesHashMap.java | 101 + .../VectorMapJoinFastBytesHashMultiSet.java | 93 + .../fast/VectorMapJoinFastBytesHashSet.java | 85 + .../fast/VectorMapJoinFastBytesHashTable.java | 221 + .../fast/VectorMapJoinFastBytesHashUtil.java | 41 + .../mapjoin/fast/VectorMapJoinFastHashMap.java | 38 + .../fast/VectorMapJoinFastHashMultiSet.java | 48 + .../mapjoin/fast/VectorMapJoinFastHashSet.java | 44 + .../fast/VectorMapJoinFastHashTable.java | 68 + .../fast/VectorMapJoinFastHashTableLoader.java | 114 + .../fast/VectorMapJoinFastIntHashUtil.java | 32 + .../mapjoin/fast/VectorMapJoinFastKeyStore.java | 173 + .../fast/VectorMapJoinFastLongHashMap.java | 94 + .../fast/VectorMapJoinFastLongHashMultiSet.java | 91 + .../fast/VectorMapJoinFastLongHashSet.java | 84 + .../fast/VectorMapJoinFastLongHashTable.java | 284 + .../fast/VectorMapJoinFastLongHashUtil.java | 63 + .../fast/VectorMapJoinFastMultiKeyHashMap.java | 39 + .../VectorMapJoinFastMultiKeyHashMultiSet.java | 32 + .../fast/VectorMapJoinFastMultiKeyHashSet.java | 32 + .../fast/VectorMapJoinFastStringCommon.java | 67 + .../fast/VectorMapJoinFastStringHashMap.java | 44 + .../VectorMapJoinFastStringHashMultiSet.java | 44 + .../fast/VectorMapJoinFastStringHashSet.java | 44 + .../fast/VectorMapJoinFastTableContainer.java | 222 + .../fast/VectorMapJoinFastValueStore.java | 557 + .../hashtable/VectorMapJoinBytesHashMap.java | 51 + .../VectorMapJoinBytesHashMultiSet.java | 51 + .../hashtable/VectorMapJoinBytesHashSet.java | 51 + .../hashtable/VectorMapJoinBytesHashTable.java | 26 + .../mapjoin/hashtable/VectorMapJoinHashMap.java | 34 + .../hashtable/VectorMapJoinHashMapResult.java | 63 + .../hashtable/VectorMapJoinHashMultiSet.java | 31 + .../VectorMapJoinHashMultiSetResult.java | 34 + .../mapjoin/hashtable/VectorMapJoinHashSet.java | 34 + .../hashtable/VectorMapJoinHashSetResult.java | 28 + .../hashtable/VectorMapJoinHashTable.java | 43 + .../hashtable/VectorMapJoinHashTableResult.java | 81 + .../hashtable/VectorMapJoinLongHashMap.java | 46 + .../VectorMapJoinLongHashMultiSet.java | 46 + .../hashtable/VectorMapJoinLongHashSet.java | 46 + .../hashtable/VectorMapJoinLongHashTable.java | 31 + .../hashtable/VectorMapJoinTableContainer.java | 28 + .../VectorMapJoinOptimizedCreateHashTable.java | 129 + .../VectorMapJoinOptimizedHashMap.java | 128 + .../VectorMapJoinOptimizedHashMultiSet.java | 103 + .../VectorMapJoinOptimizedHashSet.java | 78 + .../VectorMapJoinOptimizedHashTable.java | 95 + .../VectorMapJoinOptimizedLongCommon.java | 171 + .../VectorMapJoinOptimizedLongHashMap.java | 82 + .../VectorMapJoinOptimizedLongHashMultiSet.java | 83 + .../VectorMapJoinOptimizedLongHashSet.java | 83 + .../VectorMapJoinOptimizedMultiKeyHashMap.java | 36 + ...torMapJoinOptimizedMultiKeyHashMultiSet.java | 36 + .../VectorMapJoinOptimizedMultiKeyHashSet.java | 36 + .../VectorMapJoinOptimizedStringCommon.java | 98 + .../VectorMapJoinOptimizedStringHashMap.java | 63 + ...ectorMapJoinOptimizedStringHashMultiSet.java | 64 + .../VectorMapJoinOptimizedStringHashSet.java | 63 + .../hadoop/hive/ql/history/HiveHistory.java | 3 +- .../hive/ql/io/BucketizedHiveInputSplit.java | 5 +- .../hive/ql/io/CombineHiveInputFormat.java | 4 + .../hadoop/hive/ql/io/HiveFileFormatUtils.java | 4 +- .../hadoop/hive/ql/io/HiveInputFormat.java | 8 +- .../hadoop/hive/ql/io/filters/BloomFilter.java | 298 - .../hive/ql/io/filters/BloomFilterIO.java | 44 + .../hadoop/hive/ql/io/filters/Murmur3.java | 334 - .../hive/ql/io/orc/ColumnStatisticsImpl.java | 35 +- .../ql/io/orc/ConversionTreeReaderFactory.java | 38 + .../hive/ql/io/orc/DateColumnStatistics.java | 6 +- .../apache/hadoop/hive/ql/io/orc/FileDump.java | 99 +- .../hadoop/hive/ql/io/orc/IntegerWriter.java | 5 - .../hadoop/hive/ql/io/orc/JsonFileDump.java | 365 + .../hadoop/hive/ql/io/orc/MetadataReader.java | 1 - .../apache/hadoop/hive/ql/io/orc/OrcFile.java | 4 +- .../hadoop/hive/ql/io/orc/OrcInputFormat.java | 8 +- .../hadoop/hive/ql/io/orc/OrcRecordUpdater.java | 24 +- .../apache/hadoop/hive/ql/io/orc/OrcUtils.java | 2 +- .../hive/ql/io/orc/RecordReaderFactory.java | 269 + .../hadoop/hive/ql/io/orc/RecordReaderImpl.java | 236 +- .../hive/ql/io/orc/RecordReaderUtils.java | 12 +- .../hive/ql/io/orc/RunLengthIntegerWriter.java | 9 +- .../ql/io/orc/RunLengthIntegerWriterV2.java | 11 +- .../hive/ql/io/orc/TreeReaderFactory.java | 98 +- .../hadoop/hive/ql/io/orc/WriterImpl.java | 15 +- .../ql/io/parquet/MapredParquetInputFormat.java | 7 +- .../io/parquet/MapredParquetOutputFormat.java | 7 +- .../parquet/VectorizedParquetInputFormat.java | 2 +- .../convert/HiveCollectionConverter.java | 18 + .../hive/ql/io/parquet/convert/Repeated.java | 18 + .../read/ParquetRecordReaderWrapper.java | 7 +- .../write/ParquetRecordWriterWrapper.java | 7 +- .../hive/ql/io/sarg/SearchArgumentImpl.java | 85 +- .../hadoop/hive/ql/lib/PreOrderWalker.java | 8 + .../hadoop/hive/ql/lockmgr/DbLockManager.java | 41 +- .../hadoop/hive/ql/lockmgr/DbTxnManager.java | 22 +- .../ql/lockmgr/zookeeper/ZooKeeperHiveLock.java | 22 + .../apache/hadoop/hive/ql/metadata/Hive.java | 30 +- .../apache/hadoop/hive/ql/metadata/Table.java | 18 +- .../BucketingSortingReduceSinkOptimizer.java | 13 + .../ql/optimizer/ColumnPrunerProcFactory.java | 25 +- .../optimizer/ConstantPropagateProcFactory.java | 144 +- .../hive/ql/optimizer/ConvertJoinMapJoin.java | 27 +- .../hive/ql/optimizer/GroupByOptimizer.java | 7 +- .../ql/optimizer/IdentityProjectRemover.java | 12 + .../ql/optimizer/LimitPushdownOptimizer.java | 9 +- .../ql/optimizer/NonBlockingOpDeDupProc.java | 13 +- .../hadoop/hive/ql/optimizer/Optimizer.java | 16 +- .../PrunerExpressionOperatorFactory.java | 3 - .../optimizer/RemoveDynamicPruningBySize.java | 19 +- .../hive/ql/optimizer/SimpleFetchOptimizer.java | 2 - .../ql/optimizer/calcite/HiveCalciteUtil.java | 412 +- .../ql/optimizer/calcite/HiveConfigContext.java | 37 + .../calcite/HiveDefaultRelMetadataProvider.java | 55 +- .../ql/optimizer/calcite/HiveRelCollation.java | 34 + .../optimizer/calcite/HiveRelDistribution.java | 80 + .../ql/optimizer/calcite/HiveRelOptUtil.java | 414 + .../ql/optimizer/calcite/RelOptHiveTable.java | 217 +- .../calcite/cost/HiveAlgorithmsConf.java | 39 + .../calcite/cost/HiveAlgorithmsUtil.java | 363 + .../ql/optimizer/calcite/cost/HiveCost.java | 52 +- .../optimizer/calcite/cost/HiveCostModel.java | 102 + .../ql/optimizer/calcite/cost/HiveCostUtil.java | 43 - .../calcite/cost/HiveDefaultCostModel.java | 123 + .../calcite/cost/HiveOnTezCostModel.java | 635 + .../optimizer/calcite/cost/HiveRelMdCost.java | 71 + .../calcite/cost/HiveVolcanoPlanner.java | 9 +- .../calcite/reloperators/HiveAggregate.java | 10 +- .../calcite/reloperators/HiveFilter.java | 4 +- .../calcite/reloperators/HiveGroupingID.java | 17 + .../calcite/reloperators/HiveJoin.java | 151 +- .../calcite/reloperators/HiveLimit.java | 4 +- .../calcite/reloperators/HiveProject.java | 4 +- .../calcite/reloperators/HiveSortExchange.java | 84 + .../calcite/reloperators/HiveTableScan.java | 126 +- .../rules/HiveExpandDistinctAggregatesRule.java | 278 + .../rules/HiveInsertExchange4JoinRule.java | 134 + .../calcite/rules/HiveJoinAddNotNullRule.java | 197 + .../calcite/rules/HiveJoinToMultiJoinRule.java | 333 + .../calcite/rules/HiveWindowingFixRule.java | 163 + .../calcite/stats/HiveRelMdCollation.java | 67 + .../calcite/stats/HiveRelMdDistribution.java | 56 + .../calcite/stats/HiveRelMdMemory.java | 102 + .../calcite/stats/HiveRelMdParallelism.java | 117 + .../calcite/stats/HiveRelMdRowCount.java | 1 - .../optimizer/calcite/stats/HiveRelMdSize.java | 148 + .../calcite/stats/HiveRelMdUniqueKeys.java | 1 - .../calcite/translator/ASTBuilder.java | 3 +- .../calcite/translator/ASTConverter.java | 24 +- .../calcite/translator/ExprNodeConverter.java | 245 +- .../calcite/translator/HiveGBOpConvUtil.java | 1226 ++ .../calcite/translator/HiveOpConverter.java | 986 ++ .../translator/HiveOpConverterPostProc.java | 175 + .../translator/PlanModifierForASTConv.java | 111 +- .../translator/PlanModifierForReturnPath.java | 41 + .../calcite/translator/PlanModifierUtil.java | 138 + .../calcite/translator/RexNodeConverter.java | 5 +- .../calcite/translator/TypeConverter.java | 4 +- .../ql/optimizer/lineage/ExprProcFactory.java | 3 +- .../ql/optimizer/pcr/PcrExprProcFactory.java | 3 +- .../physical/GenSparkSkewJoinProcessor.java | 9 +- .../physical/SparkMapJoinResolver.java | 185 +- .../hive/ql/optimizer/physical/Vectorizer.java | 405 +- .../spark/SparkSkewJoinProcFactory.java | 57 +- .../optimizer/spark/SparkSkewJoinResolver.java | 5 +- .../stats/annotation/StatsRulesProcFactory.java | 4 +- .../hive/ql/parse/BaseSemanticAnalyzer.java | 96 +- .../hadoop/hive/ql/parse/CalcitePlanner.java | 306 +- .../ql/parse/ColumnStatsSemanticAnalyzer.java | 24 +- .../hive/ql/parse/DDLSemanticAnalyzer.java | 159 +- .../apache/hadoop/hive/ql/parse/EximUtil.java | 173 +- .../hive/ql/parse/ExportSemanticAnalyzer.java | 121 +- .../hadoop/hive/ql/parse/GenTezProcContext.java | 5 +- .../hadoop/hive/ql/parse/GenTezUtils.java | 21 +- .../apache/hadoop/hive/ql/parse/GenTezWork.java | 40 +- .../hadoop/hive/ql/parse/GenTezWorkWalker.java | 6 + .../org/apache/hadoop/hive/ql/parse/HiveLexer.g | 2 + .../apache/hadoop/hive/ql/parse/HiveParser.g | 51 +- .../hadoop/hive/ql/parse/IdentifiersParser.g | 8 +- .../hive/ql/parse/ImportSemanticAnalyzer.java | 768 +- .../hive/ql/parse/MetaDataExportListener.java | 2 +- .../hive/ql/parse/OptimizeTezProcContext.java | 14 + .../hadoop/hive/ql/parse/PTFTranslator.java | 14 +- .../apache/hadoop/hive/ql/parse/ParseUtils.java | 42 +- .../org/apache/hadoop/hive/ql/parse/QB.java | 10 +- .../hadoop/hive/ql/parse/ReplicationSpec.java | 314 + .../hadoop/hive/ql/parse/SemanticAnalyzer.java | 168 +- .../hive/ql/parse/TableAccessAnalyzer.java | 6 +- .../hadoop/hive/ql/parse/TezCompiler.java | 19 +- .../hive/ql/parse/TypeCheckProcFactory.java | 6 +- .../hadoop/hive/ql/parse/UnparseTranslator.java | 2 +- .../ql/parse/UpdateDeleteSemanticAnalyzer.java | 13 +- .../hadoop/hive/ql/parse/WindowingSpec.java | 9 +- .../hive/ql/parse/spark/GenSparkUtils.java | 33 +- .../hadoop/hive/ql/plan/AddPartitionDesc.java | 15 + .../apache/hadoop/hive/ql/plan/BaseWork.java | 1 + .../hadoop/hive/ql/plan/CreateTableDesc.java | 14 + .../hadoop/hive/ql/plan/DropTableDesc.java | 23 +- .../hive/ql/plan/ExprNodeConstantDesc.java | 15 +- .../hadoop/hive/ql/plan/ExprNodeDescUtils.java | 39 + .../hadoop/hive/ql/plan/ExprNodeNullDesc.java | 69 - .../apache/hadoop/hive/ql/plan/JoinDesc.java | 19 + .../apache/hadoop/hive/ql/plan/MapJoinDesc.java | 29 +- .../org/apache/hadoop/hive/ql/plan/MapWork.java | 12 +- .../hadoop/hive/ql/plan/PartitionDesc.java | 4 +- .../apache/hadoop/hive/ql/plan/PlanUtils.java | 50 +- .../hadoop/hive/ql/plan/ReduceSinkDesc.java | 12 + .../apache/hadoop/hive/ql/plan/ReduceWork.java | 5 +- .../apache/hadoop/hive/ql/plan/SparkWork.java | 11 +- .../apache/hadoop/hive/ql/plan/TableDesc.java | 5 +- .../hadoop/hive/ql/plan/VectorMapJoinDesc.java | 107 + .../hadoop/hive/ql/plan/ptf/BoundaryDef.java | 42 +- .../hadoop/hive/ql/plan/ptf/CurrentRowDef.java | 12 +- .../hive/ql/plan/ptf/RangeBoundaryDef.java | 38 +- .../hive/ql/plan/ptf/ValueBoundaryDef.java | 50 +- .../hadoop/hive/ql/plan/ptf/WindowFrameDef.java | 31 +- .../AuthorizationPreEventListener.java | 68 +- ...MultiPartitionAuthorizationProviderBase.java | 43 + .../MetaStoreAuthzAPIAuthorizerEmbedOnly.java | 3 +- .../StorageBasedAuthorizationProvider.java | 11 +- .../hadoop/hive/ql/session/SessionState.java | 33 +- .../apache/hadoop/hive/ql/stats/StatsUtils.java | 12 +- .../hadoop/hive/ql/txn/compactor/Cleaner.java | 38 +- .../hive/ql/txn/compactor/CompactorThread.java | 12 +- .../hadoop/hive/ql/txn/compactor/Initiator.java | 11 +- .../hadoop/hive/ql/txn/compactor/Worker.java | 42 +- .../org/apache/hadoop/hive/ql/udf/UDFCrc32.java | 75 + .../org/apache/hadoop/hive/ql/udf/UDFMd5.java | 88 + .../apache/hadoop/hive/ql/udf/UDFRegExp.java | 76 - .../org/apache/hadoop/hive/ql/udf/UDFSha1.java | 88 + .../hive/ql/udf/generic/GenericUDAFAverage.java | 43 +- .../ql/udf/generic/GenericUDAFFirstValue.java | 20 +- .../ql/udf/generic/GenericUDAFLastValue.java | 18 +- .../hive/ql/udf/generic/GenericUDAFMax.java | 20 +- .../generic/GenericUDAFStreamingEvaluator.java | 97 +- .../hive/ql/udf/generic/GenericUDAFSum.java | 41 +- .../hadoop/hive/ql/udf/generic/GenericUDF.java | 15 +- .../ql/udf/generic/GenericUDFAddMonths.java | 5 +- .../hive/ql/udf/generic/GenericUDFArray.java | 5 +- .../hive/ql/udf/generic/GenericUDFBaseDTI.java | 18 + .../hive/ql/udf/generic/GenericUDFCbrt.java | 3 +- .../hive/ql/udf/generic/GenericUDFCoalesce.java | 2 +- .../ql/udf/generic/GenericUDFDateFormat.java | 125 + .../hive/ql/udf/generic/GenericUDFDecode.java | 26 +- .../hive/ql/udf/generic/GenericUDFEncode.java | 2 +- .../ql/udf/generic/GenericUDFFactorial.java | 3 +- .../hive/ql/udf/generic/GenericUDFGreatest.java | 1 + .../hive/ql/udf/generic/GenericUDFInstr.java | 2 +- .../hive/ql/udf/generic/GenericUDFLastDay.java | 4 +- .../ql/udf/generic/GenericUDFLevenshtein.java | 5 +- .../hive/ql/udf/generic/GenericUDFLocate.java | 2 +- .../hive/ql/udf/generic/GenericUDFMap.java | 20 +- .../hive/ql/udf/generic/GenericUDFNextDay.java | 11 +- .../ql/udf/generic/GenericUDFParamUtils.java | 71 + .../hive/ql/udf/generic/GenericUDFPrintf.java | 3 +- .../hive/ql/udf/generic/GenericUDFQuarter.java | 85 + .../hive/ql/udf/generic/GenericUDFRegExp.java | 133 + .../hive/ql/udf/generic/GenericUDFSha2.java | 137 + .../hive/ql/udf/generic/GenericUDFSoundex.java | 6 +- .../ql/udf/generic/GenericUDFTimestamp.java | 24 + .../ql/udf/generic/GenericUDFTranslate.java | 8 +- .../hive/ql/udf/generic/GenericUDFTrunc.java | 11 +- .../hive/ql/udf/generic/GenericUDFUtils.java | 6 +- .../apache/hadoop/hive/ql/TestTxnCommands2.java | 77 +- .../session/TestSparkSessionManagerImpl.java | 5 + .../exec/vector/mapjoin/TestDebugDisplay.java | 63 + .../mapjoin/fast/CommonFastHashTable.java | 128 + .../mapjoin/fast/RandomByteArrayStream.java | 92 + .../vector/mapjoin/fast/RandomLongStream.java | 49 + .../fast/TestVectorMapJoinFastLongHashMap.java | 219 + .../TestVectorMapJoinFastMultiKeyHashMap.java | 231 + .../TestVectorMapJoinRowBytesContainer.java | 74 + .../hive/ql/io/filters/TestBloomFilter.java | 458 - .../hadoop/hive/ql/io/filters/TestMurmur3.java | 189 - .../hive/ql/io/orc/TestColumnStatistics.java | 20 +- .../hadoop/hive/ql/io/orc/TestFileDump.java | 2 +- .../hadoop/hive/ql/io/orc/TestJsonFileDump.java | 138 + .../hadoop/hive/ql/io/orc/TestOrcFile.java | 105 +- .../hive/ql/io/orc/TestOrcSerDeStats.java | 12 +- .../hadoop/hive/ql/io/orc/TestOrcTimezone1.java | 4 +- .../hadoop/hive/ql/io/orc/TestOrcTimezone2.java | 2 +- .../hive/ql/io/orc/TestRecordReaderImpl.java | 328 +- .../io/parquet/AbstractTestParquetDirect.java | 23 +- .../ql/io/parquet/TestArrayCompatibility.java | 18 + .../hive/ql/io/parquet/TestMapStructures.java | 18 + .../hive/ql/io/sarg/TestSearchArgumentImpl.java | 104 +- .../hive/ql/lockmgr/TestDbTxnManager.java | 2 + .../hive/ql/lockmgr/TestDbTxnManager2.java | 233 + .../ql/optimizer/physical/TestVectorizer.java | 2 +- ...tedCharsInColumnNameCreateTableNegative.java | 87 + .../hadoop/hive/ql/plan/TestViewEntity.java | 108 + .../hadoop/hive/ql/session/TestAddResource.java | 116 +- .../hive/ql/txn/compactor/TestCleaner.java | 56 +- .../hive/ql/txn/compactor/TestInitiator.java | 63 +- .../hive/ql/txn/compactor/TestWorker.java | 45 + .../hadoop/hive/ql/udaf/TestStreamingSum.java | 16 +- .../hadoop/hive/ql/udf/TestGenericUDFDate.java | 92 - .../hive/ql/udf/TestGenericUDFDateAdd.java | 145 - .../hive/ql/udf/TestGenericUDFDateDiff.java | 116 - .../hive/ql/udf/TestGenericUDFDateSub.java | 143 - .../hadoop/hive/ql/udf/TestGenericUDFUtils.java | 58 - .../apache/hadoop/hive/ql/udf/TestUDFCrc32.java | 74 + .../apache/hadoop/hive/ql/udf/TestUDFJson.java | 18 + .../apache/hadoop/hive/ql/udf/TestUDFMd5.java | 57 + .../apache/hadoop/hive/ql/udf/TestUDFSha1.java | 57 + .../hive/ql/udf/generic/TestGenericUDFDate.java | 92 + .../ql/udf/generic/TestGenericUDFDateAdd.java | 143 + .../ql/udf/generic/TestGenericUDFDateDiff.java | 116 + .../udf/generic/TestGenericUDFDateFormat.java | 173 + .../ql/udf/generic/TestGenericUDFDateSub.java | 143 + .../udf/generic/TestGenericUDFLevenshtein.java | 4 +- .../ql/udf/generic/TestGenericUDFNextDay.java | 4 +- .../ql/udf/generic/TestGenericUDFQuarter.java | 182 + .../ql/udf/generic/TestGenericUDFRegexp.java | 135 + .../hive/ql/udf/generic/TestGenericUDFSha2.java | 271 + .../ql/udf/generic/TestGenericUDFUtils.java | 57 + .../authorization_set_nonexistent_conf.q | 7 + .../queries/clientnegative/insertsel_fail.q | 1 + .../clientnegative/protectmode_part_no_drop2.q | 11 + .../clientpositive/alter_change_db_location.q | 5 + .../clientpositive/alter_partition_coltype.q | 1 + .../clientpositive/annotate_stats_part.q | 1 + .../clientpositive/auto_sortmerge_join_13.q | 2 + .../test/queries/clientpositive/bucket_many.q | 16 + ql/src/test/queries/clientpositive/cbo_join.q | 1 + ql/src/test/queries/clientpositive/cbo_limit.q | 2 +- .../queries/clientpositive/cbo_rp_auto_join0.q | 24 + .../queries/clientpositive/cbo_rp_auto_join1.q | 274 + .../queries/clientpositive/cbo_simple_select.q | 3 +- .../test/queries/clientpositive/cbo_subq_in.q | 8 +- .../queries/clientpositive/cbo_subq_not_in.q | 4 +- .../test/queries/clientpositive/cbo_udf_udaf.q | 12 +- .../queries/clientpositive/decimal_precision2.q | 15 + .../dynamic_partition_pruning_2.q | 17 + .../clientpositive/encryption_move_tbl.q | 18 + .../test/queries/clientpositive/explainuser_2.q | 23 + .../extrapolate_part_stats_partial.q | 2 + .../extrapolate_part_stats_partial_ndv.q | 2 + ql/src/test/queries/clientpositive/fold_case.q | 12 + .../clientpositive/fold_eq_with_case_when.q | 21 + ql/src/test/queries/clientpositive/fold_when.q | 31 + .../clientpositive/hybridgrace_hashjoin_1.q | 258 + .../clientpositive/hybridgrace_hashjoin_2.q | 152 + .../queries/clientpositive/hybridhashjoin.q | 250 - .../clientpositive/insert_overwrite_directory.q | 141 + .../queries/clientpositive/join_on_varchar.q | 12 + .../clientpositive/json_serde_qualified_types.q | 12 + .../queries/clientpositive/limit_pushdown.q | 4 + .../queries/clientpositive/mapjoin_mapjoin.q | 1 + ql/src/test/queries/clientpositive/mergejoin.q | 124 + .../queries/clientpositive/optimize_nullscan.q | 4 + .../clientpositive/orc_int_type_promotion.q | 79 + .../clientpositive/partition_coltype_literals.q | 75 + .../test/queries/clientpositive/repl_1_drop.q | 84 + .../queries/clientpositive/repl_2_exim_basic.q | 79 + .../clientpositive/repl_3_exim_metadata.q | 40 + .../runtime_skewjoin_mapjoin_spark.q | 24 + .../test/queries/clientpositive/tez_join_hash.q | 2 + ql/src/test/queries/clientpositive/tez_smb_1.q | 1 + .../test/queries/clientpositive/tez_smb_main.q | 7 + ql/src/test/queries/clientpositive/tez_union.q | 18 + .../clientpositive/tez_union_multiinsert.q | 120 + .../clientpositive/timestamp_ints_casts.q | 74 + .../clientpositive/udaf_percentile_approx_23.q | 2 +- ql/src/test/queries/clientpositive/udf_crc32.q | 13 + .../queries/clientpositive/udf_date_format.q | 60 + ql/src/test/queries/clientpositive/udf_md5.q | 13 + .../test/queries/clientpositive/udf_quarter.q | 100 + ql/src/test/queries/clientpositive/udf_sha1.q | 13 + ql/src/test/queries/clientpositive/udf_sha2.q | 41 + ql/src/test/queries/clientpositive/union12.q | 2 + ql/src/test/queries/clientpositive/union17.q | 2 +- ql/src/test/queries/clientpositive/union20.q | 2 +- ql/src/test/queries/clientpositive/union21.q | 2 +- ql/src/test/queries/clientpositive/union27.q | 2 +- .../queries/clientpositive/union_remove_22.q | 2 + .../clientpositive/union_remove_6_subq.q | 2 + .../queries/clientpositive/vector_aggregate_9.q | 2 + .../clientpositive/vector_binary_join_groupby.q | 55 + .../clientpositive/vector_char_mapjoin1.q | 9 + .../clientpositive/vector_decimal_mapjoin.q | 43 +- .../queries/clientpositive/vector_inner_join.q | 61 + .../test/queries/clientpositive/vector_join30.q | 160 + .../clientpositive/vector_join_filters.q | 38 + .../queries/clientpositive/vector_join_nulls.q | 33 + .../clientpositive/vector_left_outer_join.q | 1 + .../clientpositive/vector_left_outer_join2.q | 64 + .../clientpositive/vector_leftsemi_mapjoin.q | 403 + .../vector_mr_diff_schema_alias.q | 115 + .../clientpositive/vector_nullsafe_join.q | 72 + .../queries/clientpositive/vector_outer_join0.q | 25 + .../queries/clientpositive/vector_outer_join1.q | 64 + .../queries/clientpositive/vector_outer_join2.q | 39 + .../queries/clientpositive/vector_outer_join3.q | 80 + .../queries/clientpositive/vector_outer_join4.q | 66 + .../queries/clientpositive/vector_outer_join5.q | 173 + .../queries/clientpositive/vectorized_casts.q | 2 +- .../queries/clientpositive/vectorized_context.q | 1 + .../clientpositive/vectorized_timestamp_funcs.q | 36 +- .../vectorized_timestamp_ints_casts.q | 76 + .../queries/clientpositive/windowing_navfn.q | 4 +- .../clientpositive/windowing_windowspec2.q | 24 + ql/src/test/resources/orc-file-dump.json | 1354 ++ ql/src/test/resources/orc-file-has-null.out | 42 +- .../alter_rename_partition_failure3.q.out | 2 +- .../alter_table_add_partition.q.out | 2 +- .../clientnegative/alter_view_failure5.q.out | 2 +- .../clientnegative/alter_view_failure7.q.out | 2 +- .../clientnegative/archive_partspec1.q.out | 2 +- .../clientnegative/archive_partspec5.q.out | 8 +- .../authorization_set_nonexistent_conf.q.out | 9 + .../results/clientnegative/insertsel_fail.q.out | 1 + .../clientnegative/join_nonexistent_part.q.out | 2 +- .../protectmode_part_no_drop2.q.out | 51 + ql/src/test/results/clientnegative/touch2.q.out | 2 +- .../truncate_partition_column.q.out | 2 +- .../clientnegative/udf_add_months_error_1.q.out | 2 +- .../clientnegative/udf_last_day_error_1.q.out | 2 +- .../clientnegative/udf_next_day_error_1.q.out | 2 +- .../clientnegative/udf_next_day_error_2.q.out | 2 +- .../alter_change_db_location.q.out | 35 + .../clientpositive/annotate_stats_filter.q.out | 4 +- .../clientpositive/annotate_stats_join.q.out | 32 +- .../clientpositive/annotate_stats_select.q.out | 2 +- .../results/clientpositive/auto_join32.q.out | 30 + .../clientpositive/auto_sortmerge_join_1.q.out | 3 + .../clientpositive/auto_sortmerge_join_2.q.out | 2 + .../clientpositive/auto_sortmerge_join_3.q.out | 3 + .../clientpositive/auto_sortmerge_join_4.q.out | 3 + .../clientpositive/auto_sortmerge_join_5.q.out | 3 + .../clientpositive/auto_sortmerge_join_7.q.out | 3 + .../clientpositive/auto_sortmerge_join_8.q.out | 3 + .../results/clientpositive/bucket_many.q.out | 230 + .../clientpositive/bucketcontext_1.q.out | 1 + .../clientpositive/bucketcontext_2.q.out | 1 + .../clientpositive/bucketcontext_3.q.out | 1 + .../clientpositive/bucketcontext_4.q.out | 1 + .../clientpositive/bucketcontext_5.q.out | 1 + .../clientpositive/bucketcontext_6.q.out | 1 + .../clientpositive/bucketcontext_7.q.out | 1 + .../clientpositive/bucketcontext_8.q.out | 1 + .../results/clientpositive/bucketmapjoin1.q.out | 88 + .../test/results/clientpositive/cbo_join.q.out | 350 +- .../test/results/clientpositive/cbo_limit.q.out | 4 +- .../clientpositive/cbo_rp_auto_join0.q.out | 258 + .../clientpositive/cbo_rp_auto_join1.q.out | 1512 ++ .../clientpositive/cbo_simple_select.q.out | 10 + .../results/clientpositive/cbo_subq_in.q.out | 38 +- .../clientpositive/cbo_subq_not_in.q.out | 32 +- .../results/clientpositive/cbo_udf_udaf.q.out | 34 +- .../clientpositive/correlationoptimizer12.q.out | 8 +- .../clientpositive/correlationoptimizer3.q.out | 60 +- .../clientpositive/correlationoptimizer6.q.out | 32 +- .../results/clientpositive/ctas_colname.q.out | 8 +- .../clientpositive/decimal_precision2.q.out | 163 + .../results/clientpositive/decimal_udf.q.out | 2 +- .../encryption_insert_partition_static.q.out | 14 +- .../encrypted/encryption_move_tbl.q.out | 43 + .../test/results/clientpositive/fold_case.q.out | 301 + .../clientpositive/fold_eq_with_case_when.q.out | 231 + .../test/results/clientpositive/fold_when.q.out | 480 + .../groupby_grouping_window.q.out | 4 +- .../clientpositive/groupby_resolution.q.out | 4 +- .../results/clientpositive/groupby_sort_6.q.out | 54 + .../test/results/clientpositive/input23.q.out | 22 +- .../test/results/clientpositive/input26.q.out | 18 + ql/src/test/results/clientpositive/input6.q.out | 2 +- ql/src/test/results/clientpositive/input8.q.out | 2 +- ql/src/test/results/clientpositive/input9.q.out | 8 +- .../insert_overwrite_directory.q.out | 1813 +++ .../join_cond_pushdown_unqual2.q.out | 24 +- .../join_cond_pushdown_unqual4.q.out | 24 +- .../results/clientpositive/join_nullsafe.q.out | 10 +- .../clientpositive/join_on_varchar.q.out | 146 + .../test/results/clientpositive/join_view.q.out | 25 + .../json_serde_qualified_types.q.out | 33 + .../results/clientpositive/limit_pushdown.q.out | 88 + .../clientpositive/literal_decimal.q.out | 6 +- .../clientpositive/load_dyn_part14.q.out | 24 +- .../test/results/clientpositive/mergejoin.q.out | 3157 ++++ .../results/clientpositive/metadataonly1.q.out | 20 + .../results/clientpositive/nullgroup5.q.out | 19 + .../clientpositive/num_op_type_conv.q.out | 6 +- .../clientpositive/optimize_nullscan.q.out | 269 +- .../clientpositive/orc_int_type_promotion.q.out | 377 + .../clientpositive/partition_boolexpr.q.out | 32 + .../partition_coltype_literals.q.out | 647 + .../clientpositive/partition_timestamp.q.out | 116 +- .../clientpositive/partition_timestamp2.q.out | 250 +- .../clientpositive/ppd_constant_expr.q.out | 4 +- .../results/clientpositive/ppd_union_view.q.out | 80 + ql/src/test/results/clientpositive/ptf.q.out | 160 +- .../results/clientpositive/ptf_streaming.q.out | 84 +- .../ql_rewrite_gbtoidx_cbo_2.q.out | 14 +- .../results/clientpositive/quotedid_basic.q.out | 8 +- .../clientpositive/reduce_deduplicate.q.out | 34 + .../results/clientpositive/repl_1_drop.q.out | 345 + .../clientpositive/repl_2_exim_basic.q.out | 494 + .../clientpositive/repl_3_exim_metadata.q.out | 222 + .../runtime_skewjoin_mapjoin_spark.q.out | 669 + .../test/results/clientpositive/sample6.q.out | 20 + .../results/clientpositive/show_functions.q.out | 10 + .../results/clientpositive/smb_mapjoin9.q.out | 82 + .../results/clientpositive/smb_mapjoin_11.q.out | 1 + .../results/clientpositive/smb_mapjoin_12.q.out | 2 + .../results/clientpositive/smb_mapjoin_13.q.out | 1 + .../results/clientpositive/smb_mapjoin_15.q.out | 3 + .../clientpositive/sort_merge_join_desc_5.q.out | 1 + .../clientpositive/spark/auto_join0.q.out | 2 +- .../clientpositive/spark/auto_join15.q.out | 2 +- .../clientpositive/spark/auto_join20.q.out | 4 +- .../clientpositive/spark/auto_join21.q.out | 2 +- .../clientpositive/spark/auto_join23.q.out | 2 +- .../clientpositive/spark/auto_join28.q.out | 8 +- .../clientpositive/spark/auto_join29.q.out | 18 +- .../clientpositive/spark/auto_join30.q.out | 42 +- .../clientpositive/spark/auto_join31.q.out | 6 +- .../clientpositive/spark/auto_join32.q.out | 26 + .../clientpositive/spark/auto_join8.q.out | 2 +- .../spark/auto_sortmerge_join_1.q.out | 2 + .../spark/auto_sortmerge_join_2.q.out | 1 + .../spark/auto_sortmerge_join_3.q.out | 2 + .../spark/auto_sortmerge_join_4.q.out | 2 + .../spark/auto_sortmerge_join_5.q.out | 2 + .../spark/auto_sortmerge_join_7.q.out | 2 + .../spark/auto_sortmerge_join_8.q.out | 2 + .../results/clientpositive/spark/bucket5.q.out | 2 +- .../clientpositive/spark/bucketmapjoin1.q.out | 120 + .../results/clientpositive/spark/cbo_gby.q.out | 120 + .../clientpositive/spark/cbo_limit.q.out | 90 + .../clientpositive/spark/cbo_semijoin.q.out | 440 + .../spark/cbo_simple_select.q.out | 755 + .../clientpositive/spark/cbo_stats.q.out | 14 + .../clientpositive/spark/cbo_subq_in.q.out | 149 + .../clientpositive/spark/cbo_subq_not_in.q.out | 365 + .../clientpositive/spark/cbo_udf_udaf.q.out | 121 + .../clientpositive/spark/cbo_union.q.out | 920 ++ .../results/clientpositive/spark/ctas.q.out | 20 +- .../spark/escape_clusterby1.q.out | 4 +- .../clientpositive/spark/escape_sortby1.q.out | 4 +- .../clientpositive/spark/groupby10.q.out | 2 +- .../groupby7_map_multi_single_reducer.q.out | 2 +- .../groupby7_noskew_multi_single_reducer.q.out | 4 +- .../clientpositive/spark/groupby8_map.q.out | 2 +- .../clientpositive/spark/groupby8_noskew.q.out | 2 +- ...pby_complex_types_multi_single_reducer.q.out | 38 +- .../spark/groupby_multi_single_reducer3.q.out | 8 +- .../spark/identity_project_remove_skip.q.out | 4 +- .../results/clientpositive/spark/input14.q.out | 2 +- .../results/clientpositive/spark/input17.q.out | 2 +- .../results/clientpositive/spark/input18.q.out | 2 +- .../clientpositive/spark/join0.q.java1.7.out | 2 +- .../clientpositive/spark/join0.q.java1.8.out | 238 + .../results/clientpositive/spark/join15.q.out | 2 +- .../results/clientpositive/spark/join20.q.out | 4 +- .../results/clientpositive/spark/join21.q.out | 2 +- .../results/clientpositive/spark/join23.q.out | 2 +- .../results/clientpositive/spark/join40.q.out | 4 +- .../results/clientpositive/spark/join8.q.out | 2 +- .../clientpositive/spark/join_nullsafe.q.out | 10 +- .../clientpositive/spark/join_view.q.out | 26 + .../spark/lateral_view_explode2.q.out | 4 +- .../clientpositive/spark/limit_pushdown.q.out | 94 + .../spark/list_bucket_dml_2.q.java1.8.out | 665 + .../clientpositive/spark/load_dyn_part14.q.out | 8 +- .../clientpositive/spark/mapjoin_decimal.q.out | 8 +- .../spark/mapjoin_filter_on_outerjoin.q.out | 4 +- .../spark/mapjoin_test_outer.q.out | 4 +- .../clientpositive/spark/multi_insert.q.out | 8 +- .../clientpositive/spark/multi_insert_gby.q.out | 4 +- .../spark/multi_insert_gby3.q.out | 6 +- .../spark/multi_insert_lateral_view.q.out | 2 +- ...i_insert_move_tasks_share_dependencies.q.out | 72 +- .../spark/multigroupby_singlemr.q.out | 2 +- .../spark/optimize_nullscan.q.out | 192 + .../results/clientpositive/spark/parallel.q.out | 2 +- .../clientpositive/spark/parallel_join0.q.out | 2 +- .../clientpositive/spark/ppd_join4.q.out | 2 +- .../clientpositive/spark/ppd_transform.q.out | 4 +- .../test/results/clientpositive/spark/ptf.q.out | 194 +- .../clientpositive/spark/ptf_streaming.q.out | 100 +- .../spark/reduce_deduplicate.q.out | 36 +- .../spark/reduce_deduplicate_exclude_join.q.out | 2 +- .../spark/runtime_skewjoin_mapjoin_spark.q.out | 314 + .../results/clientpositive/spark/sample6.q.out | 20 + .../results/clientpositive/spark/semijoin.q.out | 40 +- .../clientpositive/spark/smb_mapjoin_11.q.out | 1 + .../clientpositive/spark/smb_mapjoin_12.q.out | 2 + .../clientpositive/spark/smb_mapjoin_13.q.out | 1 + .../clientpositive/spark/smb_mapjoin_15.q.out | 3 + .../results/clientpositive/spark/sort.q.out | 2 +- .../spark/sort_merge_join_desc_5.q.out | 43 +- .../clientpositive/spark/subquery_in.q.out | 8 +- .../spark/subquery_multiinsert.q.java1.8.out | 890 ++ .../clientpositive/spark/transform_ppr1.q.out | 2 +- .../clientpositive/spark/transform_ppr2.q.out | 2 +- .../results/clientpositive/spark/union12.q.out | 200 + .../results/clientpositive/spark/union17.q.out | 855 + .../results/clientpositive/spark/union20.q.out | 200 + .../results/clientpositive/spark/union21.q.out | 724 + .../results/clientpositive/spark/union22.q.out | 1602 ++ .../results/clientpositive/spark/union24.q.out | 1649 ++ .../results/clientpositive/spark/union26.q.out | 1249 ++ .../results/clientpositive/spark/union27.q.out | 136 + .../results/clientpositive/spark/union3.q.out | 2 +- .../results/clientpositive/spark/union31.q.out | 815 + .../results/clientpositive/spark/union32.q.out | 669 + .../results/clientpositive/spark/union34.q.out | 406 + .../clientpositive/spark/union_date.q.out | 142 + .../clientpositive/spark/union_date_trim.q.out | 54 + .../spark/union_lateralview.q.out | 251 + .../clientpositive/spark/union_ppr.q.out | 2 +- .../clientpositive/spark/union_remove_12.q.out | 281 + .../clientpositive/spark/union_remove_13.q.out | 306 + .../clientpositive/spark/union_remove_14.q.out | 283 + .../clientpositive/spark/union_remove_22.q.out | 401 + .../clientpositive/spark/union_remove_23.q.out | 265 + .../clientpositive/spark/union_remove_25.q.out | 2 +- .../spark/union_remove_6_subq.q.out | 1181 ++ .../clientpositive/spark/union_script.q.out | 1524 ++ .../clientpositive/spark/union_top_level.q.out | 1012 ++ .../clientpositive/spark/union_view.q.out | 264 + .../spark/vector_between_in.q.out | 6088 +------- .../spark/vector_cast_constant.q.java1.7.out | 16 +- .../spark/vector_cast_constant.q.java1.8.out | 16 +- .../spark/vector_decimal_mapjoin.q.out | 366 +- .../clientpositive/spark/vector_elt.q.out | 2 +- .../spark/vectorization_decimal_date.q.out | 20 +- .../spark/vectorization_short_regress.q.out | 8 +- .../clientpositive/spark/vectorized_ptf.q.out | 194 +- .../spark/vectorized_timestamp_funcs.q.out | 106 +- .../results/clientpositive/subquery_in.q.out | 8 +- .../subquery_in_explain_rewrite.q.out | 4 +- .../clientpositive/subquery_in_having.q.out | 6 +- .../results/clientpositive/subquery_notin.q.out | 24 +- .../subquery_notin_having.q.java1.7.out | 2 +- .../subquery_unqualcolumnrefs.q.out | 16 +- .../tez/acid_vectorization_partition.q.out | 20 +- .../results/clientpositive/tez/auto_join0.q.out | 1 + .../results/clientpositive/tez/auto_join1.q.out | 1 + .../clientpositive/tez/auto_join29.q.out | 503 + .../clientpositive/tez/auto_join30.q.out | 4 + .../tez/auto_sortmerge_join_1.q.out | 3 + .../tez/auto_sortmerge_join_10.q.out | 3 + .../tez/auto_sortmerge_join_11.q.out | 4 + .../tez/auto_sortmerge_join_12.q.out | 3 + .../tez/auto_sortmerge_join_13.q.out | 3 + .../tez/auto_sortmerge_join_14.q.out | 2 + .../tez/auto_sortmerge_join_15.q.out | 2 + .../tez/auto_sortmerge_join_2.q.out | 2 + .../tez/auto_sortmerge_join_3.q.out | 3 + .../tez/auto_sortmerge_join_4.q.out | 3 + .../tez/auto_sortmerge_join_5.q.out | 1 + .../tez/auto_sortmerge_join_7.q.out | 3 + .../tez/auto_sortmerge_join_8.q.out | 3 + .../tez/auto_sortmerge_join_9.q.out | 29 + .../tez/bucket_map_join_tez1.q.out | 16 + .../tez/bucket_map_join_tez2.q.out | 8 + .../results/clientpositive/tez/cbo_join.q.out | 350 +- .../results/clientpositive/tez/cbo_limit.q.out | 4 +- .../clientpositive/tez/cbo_simple_select.q.out | 10 + .../clientpositive/tez/cbo_subq_in.q.out | 38 +- .../clientpositive/tez/cbo_subq_not_in.q.out | 32 +- .../clientpositive/tez/cbo_udf_udaf.q.out | 34 +- .../tez/correlationoptimizer1.q.out | 1 + .../tez/cross_product_check_2.q.out | 9 + .../tez/dynamic_partition_pruning.q.out | 51 + .../tez/dynamic_partition_pruning_2.q.out | 123 + .../clientpositive/tez/explainuser_1.q.out | 6 +- .../clientpositive/tez/explainuser_2.q.out | 3056 ++-- .../tez/hybridgrace_hashjoin_1.q.out | 1587 ++ .../tez/hybridgrace_hashjoin_2.q.out | 1417 ++ .../clientpositive/tez/hybridhashjoin.q.out | 1560 -- .../clientpositive/tez/join_nullsafe.q.out | 14 +- .../clientpositive/tez/limit_pushdown.q.out | 125 +- .../clientpositive/tez/lvj_mapjoin.q.out | 2 + .../clientpositive/tez/mapjoin_decimal.q.out | 10 +- .../clientpositive/tez/mapjoin_mapjoin.q.out | 6 + .../results/clientpositive/tez/mergejoin.q.out | 3118 ++++ .../clientpositive/tez/metadataonly1.q.out | 20 + .../test/results/clientpositive/tez/mrr.q.out | 54 +- .../clientpositive/tez/optimize_nullscan.q.out | 192 + .../test/results/clientpositive/tez/ptf.q.out | 160 +- .../clientpositive/tez/ptf_streaming.q.out | 84 +- .../clientpositive/tez/subquery_in.q.out | 8 +- .../tez/tez_bmj_schema_evolution.q.out | 1 + .../clientpositive/tez/tez_join_hash.q.out | 2 + .../clientpositive/tez/tez_smb_main.q.out | 12 + .../results/clientpositive/tez/tez_union.q.out | 79 + .../clientpositive/tez/tez_union_group_by.q.out | 67 + .../tez/tez_union_multiinsert.q.out | 4293 +++++ .../clientpositive/tez/unionDistinct_1.q.out | 10 + .../results/clientpositive/tez/union_view.q.out | 1004 ++ .../clientpositive/tez/vector_aggregate_9.q.out | 8 +- .../clientpositive/tez/vector_between_in.q.out | 6088 +------- .../tez/vector_binary_join_groupby.q.out | 303 + .../tez/vector_char_mapjoin1.q.out | 43 +- .../clientpositive/tez/vector_coalesce.q.out | 26 +- .../tez/vector_count_distinct.q.out | 28 +- .../clientpositive/tez/vector_decimal_2.q.out | 4 + .../tez/vector_decimal_mapjoin.q.out | 371 +- .../clientpositive/tez/vector_decimal_udf.q.out | 3 +- .../results/clientpositive/tez/vector_elt.q.out | 2 +- .../clientpositive/tez/vector_inner_join.q.out | 806 + .../clientpositive/tez/vector_join30.q.out | 1367 ++ .../tez/vector_join_filters.q.out | 222 + .../clientpositive/tez/vector_join_nulls.q.out | 195 + .../tez/vector_left_outer_join2.q.out | 555 + .../tez/vector_left_outer_join3.q.out | 222 + .../tez/vector_leftsemi_mapjoin.q.out | 13807 +++++++++++++++++ .../tez/vector_mapjoin_reduce.q.out | 4 + .../tez/vector_mr_diff_schema_alias.q.out | 381 + .../tez/vector_nullsafe_join.q.out | 1210 ++ .../clientpositive/tez/vector_outer_join.q.out | 2204 +++ .../clientpositive/tez/vector_outer_join0.q.out | 232 + .../clientpositive/tez/vector_outer_join1.q.out | 541 + .../clientpositive/tez/vector_outer_join2.q.out | 238 + .../clientpositive/tez/vector_outer_join3.q.out | 527 + .../clientpositive/tez/vector_outer_join4.q.out | 864 ++ .../clientpositive/tez/vector_outer_join5.q.out | 1328 ++ .../tez/vector_varchar_mapjoin1.q.out | 5 +- .../tez/vectorization_decimal_date.q.out | 20 +- .../tez/vectorization_limit.q.out | 31 +- .../tez/vectorization_short_regress.q.out | 8 +- .../clientpositive/tez/vectorized_casts.q.out | 52 +- .../tez/vectorized_distinct_gby.q.out | 51 +- .../vectorized_dynamic_partition_pruning.q.out | 51 + .../clientpositive/tez/vectorized_mapjoin.q.out | 1 + .../tez/vectorized_nested_mapjoin.q.out | 2 + .../tez/vectorized_parquet_types.q.out | 347 + .../clientpositive/tez/vectorized_ptf.q.out | 160 +- .../tez/vectorized_timestamp_funcs.q.out | 108 +- .../tez/vectorized_timestamp_ints_casts.q.out | 234 + .../clientpositive/timestamp_ints_casts.q.out | 264 + .../udaf_percentile_approx_23.q.out | 6 +- ql/src/test/results/clientpositive/udf4.q.out | 2 +- ql/src/test/results/clientpositive/udf6.q.out | 30 +- ql/src/test/results/clientpositive/udf7.q.out | 2 +- .../test/results/clientpositive/udf_case.q.out | 2 +- .../results/clientpositive/udf_coalesce.q.out | 4 +- .../test/results/clientpositive/udf_crc32.q.out | 60 + .../clientpositive/udf_date_format.q.out | 172 + .../results/clientpositive/udf_decode.q.out | 2 +- .../test/results/clientpositive/udf_elt.q.out | 4 +- .../results/clientpositive/udf_greatest.q.out | 4 +- .../test/results/clientpositive/udf_hour.q.out | 2 +- ql/src/test/results/clientpositive/udf_if.q.out | 4 +- .../test/results/clientpositive/udf_instr.q.out | 4 +- .../clientpositive/udf_isnull_isnotnull.q.out | 2 +- .../test/results/clientpositive/udf_least.q.out | 4 +- .../results/clientpositive/udf_locate.q.out | 4 +- .../test/results/clientpositive/udf_md5.q.out | 61 + .../results/clientpositive/udf_minute.q.out | 2 +- .../test/results/clientpositive/udf_nvl.q.out | 2 +- .../results/clientpositive/udf_parse_url.q.out | 2 +- .../results/clientpositive/udf_quarter.q.out | 246 + .../results/clientpositive/udf_second.q.out | 2 +- .../test/results/clientpositive/udf_sha1.q.out | 61 + .../test/results/clientpositive/udf_sha2.q.out | 134 + .../test/results/clientpositive/udf_size.q.out | 2 +- .../test/results/clientpositive/udf_trunc.q.out | 64 +- .../test/results/clientpositive/udf_when.q.out | 4 +- .../test/results/clientpositive/union12.q.out | 8 +- .../test/results/clientpositive/union17.q.out | 4 +- .../test/results/clientpositive/union20.q.out | 22 +- .../test/results/clientpositive/union21.q.out | 6 +- .../test/results/clientpositive/union27.q.out | 4 +- .../test/results/clientpositive/union30.q.out | 42 +- .../clientpositive/union_lateralview.q.out | 16 +- .../clientpositive/union_remove_22.q.out | 16 +- .../clientpositive/union_remove_6_subq.q.out | 12 +- .../results/clientpositive/union_view.q.out | 368 + .../clientpositive/vector_aggregate_9.q.out | 8 +- .../clientpositive/vector_between_in.q.out | 6088 +------- .../vector_binary_join_groupby.q.out | 293 + .../clientpositive/vector_char_mapjoin1.q.out | 36 +- .../clientpositive/vector_coalesce.q.out | 6 +- .../clientpositive/vector_decimal_2.q.out | 4 + .../clientpositive/vector_decimal_mapjoin.q.out | 370 +- .../clientpositive/vector_decimal_udf.q.out | 3 +- .../results/clientpositive/vector_elt.q.out | 2 +- .../clientpositive/vector_inner_join.q.out | 799 + .../results/clientpositive/vector_join30.q.out | 2194 +++ .../clientpositive/vector_join_filters.q.out | 222 + .../clientpositive/vector_join_nulls.q.out | 195 + .../vector_left_outer_join2.q.out | 572 + .../vector_leftsemi_mapjoin.q.out | 13572 ++++++++++++++++ .../vector_mr_diff_schema_alias.q.out | 396 + .../clientpositive/vector_nullsafe_join.q.out | 1208 ++ .../clientpositive/vector_outer_join0.q.out | 230 + .../clientpositive/vector_outer_join1.q.out | 534 + .../clientpositive/vector_outer_join2.q.out | 232 + .../clientpositive/vector_outer_join3.q.out | 509 + .../clientpositive/vector_outer_join4.q.out | 857 + .../clientpositive/vector_outer_join5.q.out | 1300 ++ .../vector_varchar_mapjoin1.q.out | 4 +- .../vectorization_decimal_date.q.out | 20 +- .../vectorization_short_regress.q.out | 8 +- .../clientpositive/vectorized_casts.q.out | 52 +- .../results/clientpositive/vectorized_ptf.q.out | 167 +- .../clientpositive/vectorized_shufflejoin.q.out | 1 - .../vectorized_timestamp_funcs.q.out | 108 +- .../vectorized_timestamp_ints_casts.q.out | 266 + .../clientpositive/windowing_navfn.q.out | 10 + .../clientpositive/windowing_streaming.q.out | 16 +- .../clientpositive/windowing_windowspec2.q.out | 478 + ql/src/test/templates/TestCliDriver.vm | 18 +- ql/src/test/templates/TestCompareCliDriver.vm | 21 +- ql/src/test/templates/TestNegativeCliDriver.vm | 18 +- ql/src/test/templates/TestParseNegative.vm | 17 +- serde/pom.xml | 2 +- .../hadoop/hive/ql/io/sarg/PredicateLeaf.java | 19 +- .../apache/hadoop/hive/serde2/SerDeUtils.java | 1 + .../apache/hadoop/hive/serde2/WriteBuffers.java | 25 +- .../binarysortable/BinarySortableSerDe.java | 157 +- .../fast/BinarySortableSerializeWrite.java | 102 +- .../hadoop/hive/serde2/fast/SerializeWrite.java | 12 +- .../hive/serde2/io/HiveDecimalWritable.java | 5 +- .../hive/serde2/io/HiveVarcharWritable.java | 7 +- .../hive/serde2/io/TimestampWritable.java | 11 +- .../hive/serde2/lazy/LazySerDeParameters.java | 18 + .../lazy/fast/LazySimpleSerializeWrite.java | 13 +- .../LazyObjectInspectorFactory.java | 24 +- .../LazyPrimitiveObjectInspectorFactory.java | 41 +- .../hive/serde2/lazybinary/LazyBinaryUtils.java | 19 +- .../fast/LazyBinarySerializeWrite.java | 15 +- .../LazyBinaryObjectInspectorFactory.java | 24 +- .../MetadataListStructObjectInspector.java | 18 +- .../ObjectInspectorConverters.java | 5 +- .../objectinspector/ObjectInspectorUtils.java | 4 +- .../AbstractPrimitiveObjectInspector.java | 1 + .../PrimitiveObjectInspectorConverter.java | 7 +- .../PrimitiveObjectInspectorFactory.java | 26 +- .../PrimitiveObjectInspectorUtils.java | 42 +- .../primitive/WritableVoidObjectInspector.java | 5 + .../hive/serde2/typeinfo/CharTypeInfo.java | 5 +- .../hive/serde2/typeinfo/DecimalTypeInfo.java | 7 +- .../hive/serde2/typeinfo/PrimitiveTypeInfo.java | 5 +- .../hive/serde2/typeinfo/TypeInfoFactory.java | 31 +- .../hive/serde2/typeinfo/TypeInfoUtils.java | 27 +- .../hive/serde2/typeinfo/VarcharTypeInfo.java | 5 +- .../apache/hadoop/hive/serde2/VerifyFast.java | 10 +- .../hive/serde2/binarysortable/MyTestClass.java | 105 + .../binarysortable/MyTestPrimitiveClass.java | 20 + .../binarysortable/TestBinarySortableFast.java | 13 +- .../binarysortable/TestBinarySortableSerDe.java | 10 +- .../hive/serde2/lazy/TestLazySimpleFast.java | 22 +- .../TestPrimitiveObjectInspectorUtils.java | 95 +- service/pom.xml | 7 +- .../org/apache/hive/service/CookieSigner.java | 108 + .../apache/hive/service/auth/HttpAuthUtils.java | 72 + .../auth/LdapAuthenticationProviderImpl.java | 2 +- .../apache/hive/service/cli/ColumnValue.java | 2 + .../service/cli/session/HiveSessionImpl.java | 14 +- .../service/cli/session/SessionManager.java | 49 +- .../thrift/EmbeddedThriftBinaryCLIService.java | 5 + .../cli/thrift/ThriftBinaryCLIService.java | 1 + .../service/cli/thrift/ThriftCLIService.java | 79 +- .../service/cli/thrift/ThriftHttpServlet.java | 210 +- .../apache/hive/service/server/HiveServer2.java | 138 +- .../apache/hive/service/TestCookieSigner.java | 59 + .../TestLdapAuthenticationProviderImpl.java | 18 + .../service/cli/session/TestSessionHooks.java | 3 +- .../thrift/ThriftCliServiceTestWithCookie.java | 221 + shims/0.20S/pom.xml | 2 +- .../hadoop/hive/shims/Hadoop20SShims.java | 35 + shims/0.23/pom.xml | 2 +- .../apache/hadoop/hive/shims/Hadoop23Shims.java | 50 +- shims/aggregator/pom.xml | 2 +- shims/common/pom.xml | 2 +- .../apache/hadoop/hive/shims/HadoopShims.java | 22 + .../hadoop/hive/shims/HadoopShimsSecure.java | 4 + shims/pom.xml | 2 +- shims/scheduler/pom.xml | 2 +- spark-client/pom.xml | 4 +- .../apache/hive/spark/client/JobContext.java | 6 + .../hive/spark/client/JobContextImpl.java | 10 +- .../apache/hive/spark/client/RemoteDriver.java | 34 +- .../hive/spark/client/SparkClientImpl.java | 34 +- .../hive/spark/client/SparkClientUtilities.java | 23 +- .../apache/hive/spark/client/rpc/RpcServer.java | 11 +- .../apache/hive/spark/client/rpc/TestRpc.java | 32 +- testutils/metastore/dbs/derby/execute.sh | 37 + testutils/metastore/dbs/derby/prepare.sh | 63 + testutils/metastore/dbs/postgres/execute.sh | 29 + testutils/metastore/dbs/postgres/prepare.sh | 72 + testutils/metastore/execute-test-on-lxc.sh | 20 +- testutils/metastore/metastore-upgrade-test.sh | 5 +- testutils/pom.xml | 2 +- .../org/apache/hive/ptest/execution/PTest.java | 12 +- .../ptest/execution/conf/TestConfiguration.java | 12 +- .../ptest2/src/main/resources/source-prep.vm | 11 + .../hive/ptest/execution/TestScripts.java | 23 + .../TestScripts.testPrepGit.approved.txt | 2 +- .../TestScripts.testPrepHadoop1.approved.txt | 111 + .../TestScripts.testPrepNone.approved.txt | 2 +- .../TestScripts.testPrepSvn.approved.txt | 2 +- .../execution/conf/TestTestConfiguration.java | 38 +- .../resources/test-configuration.properties | 2 + 1226 files changed, 154032 insertions(+), 31056 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hive/blob/5a0ae3c2/ql/src/java/org/apache/hadoop/hive/ql/io/parquet/MapredParquetInputFormat.java ---------------------------------------------------------------------- diff --cc ql/src/java/org/apache/hadoop/hive/ql/io/parquet/MapredParquetInputFormat.java index 000eb38,f4f0f07..f7adb39 --- a/ql/src/java/org/apache/hadoop/hive/ql/io/parquet/MapredParquetInputFormat.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/io/parquet/MapredParquetInputFormat.java @@@ -20,7 -20,8 +20,8 @@@ import org.apache.hadoop.hive.ql.exec.U import org.apache.hadoop.hive.ql.exec.vector.VectorizedInputFormatInterface; import org.apache.hadoop.hive.ql.io.parquet.read.DataWritableReadSupport; import org.apache.hadoop.hive.ql.io.parquet.read.ParquetRecordReaderWrapper; -import org.apache.hadoop.io.ArrayWritable; +import org.apache.hadoop.hive.serde2.io.ObjectArrayWritable; + import org.apache.hadoop.io.NullWritable; import org.apache.hadoop.mapred.FileInputFormat; import org.apache.hadoop.mapred.RecordReader; @@@ -34,7 -35,7 +35,7 @@@ import parquet.hadoop.ParquetInputForma * NOTE: With HIVE-9235 we removed "implements VectorizedParquetInputFormat" since all data types * are not currently supported. Removing the interface turns off vectorization. */ - public class MapredParquetInputFormat extends FileInputFormat<Void, ObjectArrayWritable> { -public class MapredParquetInputFormat extends FileInputFormat<NullWritable, ArrayWritable> { ++public class MapredParquetInputFormat extends FileInputFormat<NullWritable, ObjectArrayWritable> { private static final Log LOG = LogFactory.getLog(MapredParquetInputFormat.class); @@@ -53,7 -54,7 +54,7 @@@ @SuppressWarnings({ "unchecked", "rawtypes" }) @Override - public org.apache.hadoop.mapred.RecordReader<Void, ObjectArrayWritable> getRecordReader( - public org.apache.hadoop.mapred.RecordReader<NullWritable, ArrayWritable> getRecordReader( ++ public org.apache.hadoop.mapred.RecordReader<NullWritable, ObjectArrayWritable> getRecordReader( final org.apache.hadoop.mapred.InputSplit split, final org.apache.hadoop.mapred.JobConf job, final org.apache.hadoop.mapred.Reporter reporter @@@ -69,7 -70,7 +70,7 @@@ if (LOG.isDebugEnabled()) { LOG.debug("Using row-mode record reader"); } - return (RecordReader<Void, ObjectArrayWritable>) - return (RecordReader<NullWritable, ArrayWritable>) ++ return (RecordReader<NullWritable, ObjectArrayWritable>) new ParquetRecordReaderWrapper(realInput, split, job, reporter); } } catch (final InterruptedException e) { http://git-wip-us.apache.org/repos/asf/hive/blob/5a0ae3c2/ql/src/java/org/apache/hadoop/hive/ql/io/parquet/VectorizedParquetInputFormat.java ---------------------------------------------------------------------- diff --cc ql/src/java/org/apache/hadoop/hive/ql/io/parquet/VectorizedParquetInputFormat.java index 4e1820c,843e079..c557963 --- a/ql/src/java/org/apache/hadoop/hive/ql/io/parquet/VectorizedParquetInputFormat.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/io/parquet/VectorizedParquetInputFormat.java @@@ -51,8 -52,8 +51,8 @@@ public class VectorizedParquetInputForm private final ParquetRecordReaderWrapper internalReader; private VectorizedRowBatchCtx rbCtx; - private ArrayWritable internalValues; + private ObjectArrayWritable internalValues; - private Void internalKey; + private NullWritable internalKey; private VectorColumnAssign[] assigners; public VectorizedParquetRecordReader( http://git-wip-us.apache.org/repos/asf/hive/blob/5a0ae3c2/ql/src/java/org/apache/hadoop/hive/ql/io/parquet/convert/HiveCollectionConverter.java ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hive/blob/5a0ae3c2/ql/src/java/org/apache/hadoop/hive/ql/io/parquet/convert/Repeated.java ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hive/blob/5a0ae3c2/ql/src/java/org/apache/hadoop/hive/ql/io/parquet/read/ParquetRecordReaderWrapper.java ---------------------------------------------------------------------- diff --cc ql/src/java/org/apache/hadoop/hive/ql/io/parquet/read/ParquetRecordReaderWrapper.java index 0a5edbb,5c36564..f513572 --- a/ql/src/java/org/apache/hadoop/hive/ql/io/parquet/read/ParquetRecordReaderWrapper.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/io/parquet/read/ParquetRecordReaderWrapper.java @@@ -33,7 -33,9 +33,8 @@@ import org.apache.hadoop.hive.ql.io.par import org.apache.hadoop.hive.ql.io.sarg.SearchArgumentFactory; import org.apache.hadoop.hive.ql.plan.TableScanDesc; import org.apache.hadoop.hive.serde2.ColumnProjectionUtils; -import org.apache.hadoop.io.ArrayWritable; +import org.apache.hadoop.hive.serde2.io.ObjectArrayWritable; + import org.apache.hadoop.io.NullWritable; -import org.apache.hadoop.io.Writable; import org.apache.hadoop.mapred.FileSplit; import org.apache.hadoop.mapred.InputSplit; import org.apache.hadoop.mapred.JobConf; @@@ -58,7 -60,7 +59,7 @@@ import parquet.schema.MessageTypeParser import com.google.common.base.Strings; - public class ParquetRecordReaderWrapper implements RecordReader<Void, ObjectArrayWritable> { -public class ParquetRecordReaderWrapper implements RecordReader<NullWritable, ArrayWritable> { ++public class ParquetRecordReaderWrapper implements RecordReader<NullWritable, ObjectArrayWritable> { public static final Log LOG = LogFactory.getLog(ParquetRecordReaderWrapper.class); private final long splitLen; // for getPos() @@@ -194,7 -197,7 +195,7 @@@ } @Override - public boolean next(final Void key, final ObjectArrayWritable value) throws IOException { - public boolean next(final NullWritable key, final ArrayWritable value) throws IOException { ++ public boolean next(final NullWritable key, final ObjectArrayWritable value) throws IOException { if (eof) { return false; } http://git-wip-us.apache.org/repos/asf/hive/blob/5a0ae3c2/ql/src/test/org/apache/hadoop/hive/ql/io/parquet/AbstractTestParquetDirect.java ---------------------------------------------------------------------- diff --cc ql/src/test/org/apache/hadoop/hive/ql/io/parquet/AbstractTestParquetDirect.java index 5f7f597,94a780d..9c4cf5c --- a/ql/src/test/org/apache/hadoop/hive/ql/io/parquet/AbstractTestParquetDirect.java +++ b/ql/src/test/org/apache/hadoop/hive/ql/io/parquet/AbstractTestParquetDirect.java @@@ -14,7 -32,8 +32,8 @@@ import org.apache.hadoop.fs.FileSystem import org.apache.hadoop.fs.Path; import org.apache.hadoop.hive.ql.io.parquet.serde.ParquetHiveSerDe; import org.apache.hadoop.hive.serde.serdeConstants; -import org.apache.hadoop.io.ArrayWritable; +import org.apache.hadoop.hive.serde2.io.ObjectArrayWritable; + import org.apache.hadoop.io.NullWritable; import org.apache.hadoop.io.Writable; import org.apache.hadoop.mapred.FileSplit; import org.apache.hadoop.mapred.JobConf; @@@ -119,16 -138,16 +138,16 @@@ public abstract class AbstractTestParqu Assert.assertEquals(message, toString(expected), toString(actual)); } - public static List<ArrayWritable> read(Path parquetFile) throws IOException { - List<ArrayWritable> records = new ArrayList<ArrayWritable>(); + public static List<ObjectArrayWritable> read(Path parquetFile) throws IOException { + List<ObjectArrayWritable> records = new ArrayList<ObjectArrayWritable>(); - RecordReader<Void, ObjectArrayWritable> reader = new MapredParquetInputFormat(). - RecordReader<NullWritable, ArrayWritable> reader = new MapredParquetInputFormat(). ++ RecordReader<NullWritable, ObjectArrayWritable> reader = new MapredParquetInputFormat(). getRecordReader(new FileSplit( parquetFile, 0, fileLength(parquetFile), (String[]) null), new JobConf(), null); - Void alwaysNull = reader.createKey(); + NullWritable alwaysNull = reader.createKey(); - ArrayWritable record = reader.createValue(); + ObjectArrayWritable record = reader.createValue(); while (reader.next(alwaysNull, record)) { records.add(record); record = reader.createValue(); // a new value so the last isn't clobbered http://git-wip-us.apache.org/repos/asf/hive/blob/5a0ae3c2/ql/src/test/org/apache/hadoop/hive/ql/io/parquet/TestArrayCompatibility.java ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hive/blob/5a0ae3c2/ql/src/test/org/apache/hadoop/hive/ql/io/parquet/TestMapStructures.java ----------------------------------------------------------------------
