HIVE-11474 : LLAP: merge master into branch (Sergey Shelukhin)
Project: http://git-wip-us.apache.org/repos/asf/hive/repo Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/c8ae1fbf Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/c8ae1fbf Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/c8ae1fbf Branch: refs/heads/llap Commit: c8ae1fbf1047b29c47c589527829aee378fa8a81 Parents: ef45451 b91bf07 Author: Sergey Shelukhin <ser...@apache.org> Authored: Wed Aug 5 17:49:21 2015 -0700 Committer: Sergey Shelukhin <ser...@apache.org> Committed: Wed Aug 5 17:49:21 2015 -0700 ---------------------------------------------------------------------- .../apache/hadoop/hive/ant/GenVectorCode.java | 103 + .../java/org/apache/hive/beeline/BeeLine.java | 7 +- .../org/apache/hive/beeline/BeeLineOpts.java | 9 + .../apache/hive/beeline/DatabaseConnection.java | 9 + beeline/src/main/resources/BeeLine.properties | 2 + .../hadoop/hive/cli/TestOptionsProcessor.java | 1 - common/pom.xml | 5 + .../apache/hadoop/hive/common/FileUtils.java | 9 +- .../hadoop/hive/common/JvmPauseMonitor.java | 7 +- .../hadoop/hive/common/ValidReadTxnList.java | 2 +- .../hive/common/metrics/LegacyMetrics.java | 30 +- .../hive/common/metrics/common/Metrics.java | 27 + .../common/metrics/common/MetricsConstant.java | 35 + .../common/metrics/common/MetricsVariable.java | 26 + .../metrics/metrics2/CodahaleMetrics.java | 58 +- .../hadoop/hive/common/type/HiveDecimal.java | 306 - .../org/apache/hadoop/hive/conf/HiveConf.java | 18 +- .../metrics/metrics2/TestCodahaleMetrics.java | 42 + .../results/positive/external_table_ppd.q.out | 1 - .../positive/hbase_binary_storage_queries.q.out | 2 - .../src/test/results/positive/hbase_stats.q.out | 7 - .../test/results/positive/hbase_stats2.q.out | 7 - .../test/results/positive/hbase_stats3.q.out | 12 - .../positive/hbase_stats_empty_partition.q.out | 2 - .../SemanticAnalysis/HCatSemanticAnalyzer.java | 7 +- .../mapreduce/DefaultOutputFormatContainer.java | 7 +- ...namicPartitionFileRecordWriterContainer.java | 3 +- .../mapreduce/FileOutputFormatContainer.java | 3 +- .../hive/hcatalog/mapreduce/PartInfo.java | 32 +- .../hive/hcatalog/mapreduce/SpecialCases.java | 8 +- .../mapreduce/TestHCatMultiOutputFormat.java | 6 +- .../streaming/AbstractRecordWriter.java | 4 +- .../streaming/mutate/client/MutatorClient.java | 11 +- .../streaming/mutate/client/lock/Lock.java | 73 +- .../hive/hcatalog/streaming/mutate/package.html | 39 +- .../mutate/worker/CreatePartitionHelper.java | 83 - .../mutate/worker/MetaStorePartitionHelper.java | 102 + .../mutate/worker/MutatorCoordinator.java | 21 +- .../worker/MutatorCoordinatorBuilder.java | 41 +- .../streaming/mutate/worker/MutatorImpl.java | 4 +- .../mutate/worker/PartitionHelper.java | 17 + .../mutate/worker/WarehousePartitionHelper.java | 69 + .../streaming/mutate/client/lock/TestLock.java | 121 +- .../worker/TestMetaStorePartitionHelper.java | 112 + .../mutate/worker/TestMutatorCoordinator.java | 40 +- .../worker/TestWarehousePartitionHelper.java | 57 + .../hive/hcatalog/api/HCatClientHMSImpl.java | 14 +- .../hive/hcatalog/templeton/AppConfig.java | 28 +- .../hcatalog/templeton/SecureProxySupport.java | 6 +- hplsql/pom.xml | 9 +- .../antlr4/org/apache/hive/hplsql/Hplsql.g4 | 202 +- .../main/java/org/apache/hive/hplsql/Conn.java | 16 +- .../java/org/apache/hive/hplsql/Converter.java | 41 +- .../main/java/org/apache/hive/hplsql/Exec.java | 171 +- .../java/org/apache/hive/hplsql/Expression.java | 73 +- .../main/java/org/apache/hive/hplsql/Meta.java | 98 + .../main/java/org/apache/hive/hplsql/Query.java | 71 + .../java/org/apache/hive/hplsql/Select.java | 47 +- .../java/org/apache/hive/hplsql/Signal.java | 2 +- .../main/java/org/apache/hive/hplsql/Stmt.java | 166 +- .../main/java/org/apache/hive/hplsql/Utils.java | 7 + .../main/java/org/apache/hive/hplsql/Var.java | 50 +- .../apache/hive/hplsql/functions/Function.java | 61 +- .../hive/hplsql/functions/FunctionDatetime.java | 14 +- .../hive/hplsql/functions/FunctionMisc.java | 22 +- .../hive/hplsql/functions/FunctionOra.java | 31 +- .../hive/hplsql/functions/FunctionString.java | 46 +- hplsql/src/main/resources/hplsql-site.xml | 95 + .../org/apache/hive/hplsql/TestHplsqlLocal.java | 311 + .../apache/hive/hplsql/TestHplsqlOffline.java | 76 + .../test/queries/db/create_procedure_mssql.sql | 52 + .../db/create_procedure_return_cursor.sql | 53 + .../db/create_procedure_return_cursor2.sql | 59 + .../src/test/queries/db/cursor_attributes.sql | 60 + hplsql/src/test/queries/db/map_object.sql | 9 + hplsql/src/test/queries/db/select_into.sql | 17 + .../src/test/queries/db/set_current_schema.sql | 6 + hplsql/src/test/queries/db/sys_refcursor.sql | 65 + hplsql/src/test/queries/db/use.sql | 2 + hplsql/src/test/queries/local/add.sql | 2 + hplsql/src/test/queries/local/assign.sql | 7 + hplsql/src/test/queries/local/bool_expr.sql | 47 + hplsql/src/test/queries/local/break.sql | 10 + hplsql/src/test/queries/local/case.sql | 35 + hplsql/src/test/queries/local/cast.sql | 4 + hplsql/src/test/queries/local/char.sql | 1 + hplsql/src/test/queries/local/coalesce.sql | 4 + hplsql/src/test/queries/local/concat.sql | 2 + .../src/test/queries/local/create_function.sql | 11 + .../src/test/queries/local/create_function2.sql | 11 + .../src/test/queries/local/create_procedure.sql | 9 + hplsql/src/test/queries/local/date.sql | 5 + hplsql/src/test/queries/local/dbms_output.sql | 6 + hplsql/src/test/queries/local/declare.sql | 16 + .../test/queries/local/declare_condition.sql | 8 + .../test/queries/local/declare_condition2.sql | 10 + hplsql/src/test/queries/local/decode.sql | 10 + hplsql/src/test/queries/local/equal.sql | 55 + hplsql/src/test/queries/local/exception.sql | 14 + .../queries/local/exception_divide_by_zero.sql | 11 + hplsql/src/test/queries/local/exit.sql | 31 + hplsql/src/test/queries/local/expr.sql | 21 + hplsql/src/test/queries/local/for_range.sql | 20 + hplsql/src/test/queries/local/if.sql | 68 + hplsql/src/test/queries/local/instr.sql | 49 + hplsql/src/test/queries/local/interval.sql | 15 + hplsql/src/test/queries/local/lang.sql | 57 + hplsql/src/test/queries/local/leave.sql | 33 + hplsql/src/test/queries/local/len.sql | 1 + hplsql/src/test/queries/local/length.sql | 1 + hplsql/src/test/queries/local/lower.sql | 1 + hplsql/src/test/queries/local/nvl.sql | 4 + hplsql/src/test/queries/local/nvl2.sql | 2 + hplsql/src/test/queries/local/print.sql | 5 + hplsql/src/test/queries/local/return.sql | 3 + hplsql/src/test/queries/local/seterror.sql | 10 + hplsql/src/test/queries/local/sub.sql | 1 + hplsql/src/test/queries/local/substr.sql | 2 + hplsql/src/test/queries/local/substring.sql | 8 + hplsql/src/test/queries/local/timestamp.sql | 4 + hplsql/src/test/queries/local/timestamp_iso.sql | 2 + hplsql/src/test/queries/local/to_char.sql | 1 + hplsql/src/test/queries/local/to_timestamp.sql | 5 + hplsql/src/test/queries/local/trim.sql | 1 + hplsql/src/test/queries/local/twopipes.sql | 1 + hplsql/src/test/queries/local/upper.sql | 1 + hplsql/src/test/queries/local/values_into.sql | 6 + hplsql/src/test/queries/local/while.sql | 20 + .../test/queries/offline/create_table_mssql.sql | 43 + .../test/queries/offline/create_table_ora.sql | 4 + .../results/db/create_procedure_mssql.out.txt | 45 + .../db/create_procedure_return_cursor.out.txt | 135 + .../db/create_procedure_return_cursor2.out.txt | 139 + .../test/results/db/cursor_attributes.out.txt | 33 + hplsql/src/test/results/db/map_object.out.txt | 17 + hplsql/src/test/results/db/select_into.out.txt | 19 + .../test/results/db/set_current_schema.out.txt | 12 + .../src/test/results/db/sys_refcursor.out.txt | 36 + hplsql/src/test/results/db/use.out.txt | 4 + hplsql/src/test/results/local/add.out.txt | 2 + hplsql/src/test/results/local/assign.out.txt | 8 + hplsql/src/test/results/local/bool_expr.out.txt | 32 + hplsql/src/test/results/local/break.out.txt | 29 + hplsql/src/test/results/local/case.out.txt | 12 + hplsql/src/test/results/local/cast.out.txt | 8 + hplsql/src/test/results/local/char.out.txt | 1 + hplsql/src/test/results/local/coalesce.out.txt | 4 + hplsql/src/test/results/local/concat.out.txt | 2 + .../test/results/local/create_function.out.txt | 9 + .../test/results/local/create_function2.out.txt | 10 + .../test/results/local/create_procedure.out.txt | 8 + hplsql/src/test/results/local/date.out.txt | 4 + .../src/test/results/local/dbms_output.out.txt | 3 + hplsql/src/test/results/local/declare.out.txt | 13 + .../results/local/declare_condition.out.txt | 7 + .../results/local/declare_condition2.out.txt | 12 + hplsql/src/test/results/local/decode.out.txt | 13 + hplsql/src/test/results/local/equal.out.txt | 48 + hplsql/src/test/results/local/exception.out.txt | 13 + .../src/test/results/local/exception2.out.txt | 5 + .../local/exception_divide_by_zero.out.txt | 8 + hplsql/src/test/results/local/exit.out.txt | 42 + hplsql/src/test/results/local/expr.out.txt | 29 + hplsql/src/test/results/local/for_range.out.txt | 65 + hplsql/src/test/results/local/if.out.txt | 40 + hplsql/src/test/results/local/instr.out.txt | 33 + hplsql/src/test/results/local/interval.out.txt | 11 + hplsql/src/test/results/local/lang.out.txt | 34 + hplsql/src/test/results/local/leave.out.txt | 42 + hplsql/src/test/results/local/len.out.txt | 1 + hplsql/src/test/results/local/length.out.txt | 1 + hplsql/src/test/results/local/lower.out.txt | 1 + hplsql/src/test/results/local/nvl.out.txt | 4 + hplsql/src/test/results/local/nvl2.out.txt | 2 + .../test/results/local/plhqlexception.out.txt | 6 + .../test/results/local/plhqlexception1.out.txt | 10 + .../test/results/local/plhqlexception2.out.txt | 106 + hplsql/src/test/results/local/print.out.txt | 6 + hplsql/src/test/results/local/return.out.txt | 3 + .../results/local/select_conversion.out.txt | 9 + hplsql/src/test/results/local/seterror.out.txt | 6 + hplsql/src/test/results/local/sub.out.txt | 1 + hplsql/src/test/results/local/substr.out.txt | 2 + hplsql/src/test/results/local/substring.out.txt | 8 + hplsql/src/test/results/local/timestamp.out.txt | 4 + .../test/results/local/timestamp_iso.out.txt | 2 + hplsql/src/test/results/local/to_char.out.txt | 1 + .../src/test/results/local/to_timestamp.out.txt | 4 + hplsql/src/test/results/local/trim.out.txt | 1 + hplsql/src/test/results/local/twopipes.out.txt | 1 + hplsql/src/test/results/local/upper.out.txt | 1 + .../src/test/results/local/values_into.out.txt | 11 + hplsql/src/test/results/local/while.out.txt | 72 + .../results/offline/create_table_mssql.out.txt | 24 + .../results/offline/create_table_ora.out.txt | 4 + .../hive/metastore/TestHiveMetaStore.java | 87 +- .../hive/metastore/TestMetaStoreMetrics.java | 66 +- .../org/apache/hive/jdbc/TestJdbcDriver2.java | 16 + itests/qtest-spark/pom.xml | 24 + itests/qtest/pom.xml | 2 +- .../test/resources/testconfiguration.properties | 55 +- .../org/apache/hive/jdbc/HiveConnection.java | 4 + .../apache/hive/jdbc/HivePreparedStatement.java | 2 +- .../hive/llap/io/api/impl/LlapInputFormat.java | 5 +- .../llap/io/encoded/OrcEncodedDataReader.java | 4 +- metastore/if/hive_metastore.thrift | 5 + .../gen/thrift/gen-cpp/ThriftHiveMetastore.cpp | 5586 ++- .../gen/thrift/gen-cpp/ThriftHiveMetastore.h | 7942 ++-- .../ThriftHiveMetastore_server.skeleton.cpp | 5 + .../thrift/gen-cpp/hive_metastore_constants.cpp | 2 +- .../thrift/gen-cpp/hive_metastore_constants.h | 2 +- .../gen/thrift/gen-cpp/hive_metastore_types.cpp | 6204 ++- .../gen/thrift/gen-cpp/hive_metastore_types.h | 2666 +- .../hive/metastore/api/AbortTxnRequest.java | 24 +- .../metastore/api/AddDynamicPartitions.java | 69 +- .../metastore/api/AddPartitionsRequest.java | 80 +- .../hive/metastore/api/AddPartitionsResult.java | 48 +- .../hadoop/hive/metastore/api/AggrStats.java | 54 +- .../metastore/api/AlreadyExistsException.java | 24 +- .../metastore/api/BinaryColumnStatsData.java | 40 +- .../metastore/api/BooleanColumnStatsData.java | 40 +- .../hive/metastore/api/CheckLockRequest.java | 24 +- .../hive/metastore/api/ColumnStatistics.java | 54 +- .../metastore/api/ColumnStatisticsData.java | 20 +- .../metastore/api/ColumnStatisticsDesc.java | 58 +- .../hive/metastore/api/ColumnStatisticsObj.java | 40 +- .../hive/metastore/api/CommitTxnRequest.java | 24 +- .../hive/metastore/api/CompactionRequest.java | 62 +- .../hive/metastore/api/CompactionType.java | 2 +- .../api/ConfigValSecurityException.java | 24 +- .../api/CurrentNotificationEventId.java | 24 +- .../hadoop/hive/metastore/api/Database.java | 115 +- .../apache/hadoop/hive/metastore/api/Date.java | 24 +- .../hive/metastore/api/DateColumnStatsData.java | 50 +- .../hadoop/hive/metastore/api/Decimal.java | 41 +- .../metastore/api/DecimalColumnStatsData.java | 50 +- .../metastore/api/DoubleColumnStatsData.java | 50 +- .../hive/metastore/api/DropPartitionsExpr.java | 43 +- .../metastore/api/DropPartitionsRequest.java | 82 +- .../metastore/api/DropPartitionsResult.java | 48 +- .../hive/metastore/api/EnvironmentContext.java | 61 +- .../hive/metastore/api/EventRequestType.java | 2 +- .../hadoop/hive/metastore/api/FieldSchema.java | 58 +- .../hive/metastore/api/FireEventRequest.java | 79 +- .../metastore/api/FireEventRequestData.java | 20 +- .../hive/metastore/api/FireEventResponse.java | 16 +- .../hadoop/hive/metastore/api/Function.java | 110 +- .../hadoop/hive/metastore/api/FunctionType.java | 2 +- .../metastore/api/GetAllFunctionsResponse.java | 447 + .../metastore/api/GetOpenTxnsInfoResponse.java | 54 +- .../hive/metastore/api/GetOpenTxnsResponse.java | 53 +- .../api/GetPrincipalsInRoleRequest.java | 24 +- .../api/GetPrincipalsInRoleResponse.java | 46 +- .../api/GetRoleGrantsForPrincipalRequest.java | 36 +- .../api/GetRoleGrantsForPrincipalResponse.java | 46 +- .../api/GrantRevokePrivilegeRequest.java | 46 +- .../api/GrantRevokePrivilegeResponse.java | 26 +- .../metastore/api/GrantRevokeRoleRequest.java | 86 +- .../metastore/api/GrantRevokeRoleResponse.java | 26 +- .../hive/metastore/api/GrantRevokeType.java | 2 +- .../hive/metastore/api/HeartbeatRequest.java | 34 +- .../metastore/api/HeartbeatTxnRangeRequest.java | 32 +- .../api/HeartbeatTxnRangeResponse.java | 74 +- .../hive/metastore/api/HiveObjectPrivilege.java | 52 +- .../hive/metastore/api/HiveObjectRef.java | 81 +- .../hive/metastore/api/HiveObjectType.java | 2 +- .../apache/hadoop/hive/metastore/api/Index.java | 133 +- .../api/IndexAlreadyExistsException.java | 24 +- .../metastore/api/InsertEventRequestData.java | 45 +- .../metastore/api/InvalidInputException.java | 24 +- .../metastore/api/InvalidObjectException.java | 24 +- .../api/InvalidOperationException.java | 24 +- .../api/InvalidPartitionException.java | 24 +- .../hive/metastore/api/LockComponent.java | 66 +- .../hadoop/hive/metastore/api/LockLevel.java | 2 +- .../hadoop/hive/metastore/api/LockRequest.java | 72 +- .../hadoop/hive/metastore/api/LockResponse.java | 36 +- .../hadoop/hive/metastore/api/LockState.java | 2 +- .../hadoop/hive/metastore/api/LockType.java | 2 +- .../hive/metastore/api/LongColumnStatsData.java | 50 +- .../hive/metastore/api/MetaException.java | 24 +- .../hive/metastore/api/NoSuchLockException.java | 24 +- .../metastore/api/NoSuchObjectException.java | 24 +- .../hive/metastore/api/NoSuchTxnException.java | 24 +- .../hive/metastore/api/NotificationEvent.java | 66 +- .../metastore/api/NotificationEventRequest.java | 34 +- .../api/NotificationEventResponse.java | 46 +- .../hive/metastore/api/OpenTxnRequest.java | 40 +- .../hive/metastore/api/OpenTxnsResponse.java | 45 +- .../apache/hadoop/hive/metastore/api/Order.java | 32 +- .../hadoop/hive/metastore/api/Partition.java | 156 +- .../hive/metastore/api/PartitionEventType.java | 2 +- .../api/PartitionListComposingSpec.java | 46 +- .../hive/metastore/api/PartitionSpec.java | 58 +- .../api/PartitionSpecWithSharedSD.java | 54 +- .../hive/metastore/api/PartitionWithoutSD.java | 124 +- .../metastore/api/PartitionsByExprRequest.java | 67 +- .../metastore/api/PartitionsByExprResult.java | 54 +- .../metastore/api/PartitionsStatsRequest.java | 90 +- .../metastore/api/PartitionsStatsResult.java | 72 +- .../metastore/api/PrincipalPrivilegeSet.java | 184 +- .../hive/metastore/api/PrincipalType.java | 2 +- .../hadoop/hive/metastore/api/PrivilegeBag.java | 46 +- .../hive/metastore/api/PrivilegeGrantInfo.java | 60 +- .../hive/metastore/api/RequestPartsSpec.java | 56 +- .../hadoop/hive/metastore/api/ResourceType.java | 2 +- .../hadoop/hive/metastore/api/ResourceUri.java | 36 +- .../apache/hadoop/hive/metastore/api/Role.java | 40 +- .../hive/metastore/api/RolePrincipalGrant.java | 80 +- .../hadoop/hive/metastore/api/Schema.java | 91 +- .../hadoop/hive/metastore/api/SerDeInfo.java | 93 +- .../api/SetPartitionsStatsRequest.java | 46 +- .../hive/metastore/api/ShowCompactRequest.java | 16 +- .../hive/metastore/api/ShowCompactResponse.java | 46 +- .../api/ShowCompactResponseElement.java | 86 +- .../hive/metastore/api/ShowLocksRequest.java | 16 +- .../hive/metastore/api/ShowLocksResponse.java | 46 +- .../metastore/api/ShowLocksResponseElement.java | 114 +- .../hadoop/hive/metastore/api/SkewedInfo.java | 147 +- .../hive/metastore/api/StorageDescriptor.java | 242 +- .../metastore/api/StringColumnStatsData.java | 48 +- .../apache/hadoop/hive/metastore/api/Table.java | 189 +- .../hive/metastore/api/TableStatsRequest.java | 61 +- .../hive/metastore/api/TableStatsResult.java | 46 +- .../hive/metastore/api/ThriftHiveMetastore.java | 33417 +++++++++++------ .../hive/metastore/api/TxnAbortedException.java | 24 +- .../hadoop/hive/metastore/api/TxnInfo.java | 52 +- .../hive/metastore/api/TxnOpenException.java | 24 +- .../hadoop/hive/metastore/api/TxnState.java | 2 +- .../apache/hadoop/hive/metastore/api/Type.java | 72 +- .../hive/metastore/api/UnknownDBException.java | 24 +- .../api/UnknownPartitionException.java | 24 +- .../metastore/api/UnknownTableException.java | 24 +- .../hive/metastore/api/UnlockRequest.java | 24 +- .../hadoop/hive/metastore/api/Version.java | 32 +- .../metastore/api/hive_metastoreConstants.java | 7 +- .../gen-php/metastore/ThriftHiveMetastore.php | 4593 ++- .../src/gen/thrift/gen-php/metastore/Types.php | 1184 +- .../hive_metastore/ThriftHiveMetastore-remote | 616 +- .../hive_metastore/ThriftHiveMetastore.py | 4601 ++- .../thrift/gen-py/hive_metastore/constants.py | 2 +- .../gen/thrift/gen-py/hive_metastore/ttypes.py | 856 +- .../thrift/gen-rb/hive_metastore_constants.rb | 2 +- .../gen/thrift/gen-rb/hive_metastore_types.rb | 18 +- .../gen/thrift/gen-rb/thrift_hive_metastore.rb | 62 +- .../hive/metastore/AggregateStatsCache.java | 5 +- .../hadoop/hive/metastore/HiveMetaStore.java | 154 +- .../hive/metastore/HiveMetaStoreClient.java | 14 +- .../hadoop/hive/metastore/IMetaStoreClient.java | 65 +- .../hive/metastore/MetaStoreDirectSql.java | 56 +- .../hadoop/hive/metastore/MetaStoreUtils.java | 26 - .../hadoop/hive/metastore/ObjectStore.java | 1896 +- .../hive/metastore/PartitionDropOptions.java | 6 - .../hadoop/hive/metastore/ProtectMode.java | 97 - .../apache/hadoop/hive/metastore/RawStore.java | 7 + .../hive/metastore/tools/HiveMetaTool.java | 23 +- .../hadoop/hive/metastore/txn/TxnHandler.java | 9 +- .../metastore/txn/ValidCompactorTxnList.java | 2 +- .../DummyRawStoreControlledCommit.java | 7 + .../DummyRawStoreForJdoConnection.java | 6 + .../hadoop/hive/metastore/TestObjectStore.java | 230 + pom.xml | 20 +- ql/if/queryplan.thrift | 1 + ql/pom.xml | 7 + .../gen/thrift/gen-cpp/queryplan_constants.cpp | 2 +- ql/src/gen/thrift/gen-cpp/queryplan_constants.h | 2 +- ql/src/gen/thrift/gen-cpp/queryplan_types.cpp | 796 +- ql/src/gen/thrift/gen-cpp/queryplan_types.h | 294 +- .../hadoop/hive/ql/plan/api/Adjacency.java | 65 +- .../hadoop/hive/ql/plan/api/AdjacencyType.java | 2 +- .../apache/hadoop/hive/ql/plan/api/Graph.java | 87 +- .../hadoop/hive/ql/plan/api/NodeType.java | 2 +- .../hadoop/hive/ql/plan/api/Operator.java | 142 +- .../hadoop/hive/ql/plan/api/OperatorType.java | 7 +- .../apache/hadoop/hive/ql/plan/api/Query.java | 176 +- .../hadoop/hive/ql/plan/api/QueryPlan.java | 62 +- .../apache/hadoop/hive/ql/plan/api/Stage.java | 172 +- .../hadoop/hive/ql/plan/api/StageType.java | 2 +- .../apache/hadoop/hive/ql/plan/api/Task.java | 182 +- .../hadoop/hive/ql/plan/api/TaskType.java | 2 +- ql/src/gen/thrift/gen-php/Types.php | 119 +- ql/src/gen/thrift/gen-py/queryplan/constants.py | 2 +- ql/src/gen/thrift/gen-py/queryplan/ttypes.py | 87 +- ql/src/gen/thrift/gen-rb/queryplan_constants.rb | 2 +- ql/src/gen/thrift/gen-rb/queryplan_types.rb | 7 +- ...tringGroupColumnCompareStringGroupColumn.txt | 112 +- ...gGroupColumnCompareStringGroupScalarBase.txt | 12 +- ...gGroupScalarCompareStringGroupColumnBase.txt | 12 +- ...tringGroupColumnCompareStringGroupColumn.txt | 112 +- ...gGroupColumnCompareStringGroupScalarBase.txt | 12 +- ...gGroupScalarCompareStringGroupColumnBase.txt | 12 +- .../UDAFTemplates/VectorUDAFMinMaxString.txt | 3 +- .../java/org/apache/hadoop/hive/ql/Context.java | 1 - .../java/org/apache/hadoop/hive/ql/Driver.java | 203 +- .../org/apache/hadoop/hive/ql/ErrorMsg.java | 2 +- .../org/apache/hadoop/hive/ql/QueryPlan.java | 18 +- .../org/apache/hadoop/hive/ql/exec/DDLTask.java | 207 +- .../apache/hadoop/hive/ql/exec/ExplainTask.java | 7 +- .../hadoop/hive/ql/exec/FetchOperator.java | 5 +- .../hadoop/hive/ql/exec/FileSinkOperator.java | 2 +- .../hive/ql/exec/HashTableSinkOperator.java | 6 +- .../apache/hadoop/hive/ql/exec/JoinUtil.java | 87 +- .../hadoop/hive/ql/exec/MapJoinOperator.java | 61 +- .../apache/hadoop/hive/ql/exec/MoveTask.java | 2 +- .../apache/hadoop/hive/ql/exec/Operator.java | 6 + .../hadoop/hive/ql/exec/OperatorFactory.java | 11 + .../ql/exec/SparkHashTableSinkOperator.java | 17 +- .../apache/hadoop/hive/ql/exec/Utilities.java | 12 +- .../persistence/HybridHashTableContainer.java | 27 + .../persistence/MapJoinBytesTableContainer.java | 5 + .../exec/persistence/MapJoinTableContainer.java | 5 + .../persistence/MapJoinTableContainerSerDe.java | 63 +- .../hive/ql/exec/spark/HashTableLoader.java | 26 +- .../ql/exec/spark/HiveSparkClientFactory.java | 10 +- .../hive/ql/exec/spark/KryoSerializer.java | 4 + .../ql/exec/spark/RemoteHiveSparkClient.java | 57 +- .../exec/spark/SparkDynamicPartitionPruner.java | 268 + .../hadoop/hive/ql/exec/spark/SparkPlan.java | 3 - .../hive/ql/exec/spark/SparkPlanGenerator.java | 15 +- .../ql/exec/spark/SparkReduceRecordHandler.java | 2 - .../hadoop/hive/ql/exec/spark/SparkTask.java | 1 + .../hive/ql/exec/spark/SparkUtilities.java | 56 + .../hive/ql/exec/tez/KeyValuesAdapter.java | 47 + .../hive/ql/exec/tez/KeyValuesFromKeyValue.java | 90 + .../ql/exec/tez/KeyValuesFromKeyValues.java | 48 + .../ql/exec/tez/MergeFileRecordProcessor.java | 42 +- .../hive/ql/exec/tez/ReduceRecordProcessor.java | 11 +- .../hive/ql/exec/tez/ReduceRecordSource.java | 15 +- .../apache/hadoop/hive/ql/exec/tez/TezTask.java | 10 +- .../hive/ql/exec/vector/BytesColumnVector.java | 347 - .../hive/ql/exec/vector/ColumnVector.java | 173 - .../ql/exec/vector/ColumnVectorVisitor.java | 31 - .../ql/exec/vector/DecimalColumnVector.java | 140 - .../hive/ql/exec/vector/DoubleColumnVector.java | 163 - .../hive/ql/exec/vector/LongColumnVector.java | 207 - .../ql/exec/vector/VectorMapJoinOperator.java | 1 - .../ql/exec/vector/VectorSelectOperator.java | 3 - .../VectorSparkHashTableSinkOperator.java | 101 + ...VectorSparkPartitionPruningSinkOperator.java | 96 + .../hive/ql/exec/vector/VectorizedRowBatch.java | 206 - .../ql/exec/vector/expressions/StringExpr.java | 24 + .../mapjoin/VectorMapJoinRowBytesContainer.java | 9 +- .../fast/VectorMapJoinFastHashTable.java | 5 + .../fast/VectorMapJoinFastTableContainer.java | 5 + .../hashtable/VectorMapJoinHashTable.java | 4 + .../VectorMapJoinOptimizedHashTable.java | 4 + .../hadoop/hive/ql/hooks/LineageLogger.java | 4 +- .../hadoop/hive/ql/hooks/WriteEntity.java | 6 +- .../hadoop/hive/ql/io/AcidInputFormat.java | 60 +- .../hadoop/hive/ql/io/AcidOutputFormat.java | 49 +- .../org/apache/hadoop/hive/ql/io/AcidUtils.java | 152 +- .../hive/ql/io/CombineHiveInputFormat.java | 59 +- .../hadoop/hive/ql/io/HiveFileFormatUtils.java | 19 +- .../hadoop/hive/ql/io/HiveInputFormat.java | 47 +- .../apache/hadoop/hive/ql/io/IOContextMap.java | 1 + .../hadoop/hive/ql/io/orc/MemoryManager.java | 4 +- .../apache/hadoop/hive/ql/io/orc/OrcConf.java | 191 + .../apache/hadoop/hive/ql/io/orc/OrcFile.java | 162 +- .../hadoop/hive/ql/io/orc/OrcInputFormat.java | 44 +- .../hive/ql/io/orc/OrcNewInputFormat.java | 16 +- .../hadoop/hive/ql/io/orc/OrcNewSplit.java | 13 +- .../hadoop/hive/ql/io/orc/OrcOutputFormat.java | 67 +- .../hive/ql/io/orc/OrcRawRecordMerger.java | 84 +- .../hadoop/hive/ql/io/orc/OrcRecordUpdater.java | 63 +- .../apache/hadoop/hive/ql/io/orc/OrcSplit.java | 16 +- .../apache/hadoop/hive/ql/io/orc/Reader.java | 34 +- .../hive/ql/io/orc/RecordReaderFactory.java | 7 +- .../hadoop/hive/ql/io/orc/RecordReaderImpl.java | 62 +- .../hive/ql/io/orc/TreeReaderFactory.java | 12 +- .../hadoop/hive/ql/io/orc/WriterImpl.java | 27 +- .../read/ParquetFilterPredicateConverter.java | 148 + .../read/ParquetRecordReaderWrapper.java | 31 +- .../ql/io/parquet/write/DataWritableWriter.java | 638 +- .../hive/ql/io/sarg/ConvertAstToSearchArg.java | 439 + .../hive/ql/io/sarg/SearchArgumentFactory.java | 56 - .../hive/ql/io/sarg/SearchArgumentImpl.java | 1230 - .../hadoop/hive/ql/lib/RuleExactMatch.java | 21 +- .../apache/hadoop/hive/ql/lib/RuleRegExp.java | 191 +- .../hadoop/hive/ql/lockmgr/DbTxnManager.java | 54 +- .../hadoop/hive/ql/lockmgr/DummyTxnManager.java | 12 + .../hadoop/hive/ql/lockmgr/HiveLockObject.java | 35 + .../hadoop/hive/ql/lockmgr/HiveTxnManager.java | 71 +- .../hive/ql/lockmgr/HiveTxnManagerImpl.java | 150 + .../hadoop/hive/ql/lockmgr/LockException.java | 8 +- .../apache/hadoop/hive/ql/log/PerfLogger.java | 1 - .../apache/hadoop/hive/ql/metadata/Hive.java | 87 +- .../hadoop/hive/ql/metadata/HiveException.java | 3 + .../hadoop/hive/ql/metadata/Partition.java | 51 - .../apache/hadoop/hive/ql/metadata/Table.java | 65 +- .../hadoop/hive/ql/metadata/TableIterable.java | 104 + .../formatting/MetaDataFormatUtils.java | 24 +- .../hadoop/hive/ql/optimizer/ColumnPruner.java | 4 + .../hive/ql/optimizer/ColumnPrunerProcCtx.java | 95 +- .../ql/optimizer/ColumnPrunerProcFactory.java | 42 +- .../hive/ql/optimizer/ConstantPropagate.java | 1 + .../ql/optimizer/ConstantPropagateProcCtx.java | 10 +- .../optimizer/ConstantPropagateProcFactory.java | 100 +- .../hive/ql/optimizer/ConvertJoinMapJoin.java | 213 +- .../DynamicPartitionPruningOptimization.java | 44 +- .../hive/ql/optimizer/GenMapRedUtils.java | 18 +- .../hive/ql/optimizer/MapJoinProcessor.java | 44 +- .../ql/optimizer/OperatorComparatorFactory.java | 552 + .../hadoop/hive/ql/optimizer/Optimizer.java | 2 +- .../hadoop/hive/ql/optimizer/PrunerUtils.java | 14 +- .../ql/optimizer/ReduceSinkMapJoinProc.java | 84 +- .../hive/ql/optimizer/SimpleFetchOptimizer.java | 5 +- .../SparkRemoveDynamicPruningBySize.java | 73 + .../ql/optimizer/calcite/HiveCalciteUtil.java | 25 +- .../ql/optimizer/calcite/HiveRelOptUtil.java | 73 +- .../calcite/cost/HiveOnTezCostModel.java | 25 +- .../calcite/reloperators/HiveJoin.java | 25 +- .../calcite/reloperators/HiveMultiJoin.java | 37 +- .../calcite/reloperators/HiveSemiJoin.java | 57 +- .../rules/HiveInsertExchange4JoinRule.java | 13 +- .../calcite/rules/HiveJoinAddNotNullRule.java | 16 +- .../rules/HiveJoinProjectTransposeRule.java | 223 + .../calcite/rules/HiveJoinToMultiJoinRule.java | 166 +- .../calcite/rules/HiveRelFieldTrimmer.java | 3 +- .../calcite/stats/HiveRelMdSelectivity.java | 11 +- .../calcite/translator/ExprNodeConverter.java | 44 +- .../calcite/translator/HiveOpConverter.java | 207 +- .../translator/HiveOpConverterPostProc.java | 34 +- .../calcite/translator/JoinTypeCheckCtx.java | 2 +- .../translator/PlanModifierForASTConv.java | 2 +- .../translator/PlanModifierForReturnPath.java | 6 +- .../BucketingSortingInferenceOptimizer.java | 8 +- .../physical/GenSparkSkewJoinProcessor.java | 14 +- .../hive/ql/optimizer/physical/Vectorizer.java | 23 + .../spark/CombineEquivalentWorkResolver.java | 292 + .../spark/SparkPartitionPruningSinkDesc.java | 100 + .../spark/SparkReduceSinkMapJoinProc.java | 2 +- .../stats/annotation/StatsRulesProcFactory.java | 91 +- .../apache/hadoop/hive/ql/parse/ASTNode.java | 139 +- .../hive/ql/parse/BaseSemanticAnalyzer.java | 13 + .../hadoop/hive/ql/parse/CalcitePlanner.java | 74 +- .../hive/ql/parse/DDLSemanticAnalyzer.java | 112 +- .../apache/hadoop/hive/ql/parse/EximUtil.java | 31 +- .../hive/ql/parse/ExplainSemanticAnalyzer.java | 2 +- .../hadoop/hive/ql/parse/GenTezProcContext.java | 12 + .../hadoop/hive/ql/parse/GenTezUtils.java | 23 +- .../apache/hadoop/hive/ql/parse/GenTezWork.java | 81 +- .../org/apache/hadoop/hive/ql/parse/HiveLexer.g | 16 +- .../apache/hadoop/hive/ql/parse/HiveParser.g | 108 +- .../hadoop/hive/ql/parse/IdentifiersParser.g | 19 +- .../hive/ql/parse/LoadSemanticAnalyzer.java | 23 +- .../apache/hadoop/hive/ql/parse/ParseUtils.java | 53 +- .../hadoop/hive/ql/parse/SemanticAnalyzer.java | 106 +- .../hive/ql/parse/SemanticAnalyzerFactory.java | 20 +- .../hadoop/hive/ql/parse/TypeCheckCtx.java | 15 +- .../hive/ql/parse/TypeCheckProcFactory.java | 4 + .../ql/parse/spark/GenSparkProcContext.java | 14 +- .../hive/ql/parse/spark/GenSparkUtils.java | 111 +- .../parse/spark/OptimizeSparkProcContext.java | 16 +- .../hive/ql/parse/spark/SparkCompiler.java | 180 +- .../SparkPartitionPruningSinkOperator.java | 141 + .../hive/ql/parse/spark/SplitOpTreeForDPP.java | 151 + .../hadoop/hive/ql/plan/AlterTableDesc.java | 20 +- .../apache/hadoop/hive/ql/plan/BaseWork.java | 2 +- .../hive/ql/plan/CommonMergeJoinDesc.java | 4 + .../hadoop/hive/ql/plan/DropTableDesc.java | 26 +- .../apache/hadoop/hive/ql/plan/ExplainWork.java | 19 +- .../hadoop/hive/ql/plan/ExprNodeDescUtils.java | 115 + .../hadoop/hive/ql/plan/FileSinkDesc.java | 27 +- .../hadoop/hive/ql/plan/HiveOperation.java | 34 +- .../hadoop/hive/ql/plan/JoinCondDesc.java | 14 + .../apache/hadoop/hive/ql/plan/JoinDesc.java | 4 + .../apache/hadoop/hive/ql/plan/MapJoinDesc.java | 11 + .../org/apache/hadoop/hive/ql/plan/MapWork.java | 10 + .../hadoop/hive/ql/plan/PartitionDesc.java | 39 +- .../hadoop/hive/ql/plan/ReduceSinkDesc.java | 1 + .../apache/hadoop/hive/ql/plan/ReduceWork.java | 2 +- .../hive/ql/plan/SparkHashTableSinkDesc.java | 11 + .../hadoop/hive/ql/plan/TableScanDesc.java | 6 +- .../hive/ql/ppd/ExprWalkerProcFactory.java | 17 +- .../hive/ql/ppd/SyntheticJoinPredicate.java | 14 +- .../ql/processors/CommandProcessorResponse.java | 21 +- .../hadoop/hive/ql/processors/HiveCommand.java | 3 + .../authorization/plugin/HiveAuthorizer.java | 11 + .../plugin/HiveAuthorizerImpl.java | 22 + .../authorization/plugin/HiveOperationType.java | 5 + .../authorization/plugin/HiveV1Authorizer.java | 20 + .../plugin/sqlstd/Operation2Privilege.java | 11 + .../hadoop/hive/ql/session/SessionState.java | 44 +- .../apache/hadoop/hive/ql/stats/StatsUtils.java | 10 +- .../hive/ql/txn/compactor/CompactorMR.java | 4 +- .../ql/udf/generic/GenericUDAFCollectList.java | 5 - .../ql/udf/generic/GenericUDAFCollectSet.java | 5 - .../hive/ql/udf/generic/GenericUDAFCount.java | 4 - .../ql/udf/generic/GenericUDAFCovariance.java | 4 - .../ql/udf/generic/GenericUDAFCumeDist.java | 4 - .../ql/udf/generic/GenericUDAFDenseRank.java | 4 - .../hive/ql/udf/generic/GenericUDFBridge.java | 8 +- .../hive/ql/udf/generic/GenericUDFMapKeys.java | 6 +- .../apache/hadoop/hive/ql/TestTxnCommands.java | 473 + .../apache/hadoop/hive/ql/TestTxnCommands2.java | 68 +- .../hive/ql/exec/TestFileSinkOperator.java | 3 +- .../exec/vector/TestVectorizedRowBatchCtx.java | 6 +- .../apache/hadoop/hive/ql/io/TestAcidUtils.java | 73 +- .../hadoop/hive/ql/io/TestIOContextMap.java | 2 +- .../hive/ql/io/orc/TestInputOutputFormat.java | 42 +- .../hadoop/hive/ql/io/orc/TestOrcFile.java | 11 +- .../hive/ql/io/orc/TestOrcRawRecordMerger.java | 57 +- .../hive/ql/io/orc/TestOrcRecordUpdater.java | 6 +- .../hive/ql/io/orc/TestRecordReaderImpl.java | 63 +- .../hive/ql/io/orc/TestVectorizedORCReader.java | 75 +- .../parquet/TestParquetRecordReaderWrapper.java | 155 + .../read/TestParquetFilterPredicate.java | 51 + .../ql/io/sarg/TestConvertAstToSearchArg.java | 2856 ++ .../hive/ql/io/sarg/TestSearchArgumentImpl.java | 2889 +- .../hadoop/hive/ql/lib/TestRuleRegExp.java | 118 + .../parse/TestUpdateDeleteSemanticAnalyzer.java | 34 +- .../positive/TestTransactionStatement.java | 102 + .../hive/ql/session/TestSessionState.java | 2 +- .../hive/ql/txn/compactor/CompactorTest.java | 20 +- .../hive/ql/txn/compactor/TestCleaner.java | 8 +- .../hive/ql/txn/compactor/TestCleaner2.java | 14 + .../hive/ql/txn/compactor/TestInitiator.java | 4 + .../hive/ql/txn/compactor/TestWorker.java | 49 +- .../hive/ql/txn/compactor/TestWorker2.java | 16 + .../alter_partition_invalidspec.q | 8 - .../clientnegative/alter_partition_nodrop.q | 9 - .../alter_partition_nodrop_table.q | 9 - .../clientnegative/alter_partition_offline.q | 11 - .../clientnegative/drop_table_failure3.q | 12 - .../queries/clientnegative/exchange_partition.q | 19 + .../clientnegative/load_orc_negative_part.q | 14 + .../queries/clientnegative/protectmode_part.q | 15 - .../queries/clientnegative/protectmode_part1.q | 21 - .../queries/clientnegative/protectmode_part2.q | 9 - .../clientnegative/protectmode_part_no_drop.q | 10 - .../clientnegative/protectmode_part_no_drop2.q | 11 - .../queries/clientnegative/protectmode_tbl1.q | 8 - .../queries/clientnegative/protectmode_tbl2.q | 12 - .../queries/clientnegative/protectmode_tbl3.q | 10 - .../queries/clientnegative/protectmode_tbl4.q | 15 - .../queries/clientnegative/protectmode_tbl5.q | 15 - .../queries/clientnegative/protectmode_tbl6.q | 8 - .../queries/clientnegative/protectmode_tbl7.q | 13 - .../queries/clientnegative/protectmode_tbl8.q | 13 - .../clientnegative/protectmode_tbl_no_drop.q | 9 - .../test/queries/clientnegative/sa_fail_hook3.q | 4 - .../alter_partition_protect_mode.q | 26 - .../annotate_stats_deep_filters.q | 66 + .../queries/clientpositive/ba_table_union.q | 2 +- .../queries/clientpositive/cbo_rp_auto_join0.q | 1 + .../queries/clientpositive/cbo_rp_auto_join1.q | 1 + .../test/queries/clientpositive/cbo_rp_join0.q | 1 + .../queries/clientpositive/compustat_avro.q | 8 +- .../test/queries/clientpositive/constprog_dpp.q | 17 + .../drop_partitions_ignore_protection.q | 10 - .../queries/clientpositive/dynamic_rdd_cache.q | 111 + .../encryption_insert_partition_static.q | 17 - .../queries/clientpositive/fouter_join_ppr.q | 73 + ql/src/test/queries/clientpositive/groupby5.q | 2 + ql/src/test/queries/clientpositive/lineage3.q | 7 +- .../clientpositive/load_dyn_part14_win.q | 18 +- .../test/queries/clientpositive/load_orc_part.q | 15 + .../clientpositive/parquet_predicate_pushdown.q | 9 + .../test/queries/clientpositive/protectmode.q | 63 - .../test/queries/clientpositive/protectmode2.q | 23 - .../queries/clientpositive/select_same_col.q | 5 +- .../clientpositive/smb_join_partition_key.q | 35 + .../spark_dynamic_partition_pruning.q | 180 + .../spark_dynamic_partition_pruning_2.q | 118 + ...spark_vectorized_dynamic_partition_pruning.q | 192 + .../test/queries/clientpositive/stats_ppr_all.q | 24 + .../clientpositive/tez_dynpart_hashjoin_1.q | 101 + .../clientpositive/tez_dynpart_hashjoin_2.q | 83 + .../tez_vector_dynpart_hashjoin_1.q | 102 + .../tez_vector_dynpart_hashjoin_2.q | 84 + .../clientpositive/udf_from_utc_timestamp.q | 30 +- .../queries/clientpositive/udf_percentile.q | 2 + .../clientpositive/udf_to_utc_timestamp.q | 30 +- .../clientpositive/unionall_unbalancedppd.q | 192 + .../test/queries/clientpositive/vector_acid3.q | 17 + .../vector_aggregate_without_gby.q | 14 + .../alter_numbuckets_partitioned_table.q.out | 8 - .../results/beelinepositive/create_like.q.out | 3 - .../results/beelinepositive/create_like2.q.out | 1 - .../beelinepositive/create_like_view.q.out | 4 - .../beelinepositive/create_skewed_table1.q.out | 3 - .../results/beelinepositive/create_view.q.out | 14 - .../create_view_partitioned.q.out | 3 - ql/src/test/results/beelinepositive/ctas.q.out | 5 - .../describe_formatted_view_partitioned.q.out | 1 - .../beelinepositive/describe_table.q.out | 3 - .../test/results/beelinepositive/merge3.q.out | 1 - .../part_inherit_tbl_props.q.out | 1 - .../part_inherit_tbl_props_empty.q.out | 1 - .../part_inherit_tbl_props_with_star.q.out | 1 - .../results/beelinepositive/protectmode2.q.out | 2 - .../test/results/beelinepositive/stats1.q.out | 2 - .../test/results/beelinepositive/stats10.q.out | 3 - .../test/results/beelinepositive/stats11.q.out | 4 - .../test/results/beelinepositive/stats12.q.out | 5 - .../test/results/beelinepositive/stats13.q.out | 6 - .../test/results/beelinepositive/stats14.q.out | 5 - .../test/results/beelinepositive/stats15.q.out | 5 - .../test/results/beelinepositive/stats16.q.out | 2 - .../test/results/beelinepositive/stats18.q.out | 2 - .../test/results/beelinepositive/stats2.q.out | 2 - .../test/results/beelinepositive/stats3.q.out | 2 - .../test/results/beelinepositive/stats4.q.out | 6 - .../test/results/beelinepositive/stats5.q.out | 1 - .../test/results/beelinepositive/stats6.q.out | 5 - .../test/results/beelinepositive/stats7.q.out | 3 - .../test/results/beelinepositive/stats8.q.out | 10 - .../test/results/beelinepositive/stats9.q.out | 1 - .../beelinepositive/stats_empty_partition.q.out | 1 - .../clientnegative/alter_file_format.q.out | 1 - .../alter_view_as_select_with_partition.q.out | 1 - .../clientnegative/exchange_partition.q.out | 54 + .../clientnegative/load_orc_negative_part.q.out | 52 + .../groupby2_map_skew_multi_distinct.q.out | 9 + .../spark/groupby2_multi_distinct.q.out | 9 + .../groupby3_map_skew_multi_distinct.q.out | 9 + .../spark/groupby3_multi_distinct.q.out | 9 + .../spark/groupby_grouping_sets7.q.out | 9 + .../stats_partialscan_autogether.q.out | 2 - .../clientpositive/alter_file_format.q.out | 19 - .../clientpositive/alter_merge_stats_orc.q.out | 8 - .../alter_numbuckets_partitioned_table.q.out | 16 - .../alter_numbuckets_partitioned_table2.q.out | 27 - ...lter_numbuckets_partitioned_table2_h23.q.out | 27 - ...alter_numbuckets_partitioned_table_h23.q.out | 16 - .../alter_partition_clusterby_sortby.q.out | 7 - .../clientpositive/alter_skewed_table.q.out | 6 - .../clientpositive/alter_table_not_sorted.q.out | 2 - .../clientpositive/alter_table_serde2.q.out | 6 - .../clientpositive/alter_view_as_select.q.out | 3 - .../annotate_stats_deep_filters.q.out | 244 + .../clientpositive/authorization_index.q.out | 1 - .../results/clientpositive/ba_table_union.q.out | 16 +- .../test/results/clientpositive/bucket5.q.out | 1 - .../clientpositive/cbo_rp_auto_join0.q.out | 32 +- .../clientpositive/cbo_rp_auto_join1.q.out | 578 +- .../results/clientpositive/cbo_rp_join0.q.out | 185 +- .../results/clientpositive/compustat_avro.q.out | 8 +- .../clientpositive/convert_enum_to_string.q.out | 9 +- .../create_alter_list_bucketing_table1.q.out | 7 - .../results/clientpositive/create_like.q.out | 9 - .../results/clientpositive/create_like2.q.out | 1 - .../clientpositive/create_like_tbl_props.q.out | 5 - .../clientpositive/create_like_view.q.out | 4 - .../clientpositive/create_or_replace_view.q.out | 5 - .../clientpositive/create_skewed_table1.q.out | 3 - .../results/clientpositive/create_view.q.out | 14 - .../create_view_partitioned.q.out | 3 - .../clientpositive/create_view_translate.q.out | 2 - ql/src/test/results/clientpositive/ctas.q.out | 5 - .../results/clientpositive/ctas_colname.q.out | 7 - .../results/clientpositive/ctas_hadoop20.q.out | 5 - .../ctas_uses_database_location.q.out | 1 - .../clientpositive/database_location.q.out | 2 - .../results/clientpositive/decimal_serde.q.out | 2 - .../clientpositive/default_file_format.q.out | 5 - .../describe_comment_indent.q.out | 1 - .../describe_comment_nonascii.q.out | 1 - .../describe_formatted_view_partitioned.q.out | 2 - .../clientpositive/describe_syntax.q.out | 6 - .../results/clientpositive/describe_table.q.out | 7 - .../clientpositive/dynamic_rdd_cache.q.out | 1428 + .../dynpart_sort_opt_vectorization.q.out | 32 - .../dynpart_sort_optimization.q.out | 32 - .../dynpart_sort_optimization2.q.out | 24 - .../encryption_insert_partition_static.q.out | 739 +- .../encrypted/encryption_insert_values.q.out | 1 - .../clientpositive/exchange_partition.q.out | 4 +- .../clientpositive/exchange_partition2.q.out | 4 +- .../clientpositive/exchange_partition3.q.out | 4 +- .../clientpositive/exim_hidden_files.q.out | 1 - .../clientpositive/fouter_join_ppr.q.out | 1694 + .../test/results/clientpositive/groupby5.q.out | 8 +- .../clientpositive/index_skewtable.q.out | 1 - .../clientpositive/infer_bucket_sort.q.out | 50 - .../infer_bucket_sort_bucketed_table.q.out | 2 - .../infer_bucket_sort_convert_join.q.out | 4 - .../infer_bucket_sort_dyn_part.q.out | 16 - .../infer_bucket_sort_grouping_operators.q.out | 12 - .../infer_bucket_sort_list_bucket.q.out | 4 - .../infer_bucket_sort_map_operators.q.out | 8 - .../infer_bucket_sort_merge.q.out | 4 - .../infer_bucket_sort_multi_insert.q.out | 16 - .../infer_bucket_sort_num_buckets.q.out | 4 - .../infer_bucket_sort_reducers_power_two.q.out | 12 - .../results/clientpositive/lb_fs_stats.q.out | 2 - .../test/results/clientpositive/lineage2.q.out | 2296 +- .../test/results/clientpositive/lineage3.q.out | 2262 +- .../clientpositive/list_bucket_dml_1.q.out | 4 - .../list_bucket_dml_10.q.java1.7.out | 2 - .../list_bucket_dml_10.q.java1.8.out | 2 - .../list_bucket_dml_11.q.java1.7.out | 2 - .../list_bucket_dml_11.q.java1.8.out | 2 - .../list_bucket_dml_12.q.java1.7.out | 2 - .../list_bucket_dml_12.q.java1.8.out | 2 - .../list_bucket_dml_13.q.java1.7.out | 2 - .../list_bucket_dml_13.q.java1.8.out | 2 - .../clientpositive/list_bucket_dml_14.q.out | 1 - .../list_bucket_dml_2.q.java1.7.out | 2 - .../list_bucket_dml_2.q.java1.8.out | 2 - .../clientpositive/list_bucket_dml_3.q.out | 2 - .../list_bucket_dml_4.q.java1.7.out | 4 - .../list_bucket_dml_4.q.java1.8.out | 4 - .../list_bucket_dml_5.q.java1.7.out | 4 - .../list_bucket_dml_5.q.java1.8.out | 4 - .../list_bucket_dml_6.q.java1.7.out | 8 - .../list_bucket_dml_6.q.java1.8.out | 8 - .../clientpositive/list_bucket_dml_7.q.out | 8 - .../list_bucket_dml_8.q.java1.7.out | 6 - .../list_bucket_dml_8.q.java1.8.out | 6 - .../list_bucket_dml_9.q.java1.7.out | 4 - .../list_bucket_dml_9.q.java1.8.out | 4 - .../list_bucket_query_multiskew_1.q.out | 2 - .../list_bucket_query_multiskew_2.q.out | 2 - .../list_bucket_query_multiskew_3.q.out | 6 - .../list_bucket_query_oneskew_1.q.out | 2 - .../list_bucket_query_oneskew_2.q.out | 2 - .../list_bucket_query_oneskew_3.q.out | 2 - .../clientpositive/load_dyn_part14_win.q.out | 167 +- .../results/clientpositive/load_orc_part.q.out | 70 + ql/src/test/results/clientpositive/merge3.q.out | 1 - .../results/clientpositive/orc_analyze.q.out | 48 - .../results/clientpositive/orc_create.q.out | 6 - .../clientpositive/parallel_orderby.q.out | 2 - .../parquet_array_null_element.q.out | 1 - .../results/clientpositive/parquet_create.q.out | 1 - .../clientpositive/parquet_partitioned.q.out | 1 - .../parquet_predicate_pushdown.q.out | 47 + .../results/clientpositive/parquet_serde.q.out | 5 - .../clientpositive/part_inherit_tbl_props.q.out | 2 - .../part_inherit_tbl_props_empty.q.out | 2 - .../part_inherit_tbl_props_with_star.q.out | 2 - .../partition_coltype_literals.q.out | 16 - .../test/results/clientpositive/ppd_join2.q.out | 18 +- .../test/results/clientpositive/ppd_join3.q.out | 46 +- .../results/clientpositive/protectmode2.q.out | 2 - .../clientpositive/rcfile_default_format.q.out | 8 - .../clientpositive/selectDistinctStar.q.out | 2 - .../clientpositive/select_same_col.q.out | 8 +- .../clientpositive/smb_join_partition_key.q.out | 128 + .../spark/alter_merge_stats_orc.q.out | 8 - .../clientpositive/spark/auto_join18.q.out | 24 +- .../clientpositive/spark/auto_join30.q.out | 51 +- .../clientpositive/spark/auto_join32.q.out | 24 +- .../spark/auto_smb_mapjoin_14.q.out | 30 +- .../spark/auto_sortmerge_join_10.q.out | 23 +- .../results/clientpositive/spark/bucket2.q.out | 3 - .../results/clientpositive/spark/bucket3.q.out | 3 - .../results/clientpositive/spark/bucket4.q.out | 3 - .../results/clientpositive/spark/bucket5.q.out | 1 - .../spark/column_access_stats.q.out | 4 - .../results/clientpositive/spark/ctas.q.out | 5 - .../spark/dynamic_rdd_cache.q.out | 1073 + .../clientpositive/spark/groupby10.q.out | 32 +- .../clientpositive/spark/groupby1_map.q.out | 412 + .../spark/groupby1_map_nomap.q.out | 408 + .../spark/groupby1_map_skew.q.out | 427 + .../clientpositive/spark/groupby1_noskew.q.out | 406 + .../clientpositive/spark/groupby2_map.q.out | 118 + .../spark/groupby2_map_multi_distinct.q.out | 232 + .../spark/groupby2_map_skew.q.out | 129 + .../clientpositive/spark/groupby2_noskew.q.out | 111 + .../spark/groupby2_noskew_multi_distinct.q.out | 114 + .../clientpositive/spark/groupby4_map.q.out | 93 + .../spark/groupby4_map_skew.q.out | 93 + .../clientpositive/spark/groupby4_noskew.q.out | 104 + .../results/clientpositive/spark/groupby5.q.out | 433 + .../clientpositive/spark/groupby5_map.q.out | 95 + .../spark/groupby5_map_skew.q.out | 95 + .../clientpositive/spark/groupby5_noskew.q.out | 418 + .../results/clientpositive/spark/groupby6.q.out | 113 + .../clientpositive/spark/groupby6_map.q.out | 109 + .../spark/groupby6_map_skew.q.out | 122 + .../clientpositive/spark/groupby6_noskew.q.out | 104 + .../clientpositive/spark/groupby7_map.q.out | 23 +- .../spark/groupby7_map_skew.q.out | 38 +- .../clientpositive/spark/groupby7_noskew.q.out | 17 +- .../groupby7_noskew_multi_single_reducer.q.out | 18 +- .../results/clientpositive/spark/groupby8.q.out | 62 +- .../spark/groupby8_map_skew.q.out | 37 +- .../spark/groupby_grouping_id2.q.out | 230 + .../clientpositive/spark/groupby_position.q.out | 37 +- .../spark/groupby_ppr_multi_distinct.q.out | 346 + .../spark/groupby_resolution.q.out | 796 + .../infer_bucket_sort_bucketed_table.q.out | 2 - .../spark/infer_bucket_sort_convert_join.q.out | 4 - .../spark/infer_bucket_sort_map_operators.q.out | 8 - .../spark/infer_bucket_sort_merge.q.out | 4 - .../spark/infer_bucket_sort_num_buckets.q.out | 4 - .../infer_bucket_sort_reducers_power_two.q.out | 12 - .../clientpositive/spark/insert_into3.q.out | 33 +- .../results/clientpositive/spark/join18.q.out | 24 +- .../results/clientpositive/spark/join22.q.out | 19 +- .../spark/limit_partition_metadataonly.q.out | 2 - .../clientpositive/spark/limit_pushdown.q.out | 31 +- .../spark/list_bucket_dml_10.q.java1.7.out | 2 - .../spark/list_bucket_dml_10.q.java1.8.out | 2 - .../spark/list_bucket_dml_2.q.java1.7.out | 5 - .../spark/list_bucket_dml_2.q.java1.8.out | 2 - .../spark/list_bucket_dml_2.q.out | Bin 28747 -> 28667 bytes .../clientpositive/spark/load_dyn_part14.q.out | 30 +- .../clientpositive/spark/nullgroup.q.out | 265 + .../clientpositive/spark/nullgroup2.q.out | 300 + .../clientpositive/spark/nullgroup4.q.out | 292 + .../spark/nullgroup4_multi_distinct.q.out | 133 + .../spark/optimize_nullscan.q.out | 3 - .../clientpositive/spark/orc_analyze.q.out | 22 - .../spark/outer_join_ppr.q.java1.7.out | 7 +- .../clientpositive/spark/parallel_orderby.q.out | 2 - .../test/results/clientpositive/spark/pcr.q.out | 6 - .../clientpositive/spark/ppd_join2.q.out | 18 +- .../clientpositive/spark/ppd_join3.q.out | 46 +- .../results/clientpositive/spark/sample3.q.out | 3 - .../results/clientpositive/spark/sample9.q.out | 3 - .../clientpositive/spark/skewjoinopt11.q.out | 60 +- .../clientpositive/spark/skewjoinopt9.q.out | 20 +- .../clientpositive/spark/smb_mapjoin_11.q.out | 6 - .../spark/spark_dynamic_partition_pruning.q.out | 5573 +++ .../spark_dynamic_partition_pruning_2.q.out | 1015 + ...k_vectorized_dynamic_partition_pruning.q.out | 5822 +++ .../results/clientpositive/spark/stats1.q.out | 2 - .../results/clientpositive/spark/stats10.q.out | 5 - .../results/clientpositive/spark/stats12.q.out | 9 - .../results/clientpositive/spark/stats13.q.out | 10 - .../results/clientpositive/spark/stats14.q.out | 7 - .../results/clientpositive/spark/stats15.q.out | 7 - .../results/clientpositive/spark/stats16.q.out | 2 - .../results/clientpositive/spark/stats18.q.out | 4 - .../results/clientpositive/spark/stats2.q.out | 2 - .../results/clientpositive/spark/stats20.q.out | 2 - .../results/clientpositive/spark/stats3.q.out | 2 - .../results/clientpositive/spark/stats5.q.out | 1 - .../results/clientpositive/spark/stats6.q.out | 9 - .../results/clientpositive/spark/stats7.q.out | 5 - .../results/clientpositive/spark/stats8.q.out | 18 - .../results/clientpositive/spark/stats9.q.out | 1 - .../clientpositive/spark/stats_counter.q.out | 2 - .../spark/stats_counter_partitioned.q.out | 16 - .../clientpositive/spark/stats_noscan_1.q.out | 17 - .../clientpositive/spark/stats_noscan_2.q.out | 6 - .../clientpositive/spark/stats_only_null.q.out | 4 - .../spark/stats_partscan_1_23.q.out | 6 - .../results/clientpositive/spark/statsfs.q.out | 14 - .../clientpositive/spark/temp_table_gb1.q.out | 67 + .../clientpositive/spark/udaf_collect_set.q.out | 212 + .../clientpositive/spark/udf_example_add.q.out | 3 - .../clientpositive/spark/udf_in_file.q.out | 3 - .../results/clientpositive/spark/udf_max.q.out | 62 + .../results/clientpositive/spark/udf_min.q.out | 62 + .../clientpositive/spark/udf_percentile.q.out | 450 + .../results/clientpositive/spark/union10.q.out | 36 +- .../results/clientpositive/spark/union11.q.out | 38 +- .../results/clientpositive/spark/union15.q.out | 23 +- .../results/clientpositive/spark/union16.q.out | 450 +- .../results/clientpositive/spark/union2.q.out | 36 +- .../results/clientpositive/spark/union20.q.out | 18 +- .../results/clientpositive/spark/union25.q.out | 21 +- .../results/clientpositive/spark/union28.q.out | 21 +- .../results/clientpositive/spark/union3.q.out | 45 +- .../results/clientpositive/spark/union30.q.out | 21 +- .../results/clientpositive/spark/union4.q.out | 18 +- .../results/clientpositive/spark/union5.q.out | 20 +- .../results/clientpositive/spark/union9.q.out | 54 +- .../clientpositive/spark/union_remove_1.q.out | 24 +- .../clientpositive/spark/union_remove_10.q.out | 1 - .../clientpositive/spark/union_remove_11.q.out | 1 - .../clientpositive/spark/union_remove_12.q.out | 1 - .../clientpositive/spark/union_remove_13.q.out | 1 - .../clientpositive/spark/union_remove_14.q.out | 1 - .../clientpositive/spark/union_remove_15.q.out | 24 +- .../clientpositive/spark/union_remove_16.q.out | 24 +- .../clientpositive/spark/union_remove_17.q.out | 1 - .../clientpositive/spark/union_remove_18.q.out | 24 +- .../clientpositive/spark/union_remove_19.q.out | 76 +- .../clientpositive/spark/union_remove_2.q.out | 1 - .../clientpositive/spark/union_remove_20.q.out | 24 +- .../clientpositive/spark/union_remove_21.q.out | 22 +- .../clientpositive/spark/union_remove_22.q.out | 47 +- .../clientpositive/spark/union_remove_23.q.out | 1 - .../clientpositive/spark/union_remove_24.q.out | 24 +- .../clientpositive/spark/union_remove_25.q.out | 65 +- .../clientpositive/spark/union_remove_3.q.out | 1 - .../clientpositive/spark/union_remove_4.q.out | 24 +- .../clientpositive/spark/union_remove_5.q.out | 1 - .../clientpositive/spark/union_remove_6.q.out | 23 +- .../spark/union_remove_6_subq.q.out | 64 +- .../clientpositive/spark/union_remove_7.q.out | 24 +- .../clientpositive/spark/union_remove_8.q.out | 1 - .../clientpositive/spark/union_remove_9.q.out | 1 - .../clientpositive/spark/union_top_level.q.out | 59 +- .../clientpositive/spark/union_view.q.out | 33 +- .../spark/vector_count_distinct.q.out | 31 +- .../spark/vector_decimal_mapjoin.q.out | 1 + .../clientpositive/spark/vector_elt.q.out | 7 - .../spark/vector_left_outer_join.q.out | 2 + .../spark/vector_mapjoin_reduce.q.out | 1 + .../spark/vector_string_concat.q.out | 3 - .../clientpositive/spark/vectorization_17.q.out | 10 +- .../spark/vectorization_decimal_date.q.out | 4 - .../spark/vectorization_div0.q.out | 3 - .../clientpositive/spark/vectorized_case.q.out | 3 - .../spark/vectorized_mapjoin.q.out | 1 + .../spark/vectorized_math_funcs.q.out | 3 - .../spark/vectorized_nested_mapjoin.q.out | 2 + .../spark/vectorized_string_funcs.q.out | 3 - ql/src/test/results/clientpositive/stats1.q.out | 2 - .../test/results/clientpositive/stats10.q.out | 5 - .../test/results/clientpositive/stats11.q.out | 8 - .../test/results/clientpositive/stats12.q.out | 9 - .../test/results/clientpositive/stats13.q.out | 10 - .../test/results/clientpositive/stats14.q.out | 7 - .../test/results/clientpositive/stats15.q.out | 7 - .../test/results/clientpositive/stats16.q.out | 2 - .../test/results/clientpositive/stats18.q.out | 4 - .../test/results/clientpositive/stats19.q.out | 12 - ql/src/test/results/clientpositive/stats2.q.out | 2 - .../test/results/clientpositive/stats20.q.out | 2 - ql/src/test/results/clientpositive/stats3.q.out | 2 - ql/src/test/results/clientpositive/stats4.q.out | 10 - ql/src/test/results/clientpositive/stats5.q.out | 1 - ql/src/test/results/clientpositive/stats6.q.out | 9 - ql/src/test/results/clientpositive/stats7.q.out | 5 - ql/src/test/results/clientpositive/stats8.q.out | 18 - ql/src/test/results/clientpositive/stats9.q.out | 1 - .../results/clientpositive/stats_counter.q.out | 2 - .../stats_counter_partitioned.q.out | 16 - .../clientpositive/stats_empty_partition.q.out | 2 - .../clientpositive/stats_invalidation.q.out | 2 - .../stats_list_bucket.q.java1.7.out | 3 - .../stats_list_bucket.q.java1.8.out | 3 - .../results/clientpositive/stats_noscan_1.q.out | 17 - .../results/clientpositive/stats_noscan_2.q.out | 6 - .../clientpositive/stats_only_null.q.out | 4 - .../clientpositive/stats_partscan_1.q.out | 6 - .../clientpositive/stats_partscan_1_23.q.out | 6 - .../results/clientpositive/stats_ppr_all.q.out | 284 + .../test/results/clientpositive/statsfs.q.out | 14 - .../tez/alter_merge_stats_orc.q.out | 8 - .../clientpositive/tez/constprog_dpp.q.out | 113 + .../test/results/clientpositive/tez/ctas.q.out | 5 - .../tez/dynpart_sort_opt_vectorization.q.out | 32 - .../tez/dynpart_sort_optimization.q.out | 32 - .../tez/dynpart_sort_optimization2.q.out | 24 - .../clientpositive/tez/orc_analyze.q.out | 48 - .../results/clientpositive/tez/orc_merge9.q.out | 186 + .../clientpositive/tez/selectDistinctStar.q.out | 2 - .../clientpositive/tez/stats_counter.q.out | 2 - .../tez/stats_counter_partitioned.q.out | 16 - .../clientpositive/tez/stats_noscan_1.q.out | 17 - .../clientpositive/tez/stats_only_null.q.out | 4 - .../tez/tez_dynpart_hashjoin_1.q.out | 791 + .../tez/tez_dynpart_hashjoin_2.q.out | 564 + .../results/clientpositive/tez/tez_fsstat.q.out | 2 - .../tez/tez_vector_dynpart_hashjoin_1.q.out | 804 + .../tez/tez_vector_dynpart_hashjoin_2.q.out | 570 + .../results/clientpositive/tez/union2.q.out | 28 +- .../results/clientpositive/tez/union9.q.out | 40 +- .../clientpositive/tez/vector_acid3.q.out | 31 + .../tez/vector_null_projection.q.out | 4 + .../clientpositive/tez/vectorization_17.q.out | 10 +- .../clientpositive/truncate_column.q.out | 11 - .../clientpositive/udf_from_utc_timestamp.q.out | 66 +- .../results/clientpositive/udf_percentile.q.out | 104 +- .../clientpositive/udf_to_utc_timestamp.q.out | 66 +- .../clientpositive/unicode_notation.q.out | 3 - .../test/results/clientpositive/union16.q.out | 354 +- ql/src/test/results/clientpositive/union2.q.out | 32 +- ql/src/test/results/clientpositive/union9.q.out | 46 +- .../results/clientpositive/union_remove_1.q.out | 1 - .../clientpositive/union_remove_10.q.out | 1 - .../clientpositive/union_remove_11.q.out | 1 - .../clientpositive/union_remove_12.q.out | 1 - .../clientpositive/union_remove_13.q.out | 1 - .../clientpositive/union_remove_14.q.out | 1 - .../clientpositive/union_remove_15.q.out | 1 - .../clientpositive/union_remove_16.q.out | 1 - .../clientpositive/union_remove_17.q.out | 1 - .../clientpositive/union_remove_18.q.out | 1 - .../clientpositive/union_remove_19.q.out | 1 - .../results/clientpositive/union_remove_2.q.out | 1 - .../clientpositive/union_remove_20.q.out | 1 - .../clientpositive/union_remove_21.q.out | 1 - .../clientpositive/union_remove_22.q.out | 1 - .../clientpositive/union_remove_23.q.out | 1 - .../clientpositive/union_remove_24.q.out | 1 - .../clientpositive/union_remove_25.q.out | 6 - .../results/clientpositive/union_remove_3.q.out | 1 - .../results/clientpositive/union_remove_4.q.out | 1 - .../results/clientpositive/union_remove_5.q.out | 1 - .../results/clientpositive/union_remove_7.q.out | 1 - .../results/clientpositive/union_remove_8.q.out | 1 - .../results/clientpositive/union_remove_9.q.out | 1 - .../results/clientpositive/union_view.q.out | 24 + .../clientpositive/unionall_unbalancedppd.q.out | 653 + .../results/clientpositive/vector_acid3.q.out | 31 + .../vector_aggregate_without_gby.q.out | 96 + .../clientpositive/vector_null_projection.q.out | 4 + .../clientpositive/vectorization_17.q.out | 10 +- .../gen/thrift/gen-cpp/complex_constants.cpp | 2 +- .../src/gen/thrift/gen-cpp/complex_constants.h | 2 +- serde/src/gen/thrift/gen-cpp/complex_types.cpp | 442 +- serde/src/gen/thrift/gen-cpp/complex_types.h | 174 +- .../gen/thrift/gen-cpp/megastruct_constants.cpp | 2 +- .../gen/thrift/gen-cpp/megastruct_constants.h | 2 +- .../src/gen/thrift/gen-cpp/megastruct_types.cpp | 585 +- serde/src/gen/thrift/gen-cpp/megastruct_types.h | 175 +- .../src/gen/thrift/gen-cpp/serde_constants.cpp | 2 +- serde/src/gen/thrift/gen-cpp/serde_constants.h | 2 +- serde/src/gen/thrift/gen-cpp/serde_types.cpp | 5 +- serde/src/gen/thrift/gen-cpp/serde_types.h | 5 +- .../gen/thrift/gen-cpp/testthrift_constants.cpp | 2 +- .../gen/thrift/gen-cpp/testthrift_constants.h | 2 +- .../src/gen/thrift/gen-cpp/testthrift_types.cpp | 95 +- serde/src/gen/thrift/gen-cpp/testthrift_types.h | 45 +- .../hadoop/hive/serde/serdeConstants.java | 7 +- .../hadoop/hive/serde/test/InnerStruct.java | 24 +- .../hadoop/hive/serde/test/ThriftTestObj.java | 62 +- .../hadoop/hive/serde2/thrift/test/Complex.java | 279 +- .../hive/serde2/thrift/test/IntString.java | 40 +- .../hive/serde2/thrift/test/MegaStruct.java | 521 +- .../hive/serde2/thrift/test/MiniStruct.java | 38 +- .../hadoop/hive/serde2/thrift/test/MyEnum.java | 2 +- .../hive/serde2/thrift/test/PropValueUnion.java | 60 +- .../hive/serde2/thrift/test/SetIntString.java | 54 +- serde/src/gen/thrift/gen-php/Types.php | 15 +- .../org/apache/hadoop/hive/serde/Types.php | 373 +- .../src/gen/thrift/gen-py/complex/constants.py | 2 +- serde/src/gen/thrift/gen-py/complex/ttypes.py | 50 +- .../gen/thrift/gen-py/megastruct/constants.py | 2 +- .../src/gen/thrift/gen-py/megastruct/ttypes.py | 44 +- .../org_apache_hadoop_hive_serde/constants.py | 2 +- .../org_apache_hadoop_hive_serde/ttypes.py | 2 +- .../gen/thrift/gen-py/testthrift/constants.py | 2 +- .../src/gen/thrift/gen-py/testthrift/ttypes.py | 14 +- .../src/gen/thrift/gen-rb/complex_constants.rb | 2 +- serde/src/gen/thrift/gen-rb/complex_types.rb | 2 +- .../gen/thrift/gen-rb/megastruct_constants.rb | 2 +- serde/src/gen/thrift/gen-rb/megastruct_types.rb | 2 +- serde/src/gen/thrift/gen-rb/serde_constants.rb | 2 +- serde/src/gen/thrift/gen-rb/serde_types.rb | 2 +- .../gen/thrift/gen-rb/testthrift_constants.rb | 2 +- serde/src/gen/thrift/gen-rb/testthrift_types.rb | 2 +- .../hadoop/hive/ql/io/sarg/PredicateLeaf.java | 87 - .../hadoop/hive/ql/io/sarg/SearchArgument.java | 280 - .../hive/serde2/AbstractEncodingAwareSerDe.java | 7 +- .../apache/hadoop/hive/serde2/SerDeUtils.java | 14 +- .../hadoop/hive/serde2/avro/InstanceCache.java | 9 +- .../hadoop/hive/serde2/io/DateWritable.java | 14 - .../hive/serde2/io/HiveDecimalWritable.java | 185 - .../hive/serde2/lazy/LazyHiveDecimal.java | 3 +- .../lazy/fast/LazySimpleDeserializeRead.java | 16 +- .../hive/serde2/lazybinary/LazyBinaryDate.java | 4 +- .../lazybinary/LazyBinaryHiveDecimal.java | 2 +- .../hive/serde2/lazybinary/LazyBinarySerDe.java | 37 +- .../fast/LazyBinaryDeserializeRead.java | 6 +- .../fast/LazyBinarySerializeWrite.java | 5 +- .../objectinspector/ObjectInspectorUtils.java | 18 +- .../hive/serde2/typeinfo/HiveDecimalUtils.java | 35 +- .../hive/serde2/avro/TestInstanceCache.java | 40 +- .../TestObjectInspectorUtils.java | 22 + service/src/gen/thrift/gen-cpp/TCLIService.cpp | 458 +- service/src/gen/thrift/gen-cpp/TCLIService.h | 821 +- .../thrift/gen-cpp/TCLIService_constants.cpp | 2 +- .../gen/thrift/gen-cpp/TCLIService_constants.h | 2 +- .../gen/thrift/gen-cpp/TCLIService_types.cpp | 3250 +- .../src/gen/thrift/gen-cpp/TCLIService_types.h | 1482 +- service/src/gen/thrift/gen-cpp/ThriftHive.cpp | 286 +- service/src/gen/thrift/gen-cpp/ThriftHive.h | 389 +- .../thrift/gen-cpp/hive_service_constants.cpp | 2 +- .../gen/thrift/gen-cpp/hive_service_constants.h | 2 +- .../gen/thrift/gen-cpp/hive_service_types.cpp | 110 +- .../src/gen/thrift/gen-cpp/hive_service_types.h | 75 +- .../hadoop/hive/service/HiveClusterStatus.java | 68 +- .../hive/service/HiveServerException.java | 40 +- .../hadoop/hive/service/JobTrackerState.java | 2 +- .../apache/hadoop/hive/service/ThriftHive.java | 914 +- .../service/cli/thrift/TArrayTypeEntry.java | 24 +- .../hive/service/cli/thrift/TBinaryColumn.java | 64 +- .../hive/service/cli/thrift/TBoolColumn.java | 62 +- .../hive/service/cli/thrift/TBoolValue.java | 26 +- .../hive/service/cli/thrift/TByteColumn.java | 62 +- .../hive/service/cli/thrift/TByteValue.java | 26 +- .../hive/service/cli/thrift/TCLIService.java | 1734 +- .../cli/thrift/TCLIServiceConstants.java | 7 +- .../cli/thrift/TCancelDelegationTokenReq.java | 32 +- .../cli/thrift/TCancelDelegationTokenResp.java | 24 +- .../service/cli/thrift/TCancelOperationReq.java | 24 +- .../cli/thrift/TCancelOperationResp.java | 24 +- .../service/cli/thrift/TCloseOperationReq.java | 24 +- .../service/cli/thrift/TCloseOperationResp.java | 24 +- .../service/cli/thrift/TCloseSessionReq.java | 24 +- .../service/cli/thrift/TCloseSessionResp.java | 24 +- .../apache/hive/service/cli/thrift/TColumn.java | 20 +- .../hive/service/cli/thrift/TColumnDesc.java | 50 +- .../hive/service/cli/thrift/TColumnValue.java | 20 +- .../hive/service/cli/thrift/TDoubleColumn.java | 62 +- .../hive/service/cli/thrift/TDoubleValue.java | 26 +- .../cli/thrift/TExecuteStatementReq.java | 87 +- .../cli/thrift/TExecuteStatementResp.java | 34 +- .../service/cli/thrift/TFetchOrientation.java | 2 +- .../service/cli/thrift/TFetchResultsReq.java | 54 +- .../service/cli/thrift/TFetchResultsResp.java | 42 +- .../service/cli/thrift/TGetCatalogsReq.java | 24 +- .../service/cli/thrift/TGetCatalogsResp.java | 34 +- .../hive/service/cli/thrift/TGetColumnsReq.java | 58 +- .../service/cli/thrift/TGetColumnsResp.java | 34 +- .../cli/thrift/TGetDelegationTokenReq.java | 40 +- .../cli/thrift/TGetDelegationTokenResp.java | 34 +- .../service/cli/thrift/TGetFunctionsReq.java | 50 +- .../service/cli/thrift/TGetFunctionsResp.java | 34 +- .../hive/service/cli/thrift/TGetInfoReq.java | 36 +- .../hive/service/cli/thrift/TGetInfoResp.java | 32 +- .../hive/service/cli/thrift/TGetInfoType.java | 2 +- .../hive/service/cli/thrift/TGetInfoValue.java | 20 +- .../cli/thrift/TGetOperationStatusReq.java | 24 +- .../cli/thrift/TGetOperationStatusResp.java | 62 +- .../cli/thrift/TGetResultSetMetadataReq.java | 24 +- .../cli/thrift/TGetResultSetMetadataResp.java | 34 +- .../hive/service/cli/thrift/TGetSchemasReq.java | 42 +- .../service/cli/thrift/TGetSchemasResp.java | 34 +- .../service/cli/thrift/TGetTableTypesReq.java | 24 +- .../service/cli/thrift/TGetTableTypesResp.java | 34 +- .../hive/service/cli/thrift/TGetTablesReq.java | 79 +- .../hive/service/cli/thrift/TGetTablesResp.java | 34 +- .../service/cli/thrift/TGetTypeInfoReq.java | 24 +- .../service/cli/thrift/TGetTypeInfoResp.java | 34 +- .../service/cli/thrift/THandleIdentifier.java | 50 +- .../hive/service/cli/thrift/TI16Column.java | 62 +- .../hive/service/cli/thrift/TI16Value.java | 26 +- .../hive/service/cli/thrift/TI32Column.java | 62 +- .../hive/service/cli/thrift/TI32Value.java | 26 +- .../hive/service/cli/thrift/TI64Column.java | 62 +- .../hive/service/cli/thrift/TI64Value.java | 26 +- .../hive/service/cli/thrift/TMapTypeEntry.java | 32 +- .../service/cli/thrift/TOpenSessionReq.java | 91 +- .../service/cli/thrift/TOpenSessionResp.java | 91 +- .../service/cli/thrift/TOperationHandle.java | 54 +- .../service/cli/thrift/TOperationState.java | 2 +- .../hive/service/cli/thrift/TOperationType.java | 2 +- .../service/cli/thrift/TPrimitiveTypeEntry.java | 38 +- .../service/cli/thrift/TProtocolVersion.java | 2 +- .../cli/thrift/TRenewDelegationTokenReq.java | 32 +- .../cli/thrift/TRenewDelegationTokenResp.java | 24 +- .../apache/hive/service/cli/thrift/TRow.java | 46 +- .../apache/hive/service/cli/thrift/TRowSet.java | 86 +- .../hive/service/cli/thrift/TSessionHandle.java | 24 +- .../apache/hive/service/cli/thrift/TStatus.java | 83 +- .../hive/service/cli/thrift/TStatusCode.java | 2 +- .../hive/service/cli/thrift/TStringColumn.java | 62 +- .../hive/service/cli/thrift/TStringValue.java | 26 +- .../service/cli/thrift/TStructTypeEntry.java | 50 +- .../hive/service/cli/thrift/TTableSchema.java | 46 +- .../hive/service/cli/thrift/TTypeDesc.java | 46 +- .../hive/service/cli/thrift/TTypeEntry.java | 20 +- .../apache/hive/service/cli/thrift/TTypeId.java | 2 +- .../service/cli/thrift/TTypeQualifierValue.java | 20 +- .../service/cli/thrift/TTypeQualifiers.java | 54 +- .../service/cli/thrift/TUnionTypeEntry.java | 50 +- .../cli/thrift/TUserDefinedTypeEntry.java | 24 +- service/src/gen/thrift/gen-php/TCLIService.php | 269 +- service/src/gen/thrift/gen-php/ThriftHive.php | 125 +- service/src/gen/thrift/gen-php/Types.php | 30 +- .../gen-py/TCLIService/TCLIService-remote | 102 +- .../thrift/gen-py/TCLIService/TCLIService.py | 439 +- .../gen/thrift/gen-py/TCLIService/constants.py | 2 +- .../src/gen/thrift/gen-py/TCLIService/ttypes.py | 478 +- .../gen-py/hive_service/ThriftHive-remote | 1021 +- .../thrift/gen-py/hive_service/ThriftHive.py | 266 +- .../gen/thrift/gen-py/hive_service/constants.py | 2 +- .../gen/thrift/gen-py/hive_service/ttypes.py | 19 +- .../gen/thrift/gen-rb/hive_service_constants.rb | 2 +- .../src/gen/thrift/gen-rb/hive_service_types.rb | 2 +- .../src/gen/thrift/gen-rb/t_c_l_i_service.rb | 2 +- .../thrift/gen-rb/t_c_l_i_service_constants.rb | 2 +- .../gen/thrift/gen-rb/t_c_l_i_service_types.rb | 2 +- service/src/gen/thrift/gen-rb/thrift_hive.rb | 2 +- .../cli/operation/GetColumnsOperation.java | 10 +- .../cli/operation/GetTablesOperation.java | 7 +- .../cli/session/HiveSessionImplwithUGI.java | 2 +- .../cli/thrift/ThriftBinaryCLIService.java | 1 + .../service/cli/thrift/ThriftCLIService.java | 21 +- spark-client/pom.xml | 5 + .../hive/spark/client/SparkClientImpl.java | 20 +- .../hive/spark/client/SparkClientUtilities.java | 13 +- .../hive/spark/client/TestSparkClient.java | 4 +- storage-api/pom.xml | 85 + .../hadoop/hive/common/type/HiveDecimal.java | 312 + .../hive/ql/exec/vector/BytesColumnVector.java | 331 + .../hive/ql/exec/vector/ColumnVector.java | 174 + .../ql/exec/vector/DecimalColumnVector.java | 126 + .../hive/ql/exec/vector/DoubleColumnVector.java | 152 + .../hive/ql/exec/vector/LongColumnVector.java | 198 + .../hive/ql/exec/vector/VectorizedRowBatch.java | 186 + .../hadoop/hive/ql/io/sarg/ExpressionTree.java | 156 + .../hadoop/hive/ql/io/sarg/PredicateLeaf.java | 104 + .../hadoop/hive/ql/io/sarg/SearchArgument.java | 287 + .../hive/ql/io/sarg/SearchArgumentFactory.java | 28 + .../hive/ql/io/sarg/SearchArgumentImpl.java | 687 + .../hive/serde2/io/HiveDecimalWritable.java | 174 + 1299 files changed, 117903 insertions(+), 50790 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hive/blob/c8ae1fbf/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java ---------------------------------------------------------------------- diff --cc common/src/java/org/apache/hadoop/hive/conf/HiveConf.java index 202836f,fe4eb9f..864a0af --- a/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java +++ b/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java @@@ -2111,11 -2108,9 +2114,11 @@@ public class HiveConf extends Configura "Whether to send the query plan via local resource or RPC"), HIVE_AM_SPLIT_GENERATION("hive.compute.splits.in.am", true, "Whether to generate the splits locally or in the AM (tez only)"), + HIVE_TEZ_GENERATE_CONSISTENT_SPLITS("hive.tez.input.generate.consistent.splits", true, "Whether to generate consisten split" + + "locations when generating splits in the AM"), - HIVE_PREWARM_ENABLED("hive.prewarm.enabled", false, "Enables container prewarm for Tez (Hadoop 2 only)"), - HIVE_PREWARM_NUM_CONTAINERS("hive.prewarm.numcontainers", 10, "Controls the number of containers to prewarm for Tez (Hadoop 2 only)"), + HIVE_PREWARM_ENABLED("hive.prewarm.enabled", false, "Enables container prewarm for Tez/Spark (Hadoop 2 only)"), + HIVE_PREWARM_NUM_CONTAINERS("hive.prewarm.numcontainers", 10, "Controls the number of containers to prewarm for Tez/Spark (Hadoop 2 only)"), HIVESTAGEIDREARRANGE("hive.stageid.rearrange", "none", new StringSet("none", "idonly", "traverse", "execution"), ""), HIVEEXPLAINDEPENDENCYAPPENDTASKTYPES("hive.explain.dependency.append.tasktype", false, ""), http://git-wip-us.apache.org/repos/asf/hive/blob/c8ae1fbf/itests/qtest/pom.xml ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hive/blob/c8ae1fbf/itests/src/test/resources/testconfiguration.properties ----------------------------------------------------------------------