HIVE-13172: Merge master into llap branch
Project: http://git-wip-us.apache.org/repos/asf/hive/repo Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/f6be2180 Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/f6be2180 Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/f6be2180 Branch: refs/heads/master Commit: f6be218067855bb0e6893d71531bfd9c3ce835f6 Parents: 57761e3 157d825 Author: Jason Dere <jd...@hortonworks.com> Authored: Fri Feb 26 15:08:45 2016 -0800 Committer: Jason Dere <jd...@hortonworks.com> Committed: Fri Feb 26 15:08:45 2016 -0800 ---------------------------------------------------------------------- .../org/apache/hadoop/hive/conf/HiveConf.java | 11 +- .../streaming/mutate/StreamingTestUtils.java | 9 +- .../streaming/mutate/TestMutations.java | 6 +- .../hive/metastore/TestHiveMetaStore.java | 128 +- .../hive/ql/txn/compactor/TestCompactor.java | 3 +- itests/qtest/pom.xml | 4 +- .../test/resources/testconfiguration.properties | 6 +- .../hive/llap/registry/ServiceInstanceSet.java | 7 - .../ServiceInstanceStateChangeListener.java | 42 + .../hive/llap/registry/ServiceRegistry.java | 27 +- .../registry/impl/LlapFixedRegistryImpl.java | 18 +- .../llap/registry/impl/LlapRegistryService.java | 8 +- .../registry/impl/LlapYarnRegistryImpl.java | 438 - .../impl/LlapZookeeperRegistryImpl.java | 672 ++ .../hive/llap/tez/LlapProtocolClientProxy.java | 6 + .../hive/llap/security/LlapTokenIdentifier.java | 2 +- llap-server/bin/runLlapDaemon.sh | 2 +- .../hadoop/hive/llap/cli/LlapServiceDriver.java | 38 +- .../configuration/LlapDaemonConfiguration.java | 12 +- .../hive/llap/daemon/impl/LlapDaemon.java | 8 + .../daemon/impl/LlapProtocolServerImpl.java | 3 + .../llap/daemon/services/impl/LlapWebApp.java | 29 - .../daemon/services/impl/LlapWebServices.java | 11 +- .../hive/llap/security/LlapSecurityHelper.java | 24 +- .../main/resources/hive-webapps/llap/index.html | 72 +- .../resources/hive-webapps/llap/js/metrics.js | 279 +- .../llap/tezplugins/LlapTaskCommunicator.java | 3 + .../tezplugins/LlapTaskSchedulerService.java | 71 +- .../apache/hadoop/hive/metastore/Deadline.java | 63 +- .../hadoop/hive/metastore/HiveMetaStore.java | 1 + .../hadoop/hive/metastore/RawStoreProxy.java | 28 +- .../hive/metastore/RetryingHMSHandler.java | 12 +- .../metastore/SessionPropertiesListener.java | 9 +- .../TransactionalValidationListener.java | 179 + packaging/pom.xml | 43 + packaging/src/deb/control/control | 23 + packaging/src/main/assembly/bin.xml | 10 + .../hadoop/hive/ql/exec/FunctionRegistry.java | 15 + .../hadoop/hive/ql/exec/FunctionTask.java | 10 +- .../hadoop/hive/ql/exec/MapJoinOperator.java | 7 + .../apache/hadoop/hive/ql/exec/Registry.java | 44 + .../hadoop/hive/ql/exec/tez/DagUtils.java | 5 +- .../hive/ql/exec/tez/TezSessionState.java | 6 +- .../apache/hadoop/hive/ql/exec/tez/TezTask.java | 7 +- .../hadoop/hive/ql/io/NullScanFileSystem.java | 15 + .../hadoop/hive/ql/io/orc/OrcInputFormat.java | 117 +- .../ql/optimizer/calcite/HiveCalciteUtil.java | 110 +- .../hive/ql/optimizer/calcite/HiveRexUtil.java | 78 +- .../rules/HiveFilterProjectTransposeRule.java | 152 +- .../HiveJoinPushTransitivePredicatesRule.java | 2 +- .../rules/HiveReduceExpressionsRule.java | 12 +- .../calcite/stats/HiveRelMdPredicates.java | 40 +- .../hive/ql/optimizer/physical/LlapDecider.java | 77 +- .../hadoop/hive/ql/parse/CalcitePlanner.java | 3 +- .../org/apache/hadoop/hive/ql/parse/HiveLexer.g | 3 + .../hadoop/hive/ql/parse/SemanticAnalyzer.java | 6 +- .../apache/hadoop/hive/ql/plan/PlanUtils.java | 3 + .../org/apache/hadoop/hive/ql/plan/TezWork.java | 6 +- .../hive/ql/io/orc/TestInputOutputFormat.java | 225 +- .../queries/clientnegative/create_not_acid.q | 6 + .../clientnegative/delete_not_bucketed.q | 7 - .../clientnegative/orc_change_fileformat_acid.q | 2 +- .../clientnegative/orc_change_serde_acid.q | 2 +- .../clientnegative/orc_reorder_columns1_acid.q | 2 +- .../clientnegative/orc_reorder_columns2_acid.q | 2 +- .../clientnegative/orc_replace_columns1_acid.q | 2 +- .../clientnegative/orc_replace_columns2_acid.q | 2 +- .../clientnegative/orc_replace_columns3_acid.q | 2 +- .../clientnegative/orc_type_promotion1_acid.q | 2 +- .../clientnegative/orc_type_promotion2_acid.q | 2 +- .../clientnegative/orc_type_promotion3_acid.q | 2 +- .../clientnegative/update_not_bucketed.q | 7 - ql/src/test/queries/clientpositive/constprog3.q | 8 + ql/src/test/queries/clientpositive/fold_case.q | 3 + .../queries/clientpositive/infer_join_preds.q | 61 + .../clientpositive/insert_overwrite_directory.q | 8 +- ql/src/test/queries/clientpositive/llap_udf.q | 47 + .../test/queries/clientpositive/llapdecider.q | 4 + .../queries/clientpositive/ppd_windowing1.q | 46 + .../queries/clientpositive/ppd_windowing2.q | 49 + .../clientnegative/create_not_acid.q.out | 5 + .../clientnegative/delete_not_bucketed.q.out | 9 - .../orc_change_fileformat_acid.q.out | 4 +- .../clientnegative/orc_change_serde_acid.q.out | 4 +- .../orc_reorder_columns1_acid.q.out | 4 +- .../orc_reorder_columns2_acid.q.out | 4 +- .../orc_replace_columns1_acid.q.out | 4 +- .../orc_replace_columns2_acid.q.out | 4 +- .../orc_replace_columns3_acid.q.out | 4 +- .../orc_type_promotion1_acid.q.out | 4 +- .../orc_type_promotion2_acid.q.out | 4 +- .../orc_type_promotion3_acid.q.out | 4 +- .../clientnegative/update_not_bucketed.q.out | 9 - .../results/clientpositive/constprog3.q.out | 83 + .../clientpositive/correlationoptimizer12.q.out | 68 +- .../test/results/clientpositive/fold_case.q.out | 49 +- .../clientpositive/infer_join_preds.q.out | 619 ++ .../insert_overwrite_directory.q.out | 24 +- .../results/clientpositive/llap/cte_mat_1.q.out | 1 + .../results/clientpositive/llap/cte_mat_2.q.out | 1 + .../results/clientpositive/llap/cte_mat_3.q.out | 1 + .../results/clientpositive/llap/llap_udf.q.out | 307 + .../clientpositive/llap/llapdecider.q.out | 61 + .../test/results/clientpositive/llap/mrr.q.out | 1 + .../results/clientpositive/llap/tez_dml.q.out | 1 + .../llap/tez_dynpart_hashjoin_1.q.out | 1 + .../llap/tez_dynpart_hashjoin_2.q.out | 1 + .../clientpositive/llap/tez_join_tests.q.out | 1 + .../clientpositive/llap/tez_joins_explain.q.out | 1 + .../clientpositive/llap/tez_smb_main.q.out | 1 + .../results/clientpositive/llap/tez_union.q.out | 1 + .../clientpositive/llap/tez_union2.q.out | 1 + .../llap/tez_vector_dynpart_hashjoin_1.q.out | 1 + .../llap/tez_vector_dynpart_hashjoin_2.q.out | 1 + .../test/results/clientpositive/mergejoin.q.out | 2 - .../clientpositive/partition_boolexpr.q.out | 26 +- .../results/clientpositive/perf/query70.q.out | 4 +- .../results/clientpositive/ppd_udf_col.q.out | 20 +- .../results/clientpositive/ppd_windowing1.q.out | 2029 ++++ .../results/clientpositive/ppd_windowing2.q.out | 9841 ++++++++++++++++++ .../results/clientpositive/ptfgroupbyjoin.q.out | 66 +- .../clientpositive/spark/subquery_in.q.out | 15 +- .../spark/vectorization_short_regress.q.out | 4 +- .../results/clientpositive/subquery_in.q.out | 15 +- .../subquery_unqualcolumnrefs.q.out | 30 +- .../results/clientpositive/tez/auto_join0.q.out | 1 + .../clientpositive/tez/auto_join21.q.out | 1 + .../clientpositive/tez/auto_join29.q.out | 1 + .../clientpositive/tez/auto_join30.q.out | 1 + .../tez/correlationoptimizer1.q.out | 1 + .../results/clientpositive/tez/cross_join.q.out | 1 + .../results/clientpositive/tez/having.q.out | 1 + .../clientpositive/tez/join0.q.java1.7.out | 1 + .../clientpositive/tez/limit_pushdown.q.out | 1 + .../clientpositive/tez/llapdecider.q.out | 61 + .../clientpositive/tez/mapjoin_mapjoin.q.out | 1 + .../results/clientpositive/tez/mergejoin.q.out | 3 +- .../test/results/clientpositive/tez/mrr.q.out | 1 + .../test/results/clientpositive/tez/ptf.q.out | 1 + .../clientpositive/tez/ptf_streaming.q.out | 1 + .../clientpositive/tez/script_pipe.q.out | 1 + .../clientpositive/tez/selectDistinctStar.q.out | 1 + .../clientpositive/tez/subquery_exists.q.out | 1 + .../clientpositive/tez/subquery_in.q.out | 16 +- .../results/clientpositive/tez/tez_dml.q.out | 1 + .../tez/tez_dynpart_hashjoin_1.q.out | 1 + .../tez/tez_dynpart_hashjoin_2.q.out | 1 + .../tez/tez_dynpart_hashjoin_3.q.out | 1 + .../clientpositive/tez/tez_join_tests.q.out | 1 + .../clientpositive/tez/tez_joins_explain.q.out | 1 + .../clientpositive/tez/tez_smb_main.q.out | 1 + .../results/clientpositive/tez/tez_union.q.out | 1 + .../results/clientpositive/tez/tez_union2.q.out | 1 + .../tez/tez_vector_dynpart_hashjoin_1.q.out | 1 + .../tez/tez_vector_dynpart_hashjoin_2.q.out | 1 + .../clientpositive/tez/transform_ppr1.q.out | 1 + .../clientpositive/tez/transform_ppr2.q.out | 1 + .../results/clientpositive/tez/union2.q.out | 1 + .../results/clientpositive/tez/union3.q.out | 1 + .../results/clientpositive/tez/union5.q.out | 1 + .../results/clientpositive/tez/union7.q.out | 1 + .../results/clientpositive/tez/union8.q.out | 1 + .../results/clientpositive/tez/union9.q.out | 1 + .../clientpositive/tez/vector_coalesce.q.out | 1 + .../clientpositive/tez/vector_if_expr.q.out | 1 + .../tez/vector_left_outer_join.q.out | 1 + .../tez/vector_mapjoin_reduce.q.out | 1 + .../clientpositive/tez/vectorization_0.q.out | 1 + .../clientpositive/tez/vectorization_13.q.out | 1 + .../clientpositive/tez/vectorization_14.q.out | 1 + .../clientpositive/tez/vectorization_15.q.out | 1 + .../clientpositive/tez/vectorization_16.q.out | 1 + .../clientpositive/tez/vectorization_17.q.out | 1 + .../clientpositive/tez/vectorization_7.q.out | 1 + .../clientpositive/tez/vectorization_8.q.out | 1 + .../clientpositive/tez/vectorization_9.q.out | 1 + .../tez/vectorization_pushdown.q.out | 1 + .../tez/vectorization_short_regress.q.out | 5 +- .../clientpositive/tez/vectorized_mapjoin.q.out | 1 + .../tez/vectorized_nested_mapjoin.q.out | 1 + .../tez/vectorized_shufflejoin.q.out | 1 + .../vectorization_short_regress.q.out | 4 +- 182 files changed, 15861 insertions(+), 1136 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hive/blob/f6be2180/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hive/blob/f6be2180/itests/src/test/resources/testconfiguration.properties ---------------------------------------------------------------------- diff --cc itests/src/test/resources/testconfiguration.properties index 13b5113,68dbd0c..603159b --- a/itests/src/test/resources/testconfiguration.properties +++ b/itests/src/test/resources/testconfiguration.properties @@@ -494,9 -494,12 +494,13 @@@ minillap.shared.query.files=bucket_map_ vectorized_dynamic_partition_pruning.q,\ tez_multi_union.q,\ tez_join.q,\ - tez_union_multiinsert.q + tez_union_multiinsert.q,\ + udtf_get_splits.q + + minillap.query.files=llap_udf.q + + encrypted.query.files=encryption_join_unencrypted_tbl.q,\ encryption_insert_partition_static.q,\ encryption_insert_partition_dynamic.q,\ http://git-wip-us.apache.org/repos/asf/hive/blob/f6be2180/llap-server/src/java/org/apache/hadoop/hive/llap/daemon/impl/LlapDaemon.java ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hive/blob/f6be2180/llap-tez/src/java/org/apache/hadoop/hive/llap/tezplugins/LlapTaskCommunicator.java ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hive/blob/f6be2180/llap-tez/src/java/org/apache/hadoop/hive/llap/tezplugins/LlapTaskSchedulerService.java ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hive/blob/f6be2180/ql/src/java/org/apache/hadoop/hive/ql/exec/FunctionRegistry.java ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hive/blob/f6be2180/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/TezTask.java ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hive/blob/f6be2180/ql/src/java/org/apache/hadoop/hive/ql/plan/PlanUtils.java ----------------------------------------------------------------------
