Repository: hive Updated Branches: refs/heads/master a0bf9d629 -> 8e0b19b3c
HIVE-14892: Test that explicitly submit jobs via child process are slow (Prasanth Jayachandran reviewed by Siddharth Seth) Project: http://git-wip-us.apache.org/repos/asf/hive/repo Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/8e0b19b3 Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/8e0b19b3 Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/8e0b19b3 Branch: refs/heads/master Commit: 8e0b19b3c469c1e0d4165cc1e4652093195fc587 Parents: a0bf9d6 Author: Prasanth Jayachandran <prasan...@apache.org> Authored: Wed Oct 5 21:12:08 2016 -0700 Committer: Prasanth Jayachandran <prasan...@apache.org> Committed: Wed Oct 5 21:12:08 2016 -0700 ---------------------------------------------------------------------- .../apache/hadoop/hive/ql/exec/mr/MapRedTask.java | 17 ++++++++++++++++- .../queries/clientpositive/alter_merge_stats_orc.q | 4 ++-- .../clientpositive/archive_excludeHadoop20.q | 5 ++--- .../queries/clientpositive/auto_sortmerge_join_8.q | 4 ++-- ql/src/test/queries/clientpositive/nonmr_fetch.q | 4 ++-- ql/src/test/queries/clientpositive/orc_analyze.q | 4 ++-- ql/src/test/queries/clientpositive/sample10.q | 4 ++-- .../clientpositive/sample_islocalmode_hook.q | 4 ++-- .../sample_islocalmode_hook_use_metadata.q | 6 +++--- .../queries/clientpositive/vectorized_parquet.q | 4 ++-- 10 files changed, 35 insertions(+), 21 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hive/blob/8e0b19b3/ql/src/java/org/apache/hadoop/hive/ql/exec/mr/MapRedTask.java ---------------------------------------------------------------------- diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/mr/MapRedTask.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/mr/MapRedTask.java index 55bab6c..17db852 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/exec/mr/MapRedTask.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/mr/MapRedTask.java @@ -138,9 +138,24 @@ public class MapRedTask extends ExecDriver implements Serializable { runningViaChild = conf.getBoolVar(HiveConf.ConfVars.SUBMITVIACHILD); if (!runningViaChild) { + // since we are running the mapred task in the same jvm, we should update the job conf + // in ExecDriver as well to have proper local properties. + if (this.isLocalMode()) { + // save the original job tracker + ctx.setOriginalTracker(ShimLoader.getHadoopShims().getJobLauncherRpcAddress(job)); + // change it to local + ShimLoader.getHadoopShims().setJobLauncherRpcAddress(job, "local"); + } // we are not running this mapred task via child jvm // so directly invoke ExecDriver - return super.execute(driverContext); + int ret = super.execute(driverContext); + + // restore the previous properties for framework name, RM address etc. + if (this.isLocalMode()) { + // restore the local job tracker back to original + ctx.restoreOriginalTracker(); + } + return ret; } // we need to edit the configuration to setup cmdline. clone it first http://git-wip-us.apache.org/repos/asf/hive/blob/8e0b19b3/ql/src/test/queries/clientpositive/alter_merge_stats_orc.q ---------------------------------------------------------------------- diff --git a/ql/src/test/queries/clientpositive/alter_merge_stats_orc.q b/ql/src/test/queries/clientpositive/alter_merge_stats_orc.q index 66a7c10..8d0c7bd 100644 --- a/ql/src/test/queries/clientpositive/alter_merge_stats_orc.q +++ b/ql/src/test/queries/clientpositive/alter_merge_stats_orc.q @@ -1,5 +1,5 @@ -set hive.exec.submitviachild=true; -set hive.exec.submit.local.task.via.child=true; +set hive.exec.submitviachild=false; +set hive.exec.submit.local.task.via.child=false; create table src_orc_merge_test_stat(key int, value string) stored as orc; insert overwrite table src_orc_merge_test_stat select * from src; http://git-wip-us.apache.org/repos/asf/hive/blob/8e0b19b3/ql/src/test/queries/clientpositive/archive_excludeHadoop20.q ---------------------------------------------------------------------- diff --git a/ql/src/test/queries/clientpositive/archive_excludeHadoop20.q b/ql/src/test/queries/clientpositive/archive_excludeHadoop20.q index 37d5cf9..e961641 100644 --- a/ql/src/test/queries/clientpositive/archive_excludeHadoop20.q +++ b/ql/src/test/queries/clientpositive/archive_excludeHadoop20.q @@ -1,8 +1,7 @@ set hive.mapred.mode=nonstrict; set hive.archive.enabled = true; -; -set hive.exec.submitviachild=true; -set hive.exec.submit.local.task.via.child=true; +set hive.exec.submitviachild=false; +set hive.exec.submit.local.task.via.child=false; drop table tstsrc; drop table tstsrcpart; http://git-wip-us.apache.org/repos/asf/hive/blob/8e0b19b3/ql/src/test/queries/clientpositive/auto_sortmerge_join_8.q ---------------------------------------------------------------------- diff --git a/ql/src/test/queries/clientpositive/auto_sortmerge_join_8.q b/ql/src/test/queries/clientpositive/auto_sortmerge_join_8.q index 1bd5da4..cc2ba28 100644 --- a/ql/src/test/queries/clientpositive/auto_sortmerge_join_8.q +++ b/ql/src/test/queries/clientpositive/auto_sortmerge_join_8.q @@ -1,6 +1,6 @@ set hive.mapred.mode=nonstrict; -set hive.exec.submitviachild=true; -set hive.exec.submit.local.task.via.child=true; +set hive.exec.submitviachild=false; +set hive.exec.submit.local.task.via.child=false; -- small 2 part, 2 bucket & big 2 part, 4 bucket CREATE TABLE bucket_small (key string, value string) partitioned by (ds string) CLUSTERED BY (key) SORTED BY (key) INTO 2 BUCKETS STORED AS TEXTFILE; http://git-wip-us.apache.org/repos/asf/hive/blob/8e0b19b3/ql/src/test/queries/clientpositive/nonmr_fetch.q ---------------------------------------------------------------------- diff --git a/ql/src/test/queries/clientpositive/nonmr_fetch.q b/ql/src/test/queries/clientpositive/nonmr_fetch.q index a446217..dffdc2c 100644 --- a/ql/src/test/queries/clientpositive/nonmr_fetch.q +++ b/ql/src/test/queries/clientpositive/nonmr_fetch.q @@ -1,7 +1,7 @@ set hive.mapred.mode=nonstrict; set hive.fetch.task.conversion=minimal; -set hive.exec.submitviachild=true; -set hive.exec.submit.local.task.via.child=true; +set hive.exec.submitviachild=false; +set hive.exec.submit.local.task.via.child=false; -- backward compatible (minimal) explain select * from src limit 10; http://git-wip-us.apache.org/repos/asf/hive/blob/8e0b19b3/ql/src/test/queries/clientpositive/orc_analyze.q ---------------------------------------------------------------------- diff --git a/ql/src/test/queries/clientpositive/orc_analyze.q b/ql/src/test/queries/clientpositive/orc_analyze.q index 4cbe1e6..3f9a592 100644 --- a/ql/src/test/queries/clientpositive/orc_analyze.q +++ b/ql/src/test/queries/clientpositive/orc_analyze.q @@ -1,6 +1,6 @@ set hive.mapred.mode=nonstrict; -set hive.exec.submitviachild=true; -set hive.exec.submit.local.task.via.child=true; +set hive.exec.submitviachild=false; +set hive.exec.submit.local.task.via.child=false; CREATE TABLE orc_create_people_staging ( id int, http://git-wip-us.apache.org/repos/asf/hive/blob/8e0b19b3/ql/src/test/queries/clientpositive/sample10.q ---------------------------------------------------------------------- diff --git a/ql/src/test/queries/clientpositive/sample10.q b/ql/src/test/queries/clientpositive/sample10.q index b410644..a5a8c8f 100644 --- a/ql/src/test/queries/clientpositive/sample10.q +++ b/ql/src/test/queries/clientpositive/sample10.q @@ -1,6 +1,6 @@ set hive.mapred.mode=nonstrict; -set hive.exec.submitviachild=true; -set hive.exec.submit.local.task.via.child=true; +set hive.exec.submitviachild=false; +set hive.exec.submit.local.task.via.child=false; set hive.exec.dynamic.partition=true; set hive.exec.dynamic.partition.mode=nonstrict; http://git-wip-us.apache.org/repos/asf/hive/blob/8e0b19b3/ql/src/test/queries/clientpositive/sample_islocalmode_hook.q ---------------------------------------------------------------------- diff --git a/ql/src/test/queries/clientpositive/sample_islocalmode_hook.q b/ql/src/test/queries/clientpositive/sample_islocalmode_hook.q index 3f986e0..08f8cdf 100644 --- a/ql/src/test/queries/clientpositive/sample_islocalmode_hook.q +++ b/ql/src/test/queries/clientpositive/sample_islocalmode_hook.q @@ -1,6 +1,6 @@ set hive.mapred.mode=nonstrict; -set hive.exec.submitviachild=true; -set hive.exec.submit.local.task.via.child=true; +set hive.exec.submitviachild=false; +set hive.exec.submit.local.task.via.child=false; set hive.input.format=org.apache.hadoop.hive.ql.io.CombineHiveInputFormat; set mapred.max.split.size=300; set mapred.min.split.size=300; http://git-wip-us.apache.org/repos/asf/hive/blob/8e0b19b3/ql/src/test/queries/clientpositive/sample_islocalmode_hook_use_metadata.q ---------------------------------------------------------------------- diff --git a/ql/src/test/queries/clientpositive/sample_islocalmode_hook_use_metadata.q b/ql/src/test/queries/clientpositive/sample_islocalmode_hook_use_metadata.q index ac915b5..b523f06 100644 --- a/ql/src/test/queries/clientpositive/sample_islocalmode_hook_use_metadata.q +++ b/ql/src/test/queries/clientpositive/sample_islocalmode_hook_use_metadata.q @@ -1,6 +1,6 @@ set hive.mapred.mode=nonstrict; -set hive.exec.submitviachild=true; -set hive.exec.submit.local.task.via.child=true; +set hive.exec.submitviachild=false; +set hive.exec.submit.local.task.via.child=false; set hive.input.format=org.apache.hadoop.hive.ql.io.CombineHiveInputFormat; set mapred.max.split.size=300; set mapred.min.split.size=300; @@ -45,4 +45,4 @@ select count(1) from sih_src tablesample(1 percent); explain select count(1) from sih_src tablesample(10 rows); -select count(1) from sih_src tablesample(10 rows); \ No newline at end of file +select count(1) from sih_src tablesample(10 rows); http://git-wip-us.apache.org/repos/asf/hive/blob/8e0b19b3/ql/src/test/queries/clientpositive/vectorized_parquet.q ---------------------------------------------------------------------- diff --git a/ql/src/test/queries/clientpositive/vectorized_parquet.q b/ql/src/test/queries/clientpositive/vectorized_parquet.q index a49ca63..da138e0 100644 --- a/ql/src/test/queries/clientpositive/vectorized_parquet.q +++ b/ql/src/test/queries/clientpositive/vectorized_parquet.q @@ -1,6 +1,6 @@ set hive.explain.user=false; -set hive.exec.submitviachild=true; -set hive.exec.submit.local.task.via.child=true; +set hive.exec.submitviachild=false; +set hive.exec.submit.local.task.via.child=false; create table if not exists alltypes_parquet ( cint int,
