Author: gunther Date: Fri Dec 20 02:56:24 2013 New Revision: 1552482 URL: http://svn.apache.org/r1552482 Log: HIVE-6078: Choosing conditional task for merging files is not deterministic in tez (Vikram Dixit K via Gunther Hagleitner)
Modified: hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/optimizer/GenMapRedUtils.java Modified: hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/optimizer/GenMapRedUtils.java URL: http://svn.apache.org/viewvc/hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/optimizer/GenMapRedUtils.java?rev=1552482&r1=1552481&r2=1552482&view=diff ============================================================================== --- hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/optimizer/GenMapRedUtils.java (original) +++ hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/optimizer/GenMapRedUtils.java Fri Dec 20 02:56:24 2013 @@ -1628,15 +1628,21 @@ public final class GenMapRedUtils { // There are separate configuration parameters to control whether to // merge for a map-only job // or for a map-reduce job - ReduceWork reduceWork = currTask.getWork() instanceof MapredWork - ? ((MapredWork) currTask.getWork()).getReduceWork() : null; - boolean mergeMapOnly = - hconf.getBoolVar(ConfVars.HIVEMERGEMAPFILES) && reduceWork == null; - boolean mergeMapRed = - hconf.getBoolVar(ConfVars.HIVEMERGEMAPREDFILES) && - reduceWork != null; - if (mergeMapOnly || mergeMapRed) { - return true; + if (currTask.getWork() instanceof TezWork) { + return hconf.getBoolVar(ConfVars.HIVEMERGEMAPFILES) || + hconf.getBoolVar(ConfVars.HIVEMERGEMAPREDFILES); + } else if (currTask.getWork() instanceof MapredWork) { + ReduceWork reduceWork = ((MapredWork) currTask.getWork()).getReduceWork(); + boolean mergeMapOnly = + hconf.getBoolVar(ConfVars.HIVEMERGEMAPFILES) && reduceWork == null; + boolean mergeMapRed = + hconf.getBoolVar(ConfVars.HIVEMERGEMAPREDFILES) && + reduceWork != null; + if (mergeMapOnly || mergeMapRed) { + return true; + } + } else { + return false; } } }