Author: gates Date: Tue Sep 9 12:32:38 2008 New Revision: 693574 URL: http://svn.apache.org/viewvc?rev=693574&view=rev Log: PIG-416 Fixed LogicalTransformer so that when a script with a split needs to be reorganized after an optimization, the LOSplitOutput is properly patched up. Previously it was getting skipped.
Fixed PigServer so that explain will work after a split. Removed debug statement from PigMapReduce that was printing every line out as it went into reduce. Modified: incubator/pig/branches/types/src/org/apache/pig/PigServer.java incubator/pig/branches/types/src/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/PigMapReduce.java incubator/pig/branches/types/src/org/apache/pig/impl/logicalLayer/optimizer/LogicalTransformer.java Modified: incubator/pig/branches/types/src/org/apache/pig/PigServer.java URL: http://svn.apache.org/viewvc/incubator/pig/branches/types/src/org/apache/pig/PigServer.java?rev=693574&r1=693573&r2=693574&view=diff ============================================================================== --- incubator/pig/branches/types/src/org/apache/pig/PigServer.java (original) +++ incubator/pig/branches/types/src/org/apache/pig/PigServer.java Tue Sep 9 12:32:38 2008 @@ -398,7 +398,11 @@ if(null == op) { throw new IOException("Unable to find an operator for alias " + alias); } - LogicalPlan lp = compileLp(getPlanFromAlias(alias, op.getClass().getName()), "explain"); + LogicalPlan storePlan = QueryParser.generateStorePlan(opTable, + scope, getPlanFromAlias(alias, op.getClass().getName()), + "fakefile", PigStorage.class.getName(), aliasOp.get(alias), + aliases); + LogicalPlan lp = compileLp(storePlan, "explain"); stream.println("Logical Plan:"); LOPrinter lv = new LOPrinter(stream, lp); lv.visit(); Modified: incubator/pig/branches/types/src/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/PigMapReduce.java URL: http://svn.apache.org/viewvc/incubator/pig/branches/types/src/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/PigMapReduce.java?rev=693574&r1=693573&r2=693574&view=diff ============================================================================== --- incubator/pig/branches/types/src/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/PigMapReduce.java (original) +++ incubator/pig/branches/types/src/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/PigMapReduce.java Tue Sep 9 12:32:38 2008 @@ -167,7 +167,6 @@ return; } - log.info("Attaching " + packRes + " to " + rp.getRoots()); rp.attachInput(packRes); List<PhysicalOperator> leaves = rp.getLeaves(); Modified: incubator/pig/branches/types/src/org/apache/pig/impl/logicalLayer/optimizer/LogicalTransformer.java URL: http://svn.apache.org/viewvc/incubator/pig/branches/types/src/org/apache/pig/impl/logicalLayer/optimizer/LogicalTransformer.java?rev=693574&r1=693573&r2=693574&view=diff ============================================================================== --- incubator/pig/branches/types/src/org/apache/pig/impl/logicalLayer/optimizer/LogicalTransformer.java (original) +++ incubator/pig/branches/types/src/org/apache/pig/impl/logicalLayer/optimizer/LogicalTransformer.java Tue Sep 9 12:32:38 2008 @@ -21,6 +21,9 @@ import java.util.List; import java.util.Map; +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; + import org.apache.pig.impl.plan.DepthFirstWalker; import org.apache.pig.impl.plan.PlanWalker; import org.apache.pig.impl.plan.VisitorException; @@ -41,6 +44,8 @@ public abstract class LogicalTransformer extends Transformer<LogicalOperator, LogicalPlan> { + private final Log log = LogFactory.getLog(getClass()); + protected LogicalTransformer( LogicalPlan plan, PlanWalker<LogicalOperator, LogicalPlan> walker) { @@ -185,8 +190,19 @@ plans.addAll(((LOSort)before).getSortColPlans()); } else if (before instanceof LOFilter) { plans.add(((LOFilter)before).getComparisonPlan()); - } else if (before instanceof LOSplitOutput) { - plans.add(((LOSplitOutput)before).getConditionPlan()); + } else if (before instanceof LOSplit) { + // In this case we have to find each of the Split outputs, and + // add their plans. + List<LogicalOperator> splitOutputs = mPlan.getSuccessors(before); + for (LogicalOperator so : splitOutputs) { + if (!(so instanceof LOSplitOutput)) { + String msg = "Found an LOSplit with an operator other " + + "than LOSplitOutput after it!"; + log.error(msg); + throw new VisitorException(msg); + } + plans.add(((LOSplitOutput)so).getConditionPlan()); + } } else if (before instanceof LOForEach) { plans.addAll(((LOForEach)before).getForEachPlans()); }