Author: gates Date: Mon Aug 11 21:35:02 2008 New Revision: 685032 URL: http://svn.apache.org/viewvc?rev=685032&view=rev Log: PIG-343 Shravan's patch to fix split.
Modified: incubator/pig/branches/types/src/org/apache/pig/backend/hadoop/executionengine/physicalLayer/LogToPhyTranslationVisitor.java incubator/pig/branches/types/test/org/apache/pig/test/TestLogToPhyCompiler.java incubator/pig/branches/types/test/org/apache/pig/test/data/GoldenFiles/IsNull1.gld incubator/pig/branches/types/test/org/apache/pig/test/data/GoldenFiles/Split1.gld Modified: incubator/pig/branches/types/src/org/apache/pig/backend/hadoop/executionengine/physicalLayer/LogToPhyTranslationVisitor.java URL: http://svn.apache.org/viewvc/incubator/pig/branches/types/src/org/apache/pig/backend/hadoop/executionengine/physicalLayer/LogToPhyTranslationVisitor.java?rev=685032&r1=685031&r2=685032&view=diff ============================================================================== --- incubator/pig/branches/types/src/org/apache/pig/backend/hadoop/executionengine/physicalLayer/LogToPhyTranslationVisitor.java (original) +++ incubator/pig/branches/types/src/org/apache/pig/backend/hadoop/executionengine/physicalLayer/LogToPhyTranslationVisitor.java Mon Aug 11 21:35:02 2008 @@ -17,6 +17,7 @@ */ package org.apache.pig.backend.hadoop.executionengine.physicalLayer; +import java.io.IOException; import java.util.ArrayList; import java.util.HashMap; import java.util.List; @@ -28,6 +29,7 @@ import org.apache.commons.logging.LogFactory; import org.apache.pig.ComparisonFunc; import org.apache.pig.EvalFunc; +import org.apache.pig.FuncSpec; import org.apache.pig.LoadFunc; import org.apache.pig.data.DataType; import org.apache.pig.impl.PigContext; @@ -38,6 +40,9 @@ import org.apache.pig.backend.hadoop.executionengine.physicalLayer.expressionOperators.ExpressionOperator; import org.apache.pig.backend.hadoop.executionengine.physicalLayer.expressionOperators.BinaryExpressionOperator; import org.apache.pig.backend.hadoop.executionengine.physicalLayer.expressionOperators.UnaryExpressionOperator; +import org.apache.pig.builtin.BinStorage; +import org.apache.pig.impl.io.FileLocalizer; +import org.apache.pig.impl.io.FileSpec; import org.apache.pig.impl.logicalLayer.*; import org.apache.pig.impl.plan.DependencyOrderWalker; import org.apache.pig.impl.plan.DependencyOrderWalkerWOSeenChk; @@ -759,6 +764,14 @@ String scope = split.getOperatorKey().scope; PhysicalOperator physOp = new POSplit(new OperatorKey(scope, nodeGen .getNextNodeId(scope)), split.getRequestedParallelism()); + FileSpec splStrFile; + try { + splStrFile = new FileSpec(FileLocalizer.getTemporaryPath(null, pc).toString(),new FuncSpec(BinStorage.class.getName())); + } catch (IOException e1) { + log.error("Unable to obtain a temporary path because " + e1.getMessage()); + throw new VisitorException(e1); + } + ((POSplit)physOp).setSplitStore(splStrFile); LogToPhyMap.put(split, physOp); currentPlan.add(physOp); Modified: incubator/pig/branches/types/test/org/apache/pig/test/TestLogToPhyCompiler.java URL: http://svn.apache.org/viewvc/incubator/pig/branches/types/test/org/apache/pig/test/TestLogToPhyCompiler.java?rev=685032&r1=685031&r2=685032&view=diff ============================================================================== --- incubator/pig/branches/types/test/org/apache/pig/test/TestLogToPhyCompiler.java (original) +++ incubator/pig/branches/types/test/org/apache/pig/test/TestLogToPhyCompiler.java Mon Aug 11 21:35:02 2008 @@ -68,10 +68,16 @@ Random r = new Random(); PigContext pc = new PigContext(ExecType.LOCAL, new Properties()); - + private boolean generate = false; + + @Override + protected void setUp() throws Exception { + pc.connect(); + } + private void writeData(File input, int noTuples, int arityOfTuples, char separator) throws IOException { FileOutputStream dat = new FileOutputStream(input); Modified: incubator/pig/branches/types/test/org/apache/pig/test/data/GoldenFiles/IsNull1.gld URL: http://svn.apache.org/viewvc/incubator/pig/branches/types/test/org/apache/pig/test/data/GoldenFiles/IsNull1.gld?rev=685032&r1=685031&r2=685032&view=diff ============================================================================== --- incubator/pig/branches/types/test/org/apache/pig/test/data/GoldenFiles/IsNull1.gld (original) +++ incubator/pig/branches/types/test/org/apache/pig/test/data/GoldenFiles/IsNull1.gld Mon Aug 11 21:35:02 2008 @@ -1,20 +1,20 @@ Filter[tuple] - Test-Plan-Builder-214 | | -| Not[boolean] - Test-Plan-Builder-217 +| POIsNull[tuple] - Test-Plan-Builder-216 | | -| |---POIsNull[tuple] - Test-Plan-Builder-216 -| | -| |---Project[bytearray][0] - Test-Plan-Builder-215 +| |---Project[bytearray][0] - Test-Plan-Builder-215 | |---Split - Test-Plan-Builder-213 | |---Load(a:org.apache.pig.builtin.PigStorage) - Test-Plan-Builder-212 -Filter[tuple] - Test-Plan-Builder-218 +Filter[tuple] - Test-Plan-Builder-217 | | -| POIsNull[tuple] - Test-Plan-Builder-220 +| Not[boolean] - Test-Plan-Builder-220 | | -| |---Project[bytearray][0] - Test-Plan-Builder-219 +| |---POIsNull[tuple] - Test-Plan-Builder-219 +| | +| |---Project[bytearray][0] - Test-Plan-Builder-218 | |---Split - Test-Plan-Builder-213 | Modified: incubator/pig/branches/types/test/org/apache/pig/test/data/GoldenFiles/Split1.gld URL: http://svn.apache.org/viewvc/incubator/pig/branches/types/test/org/apache/pig/test/data/GoldenFiles/Split1.gld?rev=685032&r1=685031&r2=685032&view=diff ============================================================================== --- incubator/pig/branches/types/test/org/apache/pig/test/data/GoldenFiles/Split1.gld (original) +++ incubator/pig/branches/types/test/org/apache/pig/test/data/GoldenFiles/Split1.gld Mon Aug 11 21:35:02 2008 @@ -1,6 +1,6 @@ Filter[tuple] - Test-Plan-Builder-196 | | -| Greater Than[boolean] - Test-Plan-Builder-199 +| Less Than[boolean] - Test-Plan-Builder-199 | | | |---Project[bytearray][0] - Test-Plan-Builder-197 | | @@ -12,7 +12,7 @@ Filter[tuple] - Test-Plan-Builder-200 | | -| Less Than[boolean] - Test-Plan-Builder-203 +| Greater Than[boolean] - Test-Plan-Builder-203 | | | |---Project[bytearray][0] - Test-Plan-Builder-201 | |