Author: daijy Date: Sat Jan 30 02:22:30 2010 New Revision: 904712 URL: http://svn.apache.org/viewvc?rev=904712&view=rev Log: PIG-1210: fieldsToRead send the same fields more than once in some cases
Modified: hadoop/pig/trunk/CHANGES.txt hadoop/pig/trunk/src/org/apache/pig/impl/logicalLayer/LOForEach.java hadoop/pig/trunk/test/org/apache/pig/test/TestPruneColumn.java Modified: hadoop/pig/trunk/CHANGES.txt URL: http://svn.apache.org/viewvc/hadoop/pig/trunk/CHANGES.txt?rev=904712&r1=904711&r2=904712&view=diff ============================================================================== --- hadoop/pig/trunk/CHANGES.txt (original) +++ hadoop/pig/trunk/CHANGES.txt Sat Jan 30 02:22:30 2010 @@ -367,6 +367,8 @@ PIG-1195: POSort should take care of sort order (daijy) +PIG-1210: fieldsToRead send the same fields more than once in some cases (daijy) + Release 0.5.0 INCOMPATIBLE CHANGES Modified: hadoop/pig/trunk/src/org/apache/pig/impl/logicalLayer/LOForEach.java URL: http://svn.apache.org/viewvc/hadoop/pig/trunk/src/org/apache/pig/impl/logicalLayer/LOForEach.java?rev=904712&r1=904711&r2=904712&view=diff ============================================================================== --- hadoop/pig/trunk/src/org/apache/pig/impl/logicalLayer/LOForEach.java (original) +++ hadoop/pig/trunk/src/org/apache/pig/impl/logicalLayer/LOForEach.java Sat Jan 30 02:22:30 2010 @@ -861,7 +861,8 @@ ArrayList<Pair<Integer, Integer>> inputList = new ArrayList<Pair<Integer, Integer>>(); for (LOProject project : projectFinder.getProjectSet()) { for (int inputColumn : project.getProjection()) { - inputList.add(new Pair<Integer, Integer>(0, inputColumn)); + if (!inputList.contains(new Pair<Integer, Integer>(0, inputColumn))) + inputList.add(new Pair<Integer, Integer>(0, inputColumn)); } } if (inputList.size()==0) Modified: hadoop/pig/trunk/test/org/apache/pig/test/TestPruneColumn.java URL: http://svn.apache.org/viewvc/hadoop/pig/trunk/test/org/apache/pig/test/TestPruneColumn.java?rev=904712&r1=904711&r2=904712&view=diff ============================================================================== --- hadoop/pig/trunk/test/org/apache/pig/test/TestPruneColumn.java (original) +++ hadoop/pig/trunk/test/org/apache/pig/test/TestPruneColumn.java Sat Jan 30 02:22:30 2010 @@ -33,8 +33,12 @@ import org.apache.log4j.SimpleLayout; import org.apache.pig.ExecType; import org.apache.pig.FilterFunc; +import org.apache.pig.LoadFunc; import org.apache.pig.PigServer; +import org.apache.pig.LoadFunc.RequiredFieldList; +import org.apache.pig.builtin.PigStorage; import org.apache.pig.data.Tuple; +import org.apache.pig.impl.logicalLayer.FrontendException; import org.apache.pig.impl.logicalLayer.optimizer.PruneColumns; import org.junit.Before; import org.junit.Test; @@ -55,8 +59,24 @@ File tmpFile9; File tmpFile10; File logFile; - Logger logger; + static public class PigStorageWithTrace extends PigStorage { + + /** + * @param delimiter + */ + public PigStorageWithTrace() { + super(); + } + @Override + public LoadFunc.RequiredFieldResponse fieldsToRead(RequiredFieldList requiredFieldList) throws FrontendException { + LoadFunc.RequiredFieldResponse response = super.fieldsToRead(requiredFieldList); + Logger logger = Logger.getLogger(this.getClass()); + logger.info(requiredFieldList); + return response; + } + + } private static final String simpleEchoStreamingCommand; static { if (System.getProperty("os.name").toUpperCase().startsWith("WINDOWS")) @@ -76,7 +96,7 @@ @Before @Override public void setUp() throws Exception{ - logger = Logger.getLogger(PruneColumns.class); + Logger logger = Logger.getLogger(PruneColumns.class); logger.removeAllAppenders(); logger.setLevel(Level.INFO); SimpleLayout layout = new SimpleLayout(); @@ -84,6 +104,10 @@ FileAppender appender = new FileAppender(layout, logFile.toString(), false, false, 0); logger.addAppender(appender); + Logger pigStorageWithTraceLogger = Logger.getLogger(PigStorageWithTrace.class); + pigStorageWithTraceLogger.setLevel(Level.INFO); + pigStorageWithTraceLogger.addAppender(appender); + pigServer = new PigServer("local"); //pigServer = new PigServer(ExecType.MAPREDUCE, cluster.getProperties()); tmpFile1 = File.createTempFile("prune", "txt"); @@ -1799,4 +1823,27 @@ assertTrue(checkLogFileMessage(new String[]{"Columns pruned for A: $1", "No map keys pruned for A"})); } + + // See PIG-1210 + @Test + public void testFieldsToReadDuplicatedEntry() throws Exception { + pigServer.registerQuery("A = load '"+ Util.generateURI(tmpFile1.toString()) + "' using "+PigStorageWithTrace.class.getName() + +" AS (a0, a1, a2);"); + pigServer.registerQuery("B = foreach A generate a0+a0, a1, a2;"); + Iterator<Tuple> iter = pigServer.openIterator("B"); + + assertTrue(iter.hasNext()); + Tuple t = iter.next(); + assertTrue(t.toString().equals("(2.0,2,3)")); + + assertTrue(iter.hasNext()); + t = iter.next(); + assertTrue(t.toString().equals("(4.0,5,2)")); + + assertFalse(iter.hasNext()); + + assertTrue(checkLogFileMessage(new String[]{"No column pruned for A", + "No map keys pruned for A", "[0,1,2]"})); + } + }