Author: daijy Date: Fri Mar 12 01:08:30 2010 New Revision: 922097 URL: http://svn.apache.org/viewvc?rev=922097&view=rev Log: PIG-1272: Column pruner causes wrong results
Modified: hadoop/pig/trunk/CHANGES.txt hadoop/pig/trunk/src/org/apache/pig/impl/logicalLayer/optimizer/PruneColumns.java hadoop/pig/trunk/test/org/apache/pig/test/TestPruneColumn.java Modified: hadoop/pig/trunk/CHANGES.txt URL: http://svn.apache.org/viewvc/hadoop/pig/trunk/CHANGES.txt?rev=922097&r1=922096&r2=922097&view=diff ============================================================================== --- hadoop/pig/trunk/CHANGES.txt (original) +++ hadoop/pig/trunk/CHANGES.txt Fri Mar 12 01:08:30 2010 @@ -147,6 +147,8 @@ OPTIMIZATIONS BUG FIXES +PIG-1272: Column pruner causes wrong results (daijy) + PIG-1275: empty bag in PigStorage read as null (daijy) PIG-1252: Diamond splitter does not generate correct results when using Modified: hadoop/pig/trunk/src/org/apache/pig/impl/logicalLayer/optimizer/PruneColumns.java URL: http://svn.apache.org/viewvc/hadoop/pig/trunk/src/org/apache/pig/impl/logicalLayer/optimizer/PruneColumns.java?rev=922097&r1=922096&r2=922097&view=diff ============================================================================== --- hadoop/pig/trunk/src/org/apache/pig/impl/logicalLayer/optimizer/PruneColumns.java (original) +++ hadoop/pig/trunk/src/org/apache/pig/impl/logicalLayer/optimizer/PruneColumns.java Fri Mar 12 01:08:30 2010 @@ -220,12 +220,25 @@ public class PruneColumns extends Logica { List<RequiredFields> requiredInputFieldsList = new ArrayList<RequiredFields>(); RequiredFields requiredFields = new RequiredFields(false); - for (RequiredFields rf : requiredOutputInfo.requiredFieldsList) + for (int i=0;i<mPlan.getSuccessors(rlo).size();i++) { + RequiredFields rf = null; + try { + rf = requiredOutputInfo.requiredFieldsList.get(i); + } catch (Exception e) { + } if (rf!=null) { rf.reIndex(0); requiredFields.merge(rf); + } else { + // need all fields + List<Pair<Integer, Integer>> l = new ArrayList<Pair<Integer, Integer>>(); + for (int j=0;j<rlo.getSchema().size();j++) + l.add(new Pair<Integer, Integer>(0, j)); + rf = new RequiredFields(l); + requiredFields.merge(rf); + break; } } requiredInputFieldsList.add(requiredFields); Modified: hadoop/pig/trunk/test/org/apache/pig/test/TestPruneColumn.java URL: http://svn.apache.org/viewvc/hadoop/pig/trunk/test/org/apache/pig/test/TestPruneColumn.java?rev=922097&r1=922096&r2=922097&view=diff ============================================================================== --- hadoop/pig/trunk/test/org/apache/pig/test/TestPruneColumn.java (original) +++ hadoop/pig/trunk/test/org/apache/pig/test/TestPruneColumn.java Fri Mar 12 01:08:30 2010 @@ -1844,5 +1844,25 @@ public class TestPruneColumn extends Tes assertTrue(checkLogFileMessage(new String[]{"No column pruned for A", "No map keys pruned for A", "[0,1,2]"})); } + + // See PIG-1272 + @Test + public void testSplit4() throws Exception { + pigServer.registerQuery("A = load '"+ Util.generateURI(tmpFile1.toString(), pigServer.getPigContext()) + "' AS (a0, a1, a2);"); + pigServer.registerQuery("B = foreach A generate a0;"); + pigServer.registerQuery("C = join A by a0, B by a0;"); + Iterator<Tuple> iter = pigServer.openIterator("C"); + + assertTrue(iter.hasNext()); + Tuple t = iter.next(); + assertTrue(t.toString().equals("(1,2,3,1)")); + + assertTrue(iter.hasNext()); + t = iter.next(); + assertTrue(t.toString().equals("(2,5,2,2)")); + + assertTrue(checkLogFileMessage(new String[]{"No column pruned for A", + "No map keys pruned for A"})); + } }