Author: daijy
Date: Wed Sep 22 18:48:14 2010
New Revision: 1000137
URL: http://svn.apache.org/viewvc?rev=1000137&view=rev
Log:
PIG-1636: Scalar fail if the scalar variable is generated by limit
Modified:
hadoop/pig/branches/branch-0.8/CHANGES.txt
hadoop/pig/branches/branch-0.8/src/org/apache/pig/PigServer.java
hadoop/pig/branches/branch-0.8/src/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/MRCompiler.java
hadoop/pig/branches/branch-0.8/test/org/apache/pig/test/TestScalarAliases.java
Modified: hadoop/pig/branches/branch-0.8/CHANGES.txt
URL:
http://svn.apache.org/viewvc/hadoop/pig/branches/branch-0.8/CHANGES.txt?rev=1000137&r1=1000136&r2=1000137&view=diff
==============================================================================
--- hadoop/pig/branches/branch-0.8/CHANGES.txt (original)
+++ hadoop/pig/branches/branch-0.8/CHANGES.txt Wed Sep 22 18:48:14 2010
@@ -193,6 +193,8 @@ PIG-1309: Map-side Cogroup (ashutoshc)
BUG FIXES
+PIG-1636: Scalar fail if the scalar variable is generated by limit (daijy)
+
PIG-1605: Adding soft link to plan to solve input file dependency (daijy)
PIG-1598: Pig gobbles up error messages - Part 2 (nrai via daijy)
Modified: hadoop/pig/branches/branch-0.8/src/org/apache/pig/PigServer.java
URL:
http://svn.apache.org/viewvc/hadoop/pig/branches/branch-0.8/src/org/apache/pig/PigServer.java?rev=1000137&r1=1000136&r2=1000137&view=diff
==============================================================================
--- hadoop/pig/branches/branch-0.8/src/org/apache/pig/PigServer.java (original)
+++ hadoop/pig/branches/branch-0.8/src/org/apache/pig/PigServer.java Wed Sep 22
18:48:14 2010
@@ -1277,6 +1277,7 @@ public class PigServer {
fileSpec, alias);
referredPlan.addAsLeaf(store);
((LOStore)store).setTmpStore(true);
+ scalarEntry.getKey().setImplicitReferencedOperator(store);
}
lp.mergeSharedPlan(referredPlan);
Modified:
hadoop/pig/branches/branch-0.8/src/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/MRCompiler.java
URL:
http://svn.apache.org/viewvc/hadoop/pig/branches/branch-0.8/src/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/MRCompiler.java?rev=1000137&r1=1000136&r2=1000137&view=diff
==============================================================================
---
hadoop/pig/branches/branch-0.8/src/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/MRCompiler.java
(original)
+++
hadoop/pig/branches/branch-0.8/src/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/MRCompiler.java
Wed Sep 22 18:48:14 2010
@@ -243,10 +243,8 @@ public class MRCompiler extends PhyPlanV
for(MapReduceOper mrOp: mrOpList) {
for(PhysicalOperator scalar: mrOp.scalars) {
MapReduceOper mro = phyToMROpMap.get(scalar);
- List<PhysicalOperator> succs = plan.getSuccessors(scalar);
- if (succs.size() == 1 && succs.get(0) instanceof POStore) {
- POStore sto = (POStore)plan.getSuccessors(scalar).get(0);
- FileSpec oldSpec = sto.getSFile();
+ if (scalar instanceof POStore) {
+ FileSpec oldSpec = ((POStore)scalar).getSFile();
MapReduceOper mro2 = seen.get(oldSpec);
boolean hasSeen = false;
if (mro2 != null) {
Modified:
hadoop/pig/branches/branch-0.8/test/org/apache/pig/test/TestScalarAliases.java
URL:
http://svn.apache.org/viewvc/hadoop/pig/branches/branch-0.8/test/org/apache/pig/test/TestScalarAliases.java?rev=1000137&r1=1000136&r2=1000137&view=diff
==============================================================================
---
hadoop/pig/branches/branch-0.8/test/org/apache/pig/test/TestScalarAliases.java
(original)
+++
hadoop/pig/branches/branch-0.8/test/org/apache/pig/test/TestScalarAliases.java
Wed Sep 22 18:48:14 2010
@@ -538,4 +538,45 @@ public class TestScalarAliases {
assertTrue(pe.getCause().getMessage().equalsIgnoreCase("Scalars
can be only used with projections"));
}
}
+
+ // See PIG-1636
+ @Test
+ public void testScalarAliasesLimit() throws Exception{
+ String[] input = {
+ "a\t1",
+ "b\t2",
+ "c\t3",
+ "a\t4",
+ "c\t5"
+ };
+
+ // Test the use of scalars in expressions
+ Util.createLocalInputFile( "table_testScalarAliasesLimit", input);
+ // Test in script mode
+ pigServer.registerQuery("A = LOAD 'table_testScalarAliasesLimit' as
(a0:chararray, a1: int);");
+ pigServer.registerQuery("G = group A all;");
+ pigServer.registerQuery("C = foreach G generate SUM(A.$1) as total;");
+ pigServer.registerQuery("C1 = limit C 1;");
+ pigServer.registerQuery("Y = foreach A generate a0, a1 *
(double)C1.total;");
+
+ Iterator<Tuple> iter = pigServer.openIterator("Y");
+
+ // Average is 11
+ Tuple t = iter.next();
+ assertTrue(t.toString().equals("(a,15.0)"));
+
+ t = iter.next();
+ assertTrue(t.toString().equals("(b,30.0)"));
+
+ t = iter.next();
+ assertTrue(t.toString().equals("(c,45.0)"));
+
+ t = iter.next();
+ assertTrue(t.toString().equals("(a,60.0)"));
+
+ t = iter.next();
+ assertTrue(t.toString().equals("(c,75.0)"));
+
+ assertFalse(iter.hasNext());
+ }
}
