Author: olga
Date: Thu Sep 4 16:33:23 2008
New Revision: 692303
URL: http://svn.apache.org/viewvc?rev=692303&view=rev
Log:
PIG-398: Expressions not allowed inside foreach
Modified:
incubator/pig/branches/types/CHANGES.txt
incubator/pig/branches/types/src/org/apache/pig/Main.java
incubator/pig/branches/types/src/org/apache/pig/impl/logicalLayer/LOVisitor.java
incubator/pig/branches/types/src/org/apache/pig/impl/logicalLayer/optimizer/SchemaRemover.java
incubator/pig/branches/types/src/org/apache/pig/impl/logicalLayer/parser/QueryParser.jjt
incubator/pig/branches/types/src/org/apache/pig/impl/logicalLayer/schema/Schema.java
incubator/pig/branches/types/test/org/apache/pig/test/TestEvalPipeline.java
incubator/pig/branches/types/test/org/apache/pig/test/TypeGraphPrinter.java
Modified: incubator/pig/branches/types/CHANGES.txt
URL:
http://svn.apache.org/viewvc/incubator/pig/branches/types/CHANGES.txt?rev=692303&r1=692302&r2=692303&view=diff
==============================================================================
--- incubator/pig/branches/types/CHANGES.txt (original)
+++ incubator/pig/branches/types/CHANGES.txt Thu Sep 4 16:33:23 2008
@@ -140,9 +140,9 @@
PIG-258: cleaning up directories on failure (daijy via olgan)
- PIG-139: command line editing
+ PIG-139: command line editing (daijy via olgan)
- PIG-270: proper line number for parse errors
+ PIG-270: proper line number for parse errors (daijy via olgan)
PIG-363: fix for describe to produce schema name
@@ -179,3 +179,5 @@
PIG-373: unconnected load causes problem,
PIG-413: problem with float sum
+
+ PIG-398: Expressions not allowed inside foreach (sms via olgan)
Modified: incubator/pig/branches/types/src/org/apache/pig/Main.java
URL:
http://svn.apache.org/viewvc/incubator/pig/branches/types/src/org/apache/pig/Main.java?rev=692303&r1=692302&r2=692303&view=diff
==============================================================================
--- incubator/pig/branches/types/src/org/apache/pig/Main.java (original)
+++ incubator/pig/branches/types/src/org/apache/pig/Main.java Thu Sep 4
16:33:23 2008
@@ -25,6 +25,7 @@
import jline.ConsoleReader;
import jline.ConsoleReaderInputStream;
+import jline.History;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
@@ -272,6 +273,9 @@
ConsoleReader reader = new ConsoleReader(System.in, new
OutputStreamWriter(System.out));
reader.addCompletor(new PigCompletor());
reader.setDefaultPrompt("grunt> ");
+ final String HISTORYFILE = ".pig_history";
+ String historyFile = System.getProperty("user.home") +
File.separator + HISTORYFILE;
+ reader.setHistory(new History(new File(historyFile)));
ConsoleReaderInputStream inputStream = new
ConsoleReaderInputStream(reader);
grunt = new Grunt(new BufferedReader(new
InputStreamReader(inputStream)), pigContext);
grunt.setConsoleReader(reader);
Modified:
incubator/pig/branches/types/src/org/apache/pig/impl/logicalLayer/LOVisitor.java
URL:
http://svn.apache.org/viewvc/incubator/pig/branches/types/src/org/apache/pig/impl/logicalLayer/LOVisitor.java?rev=692303&r1=692302&r2=692303&view=diff
==============================================================================
---
incubator/pig/branches/types/src/org/apache/pig/impl/logicalLayer/LOVisitor.java
(original)
+++
incubator/pig/branches/types/src/org/apache/pig/impl/logicalLayer/LOVisitor.java
Thu Sep 4 16:33:23 2008
@@ -125,9 +125,7 @@
// walker the current visitor is using.
PlanWalker w = new DependencyOrderWalker(lp);
pushWalker(w);
- for(LogicalOperator logicalOp: lp.getRoots()) {
- logicalOp.visit(this);
- }
+ w.walk(this);
popWalker();
}
}
@@ -145,9 +143,7 @@
for(LogicalPlan lp: forEach.getForEachPlans()) {
PlanWalker w = new DependencyOrderWalker(lp);
pushWalker(w);
- for(LogicalOperator logicalOp: lp.getRoots()) {
- logicalOp.visit(this);
- }
+ w.walk(this);
popWalker();
}
}
@@ -163,9 +159,7 @@
for(LogicalPlan lp: s.getSortColPlans()) {
PlanWalker w = new DependencyOrderWalker(lp);
pushWalker(w);
- for(LogicalOperator logicalOp: lp.getRoots()) {
- logicalOp.visit(this);
- }
+ w.walk(this);
popWalker();
}
}
@@ -188,9 +182,7 @@
// Visit the condition for the filter followed by the input
PlanWalker w = new DependencyOrderWalker(filter.getComparisonPlan());
pushWalker(w);
- for(LogicalOperator logicalOp: filter.getComparisonPlan().getRoots()) {
- logicalOp.visit(this);
- }
+ w.walk(this);
popWalker();
}
@@ -218,9 +210,7 @@
for(LogicalPlan lp: g.getGeneratePlans()) {
PlanWalker w = new DependencyOrderWalker(lp);
pushWalker(w);
- for(LogicalOperator logicalOp: lp.getRoots()) {
- logicalOp.visit(this);
- }
+ w.walk(this);
popWalker();
}
}
@@ -287,9 +277,7 @@
if (null != lp) {
PlanWalker w = new DependencyOrderWalker(lp);
pushWalker(w);
- for(LogicalOperator logicalOp: lp.getRoots()) {
- logicalOp.visit(this);
- }
+ w.walk(this);
popWalker();
}
}
Modified:
incubator/pig/branches/types/src/org/apache/pig/impl/logicalLayer/optimizer/SchemaRemover.java
URL:
http://svn.apache.org/viewvc/incubator/pig/branches/types/src/org/apache/pig/impl/logicalLayer/optimizer/SchemaRemover.java?rev=692303&r1=692302&r2=692303&view=diff
==============================================================================
---
incubator/pig/branches/types/src/org/apache/pig/impl/logicalLayer/optimizer/SchemaRemover.java
(original)
+++
incubator/pig/branches/types/src/org/apache/pig/impl/logicalLayer/optimizer/SchemaRemover.java
Thu Sep 4 16:33:23 2008
@@ -150,7 +150,6 @@
* @throws VisitorException
*/
protected void visit(LOCast cast) throws VisitorException {
- cast.unsetFieldSchema();
super.visit(cast);
}
Modified:
incubator/pig/branches/types/src/org/apache/pig/impl/logicalLayer/parser/QueryParser.jjt
URL:
http://svn.apache.org/viewvc/incubator/pig/branches/types/src/org/apache/pig/impl/logicalLayer/parser/QueryParser.jjt?rev=692303&r1=692302&r2=692303&view=diff
==============================================================================
---
incubator/pig/branches/types/src/org/apache/pig/impl/logicalLayer/parser/QueryParser.jjt
(original)
+++
incubator/pig/branches/types/src/org/apache/pig/impl/logicalLayer/parser/QueryParser.jjt
Thu Sep 4 16:33:23 2008
@@ -499,6 +499,9 @@
public static void attachPlan(LogicalPlan lp, LogicalOperator root,
LogicalPlan rootPlan, Map<LogicalOperator, Boolean> rootProcessed) throws
ParseException {
log.trace("Entering attachPlan");
+ if(null == rootProcessed) {
+ rootProcessed = new HashMap<LogicalOperator, Boolean>();
+ }
if((rootProcessed.get(root) != null) && (rootProcessed.get(root))) {
log.trace("Root has been processed");
log.trace("Exiting attachPlan");
@@ -542,7 +545,6 @@
}
}
-
}
@@ -1637,15 +1639,30 @@
for (int planCtr = 0; planCtr < generatePlans.size(); ++planCtr) {
LogicalPlan generatePlan = generatePlans.get(planCtr);
- List<LogicalOperator> planRoots = generatePlan.getRoots();
+ List<LogicalOperator> planRoots = new
ArrayList<LogicalOperator>(generatePlan.getRoots());
boolean needGenerateInput = false;
boolean needForEachInput = false;
MultiMap<LogicalOperator, LogicalOperator> mapProjectInputs = null;
+ Map<LogicalOperator, Boolean> rootProcessed = new
HashMap<LogicalOperator, Boolean>();
+ for(LogicalOperator root: planRoots) {
+ if(root instanceof ExpressionOperator && !(root instanceof
LOProject)) {
+ if(checkGenerateInput(root)) {
+ needGenerateInput = true;
+ attachPlan(generatePlan, root, foreachPlan,
rootProcessed);
+ rootProcessed.put(root, true);
+ }
+ }
+ }
+
+ planRoots = generatePlan.getRoots();
+ needGenerateInput = false;
+ needForEachInput = false;
+
for(LogicalOperator root: planRoots) {
if(root instanceof LOProject) {
LOProject project = (LOProject)root;
LogicalOperator projectInput = project.getExpression();
- if(checkGenerateInput(projectInput)) {
+ if(checkGenerateInput(projectInput) ||
!(projectInput.equals(input))) {
needGenerateInput = true;
if(null == mapProjectInputs) {
mapProjectInputs = new MultiMap<LogicalOperator,
LogicalOperator>();
@@ -1654,7 +1671,7 @@
} else {
needForEachInput = true;
}
- }
+ }
}
if(needGenerateInput) {
/*
@@ -1664,7 +1681,6 @@
the list of foreach plans
*/
- Map<LogicalOperator, Boolean> rootProcessed = new
HashMap<LogicalOperator, Boolean>();
for(LogicalOperator project: mapProjectInputs.keySet()) {
for(LogicalOperator projectInput:
mapProjectInputs.get(project)) {
generatePlan.add(projectInput);
@@ -1933,12 +1949,20 @@
(
t = <IDENTIFIER> "="
(
- eOp = NestedProject(over,specs,lp,input)
+ LOOKAHEAD(FuncEvalSpec(over, specs, lp, input)) item =
InfixExpr(over,specs,lp, input)
+ {
+ lp.add(item);
+ }
+| LOOKAHEAD(NestedProject(over, specs, lp,input)) eOp =
NestedProject(over,specs,lp,input)
{
item = eOp;
lp.add(eOp);
log.debug("Added operator " + eOp.getClass().getName() + " " +
eOp + " to the logical plan " + lp);
}
+| item = InfixExpr(over,specs,lp, input)
+ {
+ lp.add(item);
+ }
| item = NestedFilter(over,specs,lp, input)
| item = NestedSortOrArrange(over,specs,lp, input)
| item = NestedDistinct(over,specs,lp, input)
@@ -2432,24 +2456,7 @@
{
Schema.FieldSchema fs = item.getFieldSchema();
subSchema = fs.schema;
- //TODO
- //HACK for the schema problems with LOProject
- //Check the schema to see if the constituent
- //field is a bag or a tuple/ If so, then get
- //that schema and send it out instead of the
- //actual schema
log.debug("subSchema: " + subSchema);
- if(null != subSchema) {
- log.debug("Printing subSchema Aliases");
- subSchema.printAliases();
- }
- /*
- log.debug("Printing the field schemas of subSchema");
- for(Schema.FieldSchema fs: subSchema.getFields()) {
- log.debug("fs: " + fs);
- subSchema = fs.schema;
- }
- */
}
(
"." projection = BracketedSimpleProj(subSchema,lp,item)
@@ -2494,7 +2501,6 @@
{
//ExpressionOperator bincond = new LOBinCond(lp, new
OperatorKey(scope, getNextId()), conditionPlan, truePlan, falsePlan);
ExpressionOperator bincond = new LOBinCond(lp, new
OperatorKey(scope, getNextId()), cond, ifTrue, ifFalse);
- //TODO - Need to connect expression operators with the new plan
lp.add(bincond);
log.debug("BinCond: Added operator " +
bincond.getClass().getName() + " " + bincond + " to logical plan " + lp);
lp.connect(cond, bincond);
@@ -3192,11 +3198,15 @@
LogicalOperator op = specs.get(t1.image);
if(null != op) {
log.debug("Alias: " + op.getAlias());
-
- item = new LOProject(lp, new OperatorKey(scope,
getNextId()), op, -1);
- ((LOProject)item).setStar(true);
- log.debug("Set star to true");
- item.setAlias(t1.image);
+
+ if((op instanceof ExpressionOperator) && insideGenerate()) {
+ item = (ExpressionOperator)op;
+ } else {
+ item = new LOProject(lp, new OperatorKey(scope,
getNextId()), op, -1);
+ ((LOProject)item).setStar(true);
+ log.debug("Set star to true");
+ item.setAlias(t1.image);
+ }
if(insideGenerate()) {
log.debug("AliasFieldOrSpec: Inside
generate");
@@ -3205,7 +3215,7 @@
try {
lp.add(item);
log.debug("AliasFieldOrSpec: Added
operator " + item.getClass().getName() + " " + item + " to logical plan " + lp);
- if(op instanceof ExpressionOperator) {
+ if((op instanceof ExpressionOperator) &&
!insideGenerate()) {
lp.add(op);
lp.connect(op, item);
}
Modified:
incubator/pig/branches/types/src/org/apache/pig/impl/logicalLayer/schema/Schema.java
URL:
http://svn.apache.org/viewvc/incubator/pig/branches/types/src/org/apache/pig/impl/logicalLayer/schema/Schema.java?rev=692303&r1=692302&r2=692303&view=diff
==============================================================================
---
incubator/pig/branches/types/src/org/apache/pig/impl/logicalLayer/schema/Schema.java
(original)
+++
incubator/pig/branches/types/src/org/apache/pig/impl/logicalLayer/schema/Schema.java
Thu Sep 4 16:33:23 2008
@@ -692,6 +692,8 @@
throw new AssertionError("Schema refers to itself "
+ "as inner schema") ;
}
+ } else if (fs.type == DataType.MAP) {
+ sb.append(DataType.findTypeName(fs.type) + "[ ]") ;
}
// TODO: Support Map
}
Modified:
incubator/pig/branches/types/test/org/apache/pig/test/TestEvalPipeline.java
URL:
http://svn.apache.org/viewvc/incubator/pig/branches/types/test/org/apache/pig/test/TestEvalPipeline.java?rev=692303&r1=692302&r2=692303&view=diff
==============================================================================
--- incubator/pig/branches/types/test/org/apache/pig/test/TestEvalPipeline.java
(original)
+++ incubator/pig/branches/types/test/org/apache/pig/test/TestEvalPipeline.java
Thu Sep 4 16:33:23 2008
@@ -396,6 +396,52 @@
assertEquals(LOOP_COUNT, numIdentity);
}
+ public void testNestedPlanWithExpressionAssignment() throws Exception{
+ int LOOP_COUNT = 10;
+ File tmpFile = File.createTempFile("test", "txt");
+ PrintStream ps = new PrintStream(new FileOutputStream(tmpFile));
+ Random r = new Random();
+ for(int i = 0; i < LOOP_COUNT; i++) {
+ for(int j=0;j<LOOP_COUNT;j+=2){
+ ps.println(i+"\t"+j);
+ ps.println(i+"\t"+j);
+ }
+ }
+ ps.close();
+
+ String tmpOutputFile = FileLocalizer.getTemporaryPath(null,
+ pigServer.getPigContext()).toString();
+ pigServer.registerQuery("A = LOAD 'file:" + tmpFile + "';");
+ pigServer.registerQuery("B = group A by $0;");
+ String query = "C = foreach B {"
+ + "C1 = filter A by $0 > -1;"
+ + "C2 = distinct C1;"
+ + "C3 = distinct A;"
+ + "C4 = " + Identity.class.getName() + "(*);"
+ + "C5 = COUNT(C2);"
+ + "C6 = SUM(C2.$1);"
+ + "C7 = " + TitleNGrams.class.getName() + "(C3);"
+ + "C8 = MAX(C3.$1);"
+ + "generate (int)group, C4, C5, C6, C7, C8, C2;"
+ + "};";
+
+ pigServer.registerQuery(query);
+ Iterator<Tuple> iter = pigServer.openIterator("C");
+ if(!iter.hasNext()) fail("No output found");
+ int numIdentity = 0;
+ while(iter.hasNext()){
+ Tuple t = iter.next();
+ assertEquals((Integer)numIdentity, (Integer)t.get(0));
+ assertEquals((Long)5L, (Long)t.get(2));
+ assertEquals(LOOP_COUNT*2.0, (Double)t.get(3), 0.01);
+ assertEquals(8.0, (Double)t.get(5), 0.01);
+ assertEquals(5L, ((DataBag)t.get(6)).size());
+ assertEquals(7, t.size());
+ ++numIdentity;
+ }
+ assertEquals(LOOP_COUNT, numIdentity);
+ }
+
public void testLimit() throws Exception{
int LOOP_COUNT = 20;
File tmpFile = File.createTempFile("test", "txt");
@@ -419,6 +465,6 @@
}
assertEquals(5, numIdentity);
}
-
+
}
Modified:
incubator/pig/branches/types/test/org/apache/pig/test/TypeGraphPrinter.java
URL:
http://svn.apache.org/viewvc/incubator/pig/branches/types/test/org/apache/pig/test/TypeGraphPrinter.java?rev=692303&r1=692302&r2=692303&view=diff
==============================================================================
--- incubator/pig/branches/types/test/org/apache/pig/test/TypeGraphPrinter.java
(original)
+++ incubator/pig/branches/types/test/org/apache/pig/test/TypeGraphPrinter.java
Thu Sep 4 16:33:23 2008
@@ -261,7 +261,7 @@
sb.append("=") ;
Schema schema = null ;
- if (!DataType.isComplex(op.getType())) {
+ if (!DataType.isSchemaType(op.getType())) {
sb.append(DataType.findTypeName(op.getType())) ;
}
else {
