Author: olga
Date: Mon Aug 25 15:59:27 2008
New Revision: 688915
URL: http://svn.apache.org/viewvc?rev=688915&view=rev
Log:
PIG-353: parsing of complex types
Modified:
incubator/pig/branches/types/CHANGES.txt
incubator/pig/branches/types/src/org/apache/pig/impl/logicalLayer/parser/QueryParser.jjt
incubator/pig/branches/types/src/org/apache/pig/tools/pigscript/parser/PigScriptParser.jj
incubator/pig/branches/types/test/org/apache/pig/test/TestGrunt.java
incubator/pig/branches/types/test/org/apache/pig/test/TestTypeChecking.java
Modified: incubator/pig/branches/types/CHANGES.txt
URL:
http://svn.apache.org/viewvc/incubator/pig/branches/types/CHANGES.txt?rev=688915&r1=688914&r2=688915&view=diff
==============================================================================
--- incubator/pig/branches/types/CHANGES.txt (original)
+++ incubator/pig/branches/types/CHANGES.txt Mon Aug 25 15:59:27 2008
@@ -169,3 +169,5 @@
PIG-352: java.lang.ClassCastException when invalid field is accessed
PIG-329: TestStoreOld, 2 unit tests were broken
+
+ PIG-353: parsing of complex types
Modified:
incubator/pig/branches/types/src/org/apache/pig/impl/logicalLayer/parser/QueryParser.jjt
URL:
http://svn.apache.org/viewvc/incubator/pig/branches/types/src/org/apache/pig/impl/logicalLayer/parser/QueryParser.jjt?rev=688915&r1=688914&r2=688915&view=diff
==============================================================================
---
incubator/pig/branches/types/src/org/apache/pig/impl/logicalLayer/parser/QueryParser.jjt
(original)
+++
incubator/pig/branches/types/src/org/apache/pig/impl/logicalLayer/parser/QueryParser.jjt
Mon Aug 25 15:59:27 2008
@@ -786,7 +786,7 @@
if (funcSpec == null){
funcSpec = new FuncSpec(PigStorage.class.getName());
}
-
+
lo = new LOLoad(lp, new OperatorKey(scope, getNextId()), new
FileSpec(massageFilename(filename, pigContext), funcSpec), null);
lp.add(lo);
log.debug("Added operator " + lo.getClass().getName() + " to
the logical plan");
@@ -1862,31 +1862,12 @@
}
)
)
- [ <AS> (LOOKAHEAD(2) "(" schema = TupleSchema() ")"
-
- { if (schema.size() > 1) {
- throw new ParseException("Schema mismatch");
- }
-
- fs = schema.getFields().get(0);
- }
- | fs = AtomSchema() )
- ]
+ [ <AS> (fs = FieldSchema()) ]
)
{
log.debug("item: " + item.getClass().getName());
if(null != fs) {
- // We're only interested in alias here
- // target type should be preserved
- Schema.FieldSchema tmpFs = item.getFieldSchema() ;
- if (tmpFs != null) {
- tmpFs.alias = fs.alias ;
- item.setFieldSchema(tmpFs);
- }
- else {
- tmpFs = new Schema.FieldSchema(fs.alias, DataType.BYTEARRAY) ;
- item.setFieldSchema(tmpFs);
- }
+ item.setFieldSchema(fs);
}
flattenList.add(flatten);
log.trace("Exiting FlattenedGenerateItem");
@@ -2356,7 +2337,7 @@
log.trace("Entering SchemaTuple");
}
{
- ( t1 = <IDENTIFIER> ) [":" <TUPLE>] "(" s = TupleSchema() ")"
+ ( t1 = <IDENTIFIER> ) [LOOKAHEAD(2) ":" <TUPLE> | ":"] "(" s =
TupleSchema() ")"
{
if (null != t1) {
log.debug("TUPLE alias " + t1.image);
@@ -2377,7 +2358,7 @@
log.trace("Entering SchemaBag");
}
{
- ( t1 = <IDENTIFIER> ) [":" <BAG>] "{" (fs = SchemaTuple() | {} {fs =
new Schema.FieldSchema(null, new Schema());}) "}"
+ ( t1 = <IDENTIFIER> ) [LOOKAHEAD(2) ":" <BAG> | ":"] "{" (fs =
SchemaTuple() | {} {fs = new Schema.FieldSchema(null, new Schema());}) "}"
{
s = new Schema(fs);
if (null != t1) {
Modified:
incubator/pig/branches/types/src/org/apache/pig/tools/pigscript/parser/PigScriptParser.jj
URL:
http://svn.apache.org/viewvc/incubator/pig/branches/types/src/org/apache/pig/tools/pigscript/parser/PigScriptParser.jj?rev=688915&r1=688914&r2=688915&view=diff
==============================================================================
---
incubator/pig/branches/types/src/org/apache/pig/tools/pigscript/parser/PigScriptParser.jj
(original)
+++
incubator/pig/branches/types/src/org/apache/pig/tools/pigscript/parser/PigScriptParser.jj
Mon Aug 25 15:59:27 2008
@@ -30,6 +30,7 @@
import java.io.IOException;
import java.io.InputStream;
+import java.util.Stack;
import jline.ConsoleReader;
@@ -148,9 +149,13 @@
TOKEN_MGR_DECLS : {
int pigBlockLevel = 0;
int funcBlockLevel = 0;
+ int tupleSchemaLevel = 0;
+ int bagSchemaLevel = 0;
+ int bagConstantLevel = 0;
int prevState = DEFAULT;
boolean interactive = false;
ConsoleReader consoleReader = null;
+ Stack<Integer> stack = new Stack<Integer>();
public void secondary_prompt()
{
if (interactive)
@@ -161,6 +166,15 @@
}
}
+ public int getState(int state) {
+ if(!stack.empty()) return stack.pop();
+ return state;
+ }
+
+ public void saveState(int state) {
+ stack.push(state);
+ }
+
}
<DEFAULT> MORE :
@@ -174,13 +188,15 @@
<PIG_START> MORE :
{
<"'"> {prevState = PIG_START;} : IN_STRING
+| <["A","a"]["S","s"]> {prevState = PIG_START;} : SCHEMA_DEFINITION
+| <["G","g"]["E","e"]["N","n"]["E","e"]["R","r"]["A","a"]["T","t"]["E","e"]>
{prevState = PIG_START;} : GENERATE
| <"{"> {pigBlockLevel = 1;} : IN_BLOCK
| <"}"> {if (true) throw new TokenMgrError("Unmatched '}'",
TokenMgrError.LEXICAL_ERROR);}
| <";"> : PIG_END
| <"--"> {prevState = PIG_START;} : SINGLE_LINE_COMMENT
| <"/*"> {prevState = PIG_START;} : MULTI_LINE_COMMENT
| <("\n" | "\r" | "\r\n")> {secondary_prompt();}
-| <(~[])>
+| <(~[])>
}
<SINGLE_LINE_COMMENT> MORE :
@@ -204,9 +220,65 @@
| <(~[])>
}
+<GENERATE> MORE :
+{
+ <"{">
+ {
+ bagConstantLevel++;
+ prevState = getState(prevState);
+ saveState(prevState);
+ prevState = GENERATE;
+ } : BAG_CONSTANT
+| <["A","a"]["S","s"]>
+ {
+ prevState = getState(prevState);
+ saveState(prevState);
+ prevState = GENERATE;
+ } : SCHEMA_DEFINITION
+| <";">
+ {
+ prevState = getState(prevState);
+ if(prevState == PIG_START) {
+ input_stream.backup(1);
+ image.deleteCharAt(image.length()-1);
+ }
+ SwitchTo(prevState);
+ }
+| <("\n" | "\r" | "\r\n")> {secondary_prompt();}
+| <(~[])>
+}
+
+<SCHEMA_DEFINITION> MORE :
+{
+ <"("> {tupleSchemaLevel++;}
+| <")"> {tupleSchemaLevel--; if ((tupleSchemaLevel == 0) && (bagSchemaLevel
== 0)) SwitchTo(prevState); }
+| <"{"> {bagSchemaLevel++;}
+| <"}"> {bagSchemaLevel--; if ((tupleSchemaLevel == 0) && (bagSchemaLevel ==
0)) SwitchTo(prevState); }
+| <("," | ";" )>
+ {
+ if ((tupleSchemaLevel == 0) && (bagSchemaLevel == 0)) {
+ input_stream.backup(1);
+ image.deleteCharAt(image.length()-1);
+ SwitchTo(prevState);
+ }
+ }
+| <("\n" | "\r" | "\r\n")> {secondary_prompt();}
+| <(~[])>
+}
+
+<BAG_CONSTANT> MORE :
+{
+ <"{"> {bagConstantLevel++;}
+| <"}"> {bagConstantLevel--; if (bagConstantLevel == 0) SwitchTo(prevState);}
+| <("\n" | "\r" | "\r\n")> {secondary_prompt();}
+| <(~[])>
+}
+
<IN_BLOCK> MORE :
{
<"\""> {prevState = IN_BLOCK;} : IN_STRING
+| <["A","a"]["S","s"]> {prevState = IN_BLOCK;} : SCHEMA_DEFINITION
+| <["G","g"]["E","e"]["N","n"]["E","e"]["R","r"]["A","a"]["T","t"]["E","e"]>
{prevState = IN_BLOCK;} : GENERATE
| <"{"> {pigBlockLevel++;}
| <"}"(";")?> {pigBlockLevel--; if (pigBlockLevel == 0)
SwitchTo(PIG_END);}
| <"'"> {prevState = IN_BLOCK;} : IN_STRING
Modified: incubator/pig/branches/types/test/org/apache/pig/test/TestGrunt.java
URL:
http://svn.apache.org/viewvc/incubator/pig/branches/types/test/org/apache/pig/test/TestGrunt.java?rev=688915&r1=688914&r2=688915&view=diff
==============================================================================
--- incubator/pig/branches/types/test/org/apache/pig/test/TestGrunt.java
(original)
+++ incubator/pig/branches/types/test/org/apache/pig/test/TestGrunt.java Mon
Aug 25 15:59:27 2008
@@ -71,4 +71,79 @@
}
assertTrue(null != context.getFuncSpecFromAlias("myudf"));
}
+
+ @Test
+ public void testBagSchema() throws Throwable {
+ PigServer server = new PigServer("MAPREDUCE");
+ PigContext context = server.getPigContext();
+
+ String strCmd = "a = load 'input1'as (b: bag{t(i: int, c:chararray, f:
float)});\n";
+
+ ByteArrayInputStream cmd = new ByteArrayInputStream(strCmd.getBytes());
+ InputStreamReader reader = new InputStreamReader(cmd);
+
+ Grunt grunt = new Grunt(new BufferedReader(reader), context);
+
+ grunt.exec();
+ }
+
+ @Test
+ public void testBagConstant() throws Throwable {
+ PigServer server = new PigServer("MAPREDUCE");
+ PigContext context = server.getPigContext();
+
+ String strCmd = "a = load 'input1'; b = foreach a generate {(1, '1',
0.4f),(2, '2', 0.45)};\n";
+
+ ByteArrayInputStream cmd = new ByteArrayInputStream(strCmd.getBytes());
+ InputStreamReader reader = new InputStreamReader(cmd);
+
+ Grunt grunt = new Grunt(new BufferedReader(reader), context);
+
+ grunt.exec();
+ }
+
+ @Test
+ public void testBagConstantWithSchema() throws Throwable {
+ PigServer server = new PigServer("MAPREDUCE");
+ PigContext context = server.getPigContext();
+
+ String strCmd = "a = load 'input1'; b = foreach a generate {(1, '1',
0.4f),(2, '2', 0.45)} as b: bag{t(i: int, c:chararray, f: float)};\n";
+
+ ByteArrayInputStream cmd = new ByteArrayInputStream(strCmd.getBytes());
+ InputStreamReader reader = new InputStreamReader(cmd);
+
+ Grunt grunt = new Grunt(new BufferedReader(reader), context);
+
+ grunt.exec();
+ }
+
+ @Test
+ public void testBagConstantInForeachBlock() throws Throwable {
+ PigServer server = new PigServer("MAPREDUCE");
+ PigContext context = server.getPigContext();
+
+ String strCmd = "a = load 'input1'; b = foreach a {generate {(1, '1',
0.4f),(2, '2', 0.45)};};\n";
+
+ ByteArrayInputStream cmd = new ByteArrayInputStream(strCmd.getBytes());
+ InputStreamReader reader = new InputStreamReader(cmd);
+
+ Grunt grunt = new Grunt(new BufferedReader(reader), context);
+
+ grunt.exec();
+ }
+
+ @Test
+ public void testBagConstantWithSchemaInForeachBlock() throws Throwable {
+ PigServer server = new PigServer("MAPREDUCE");
+ PigContext context = server.getPigContext();
+
+ String strCmd = "a = load 'input1'; b = foreach a {generate {(1, '1',
0.4f),(2, '2', 0.45)} as b: bag{t(i: int, c:chararray, f: float)};};\n";
+
+ ByteArrayInputStream cmd = new ByteArrayInputStream(strCmd.getBytes());
+ InputStreamReader reader = new InputStreamReader(cmd);
+
+ Grunt grunt = new Grunt(new BufferedReader(reader), context);
+
+ grunt.exec();
+ }
}
Modified:
incubator/pig/branches/types/test/org/apache/pig/test/TestTypeChecking.java
URL:
http://svn.apache.org/viewvc/incubator/pig/branches/types/test/org/apache/pig/test/TestTypeChecking.java?rev=688915&r1=688914&r2=688915&view=diff
==============================================================================
--- incubator/pig/branches/types/test/org/apache/pig/test/TestTypeChecking.java
(original)
+++ incubator/pig/branches/types/test/org/apache/pig/test/TestTypeChecking.java
Mon Aug 25 15:59:27 2008
@@ -245,7 +245,7 @@
public void testSUM1() throws Throwable {
TypeCheckingTestUtil.printCurrentMethodName() ;
planTester.buildPlan("a = load ':INPATH:/singlefile/studenttab10k' as
(name:chararray, age:int, gpa:double);") ;
- LogicalPlan plan1 = planTester.buildPlan("b = foreach a generate
(long)age as age, (int)gpa as gpa;") ;
+ LogicalPlan plan1 = planTester.buildPlan("b = foreach a generate
(long)age as age:long, (int)gpa as gpa:int;") ;
LogicalPlan plan2 = planTester.buildPlan("c = foreach b generate
SUM(age), SUM(gpa);") ;
planTester.typeCheckPlan(plan2);
}
