[
https://issues.apache.org/jira/browse/PIG-1594?page=com.atlassian.jira.plugin.system.issuetabpanels:all-tabpanel
]
Andrew Hitchcock updated PIG-1594:
----------------------------------
Description:
I've been testing the trunk version of Pig on Elastic MapReduce against our log
processing sample application(1). When I try to run the query it throws a
NullPointerException and suggests I disable the new logical plan. Disabling it
works and the script succeeds. Here is the query I'm trying to run:
{{register file:/home/hadoop/lib/pig/piggybank.jar
DEFINE EXTRACT org.apache.pig.piggybank.evaluation.string.EXTRACT();
RAW_LOGS = LOAD '$INPUT' USING TextLoader as (line:chararray);
LOGS_BASE= foreach RAW_LOGS generate FLATTEN(EXTRACT(line, '^(\\S+) (\\S+)
(\\S+) \\[([\\w:/]+\\s[+\\-]\\d{4})\\] "(.+?)" (\\S+) (\\S+) "([^"]*)"
"([^"]*)"')) as (remoteAddr:chararray, remoteLogname:chararray, user:chararray,
time:chararray, request:chararray, status:int, bytes_string:chararray,
referrer:chararray, browser:chararray);
REFERRER_ONLY = FOREACH LOGS_BASE GENERATE referrer;
FILTERED = FILTER REFERRER_ONLY BY referrer matches '.*bing.*' OR referrer
matches '.*google.*';
SEARCH_TERMS = FOREACH FILTERED GENERATE FLATTEN(EXTRACT(referrer,
'.*[&\\?]q=([^&]+).*')) as terms:chararray;
SEARCH_TERMS_FILTERED = FILTER SEARCH_TERMS BY NOT $0 IS NULL;
SEARCH_TERMS_COUNT = FOREACH (GROUP SEARCH_TERMS_FILTERED BY $0) GENERATE $0,
COUNT($1) as num;
SEARCH_TERMS_COUNT_SORTED = LIMIT(ORDER SEARCH_TERMS_COUNT BY num DESC) 50;
STORE SEARCH_TERMS_COUNT_SORTED into '$OUTPUT';}}
And here is the stack trace that results:
{{ERROR 2042: Error in new logical plan. Try -Dpig.usenewlogicalplan=false.
org.apache.pig.backend.executionengine.ExecException: ERROR 2042: Error in new
logical plan. Try -Dpig.usenewlogicalplan=false.
at
org.apache.pig.backend.hadoop.executionengine.HExecutionEngine.compile(HExecutionEngine.java:285)
at org.apache.pig.PigServer.compilePp(PigServer.java:1301)
at
org.apache.pig.PigServer.executeCompiledLogicalPlan(PigServer.java:1154)
at org.apache.pig.PigServer.execute(PigServer.java:1148)
at org.apache.pig.PigServer.access$100(PigServer.java:123)
at org.apache.pig.PigServer$Graph.execute(PigServer.java:1464)
at org.apache.pig.PigServer.executeBatchEx(PigServer.java:350)
at org.apache.pig.PigServer.executeBatch(PigServer.java:324)
at
org.apache.pig.tools.grunt.GruntParser.executeBatch(GruntParser.java:111)
at
org.apache.pig.tools.grunt.GruntParser.parseStopOnError(GruntParser.java:168)
at
org.apache.pig.tools.grunt.GruntParser.parseStopOnError(GruntParser.java:140)
at org.apache.pig.tools.grunt.Grunt.exec(Grunt.java:90)
at org.apache.pig.Main.run(Main.java:491)
at org.apache.pig.Main.main(Main.java:107)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at
sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:39)
at
sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:25)
at java.lang.reflect.Method.invoke(Method.java:597)
at org.apache.hadoop.util.RunJar.main(RunJar.java:156)
Caused by: java.lang.NullPointerException
at org.apache.pig.EvalFunc.getSchemaName(EvalFunc.java:76)
at
org.apache.pig.piggybank.impl.ErrorCatchingBase.outputSchema(ErrorCatchingBase.java:76)
at
org.apache.pig.newplan.logical.expression.UserFuncExpression.getFieldSchema(UserFuncExpression.java:111)
at
org.apache.pig.newplan.logical.optimizer.FieldSchemaResetter.execute(SchemaResetter.java:175)
at
org.apache.pig.newplan.logical.expression.AllSameExpressionVisitor.visit(AllSameExpressionVisitor.java:143)
at
org.apache.pig.newplan.logical.expression.UserFuncExpression.accept(UserFuncExpression.java:55)
at
org.apache.pig.newplan.ReverseDependencyOrderWalker.walk(ReverseDependencyOrderWalker.java:69)
at org.apache.pig.newplan.PlanVisitor.visit(PlanVisitor.java:50)
at
org.apache.pig.newplan.logical.optimizer.SchemaResetter.visit(SchemaResetter.java:87)
at
org.apache.pig.newplan.logical.relational.LOGenerate.accept(LOGenerate.java:149)
at
org.apache.pig.newplan.DependencyOrderWalker.walk(DependencyOrderWalker.java:74)
at
org.apache.pig.newplan.logical.optimizer.SchemaResetter.visit(SchemaResetter.java:76)
at
org.apache.pig.newplan.logical.relational.LOForEach.accept(LOForEach.java:71)
at
org.apache.pig.newplan.DependencyOrderWalker.walk(DependencyOrderWalker.java:74)
at org.apache.pig.newplan.PlanVisitor.visit(PlanVisitor.java:50)
at
org.apache.pig.backend.hadoop.executionengine.HExecutionEngine.compile(HExecutionEngine.java:247)
... 18 more
================================================================================}}
1. http://developer.amazonwebservices.com/connect/entry.jspa?externalID=2729
was:
I've been testing the trunk version of Pig on Elastic MapReduce against our log
processing sample application(1). When I try to run the query it throws a
NullPointerException and suggests I disable the new logical plan. Disabling it
works and the script succeeds. Here is the query I'm trying to run:
register file:/home/hadoop/lib/pig/piggybank.jar
DEFINE EXTRACT org.apache.pig.piggybank.evaluation.string.EXTRACT();
RAW_LOGS = LOAD '$INPUT' USING TextLoader as (line:chararray);
LOGS_BASE= foreach RAW_LOGS generate FLATTEN(EXTRACT(line, '^(\\S+) (\\S+)
(\\S+) \\[([\\w:/]+\\s[+\\-]\\d{4})\\] "(.+?)" (\\S+) (\\S+) "([^"]*)"
"([^"]*)"')) as (remoteAddr:chararray, remoteLogname:chararray, user:chararray,
time:chararray, request:chararray, status:int, bytes_string:chararray,
referrer:chararray, browser:chararray);
REFERRER_ONLY = FOREACH LOGS_BASE GENERATE referrer;
FILTERED = FILTER REFERRER_ONLY BY referrer matches '.*bing.*' OR referrer
matches '.*google.*';
SEARCH_TERMS = FOREACH FILTERED GENERATE FLATTEN(EXTRACT(referrer,
'.*[&\\?]q=([^&]+).*')) as terms:chararray;
SEARCH_TERMS_FILTERED = FILTER SEARCH_TERMS BY NOT $0 IS NULL;
SEARCH_TERMS_COUNT = FOREACH (GROUP SEARCH_TERMS_FILTERED BY $0) GENERATE $0,
COUNT($1) as num;
SEARCH_TERMS_COUNT_SORTED = LIMIT(ORDER SEARCH_TERMS_COUNT BY num DESC) 50;
STORE SEARCH_TERMS_COUNT_SORTED into '$OUTPUT';
And here is the stack trace that results:
ERROR 2042: Error in new logical plan. Try -Dpig.usenewlogicalplan=false.
org.apache.pig.backend.executionengine.ExecException: ERROR 2042: Error in new
logical plan. Try -Dpig.usenewlogicalplan=false.
at
org.apache.pig.backend.hadoop.executionengine.HExecutionEngine.compile(HExecutionEngine.java:285)
at org.apache.pig.PigServer.compilePp(PigServer.java:1301)
at
org.apache.pig.PigServer.executeCompiledLogicalPlan(PigServer.java:1154)
at org.apache.pig.PigServer.execute(PigServer.java:1148)
at org.apache.pig.PigServer.access$100(PigServer.java:123)
at org.apache.pig.PigServer$Graph.execute(PigServer.java:1464)
at org.apache.pig.PigServer.executeBatchEx(PigServer.java:350)
at org.apache.pig.PigServer.executeBatch(PigServer.java:324)
at
org.apache.pig.tools.grunt.GruntParser.executeBatch(GruntParser.java:111)
at
org.apache.pig.tools.grunt.GruntParser.parseStopOnError(GruntParser.java:168)
at
org.apache.pig.tools.grunt.GruntParser.parseStopOnError(GruntParser.java:140)
at org.apache.pig.tools.grunt.Grunt.exec(Grunt.java:90)
at org.apache.pig.Main.run(Main.java:491)
at org.apache.pig.Main.main(Main.java:107)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at
sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:39)
at
sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:25)
at java.lang.reflect.Method.invoke(Method.java:597)
at org.apache.hadoop.util.RunJar.main(RunJar.java:156)
Caused by: java.lang.NullPointerException
at org.apache.pig.EvalFunc.getSchemaName(EvalFunc.java:76)
at
org.apache.pig.piggybank.impl.ErrorCatchingBase.outputSchema(ErrorCatchingBase.java:76)
at
org.apache.pig.newplan.logical.expression.UserFuncExpression.getFieldSchema(UserFuncExpression.java:111)
at
org.apache.pig.newplan.logical.optimizer.FieldSchemaResetter.execute(SchemaResetter.java:175)
at
org.apache.pig.newplan.logical.expression.AllSameExpressionVisitor.visit(AllSameExpressionVisitor.java:143)
at
org.apache.pig.newplan.logical.expression.UserFuncExpression.accept(UserFuncExpression.java:55)
at
org.apache.pig.newplan.ReverseDependencyOrderWalker.walk(ReverseDependencyOrderWalker.java:69)
at org.apache.pig.newplan.PlanVisitor.visit(PlanVisitor.java:50)
at
org.apache.pig.newplan.logical.optimizer.SchemaResetter.visit(SchemaResetter.java:87)
at
org.apache.pig.newplan.logical.relational.LOGenerate.accept(LOGenerate.java:149)
at
org.apache.pig.newplan.DependencyOrderWalker.walk(DependencyOrderWalker.java:74)
at
org.apache.pig.newplan.logical.optimizer.SchemaResetter.visit(SchemaResetter.java:76)
at
org.apache.pig.newplan.logical.relational.LOForEach.accept(LOForEach.java:71)
at
org.apache.pig.newplan.DependencyOrderWalker.walk(DependencyOrderWalker.java:74)
at org.apache.pig.newplan.PlanVisitor.visit(PlanVisitor.java:50)
at
org.apache.pig.backend.hadoop.executionengine.HExecutionEngine.compile(HExecutionEngine.java:247)
... 18 more
================================================================================
1. http://developer.amazonwebservices.com/connect/entry.jspa?externalID=2729
> NullPointerException in new logical planner
> -------------------------------------------
>
> Key: PIG-1594
> URL: https://issues.apache.org/jira/browse/PIG-1594
> Project: Pig
> Issue Type: Bug
> Reporter: Andrew Hitchcock
>
> I've been testing the trunk version of Pig on Elastic MapReduce against our
> log processing sample application(1). When I try to run the query it throws a
> NullPointerException and suggests I disable the new logical plan. Disabling
> it works and the script succeeds. Here is the query I'm trying to run:
> {{register file:/home/hadoop/lib/pig/piggybank.jar
> DEFINE EXTRACT org.apache.pig.piggybank.evaluation.string.EXTRACT();
> RAW_LOGS = LOAD '$INPUT' USING TextLoader as (line:chararray);
> LOGS_BASE= foreach RAW_LOGS generate FLATTEN(EXTRACT(line, '^(\\S+) (\\S+)
> (\\S+) \\[([\\w:/]+\\s[+\\-]\\d{4})\\] "(.+?)" (\\S+) (\\S+) "([^"]*)"
> "([^"]*)"')) as (remoteAddr:chararray, remoteLogname:chararray,
> user:chararray, time:chararray, request:chararray, status:int,
> bytes_string:chararray, referrer:chararray, browser:chararray);
> REFERRER_ONLY = FOREACH LOGS_BASE GENERATE referrer;
> FILTERED = FILTER REFERRER_ONLY BY referrer matches '.*bing.*' OR referrer
> matches '.*google.*';
> SEARCH_TERMS = FOREACH FILTERED GENERATE FLATTEN(EXTRACT(referrer,
> '.*[&\\?]q=([^&]+).*')) as terms:chararray;
> SEARCH_TERMS_FILTERED = FILTER SEARCH_TERMS BY NOT $0 IS NULL;
> SEARCH_TERMS_COUNT = FOREACH (GROUP SEARCH_TERMS_FILTERED BY $0) GENERATE
> $0, COUNT($1) as num;
> SEARCH_TERMS_COUNT_SORTED = LIMIT(ORDER SEARCH_TERMS_COUNT BY num DESC) 50;
> STORE SEARCH_TERMS_COUNT_SORTED into '$OUTPUT';}}
> And here is the stack trace that results:
> {{ERROR 2042: Error in new logical plan. Try -Dpig.usenewlogicalplan=false.
> org.apache.pig.backend.executionengine.ExecException: ERROR 2042: Error in
> new logical plan. Try -Dpig.usenewlogicalplan=false.
> at
> org.apache.pig.backend.hadoop.executionengine.HExecutionEngine.compile(HExecutionEngine.java:285)
> at org.apache.pig.PigServer.compilePp(PigServer.java:1301)
> at
> org.apache.pig.PigServer.executeCompiledLogicalPlan(PigServer.java:1154)
> at org.apache.pig.PigServer.execute(PigServer.java:1148)
> at org.apache.pig.PigServer.access$100(PigServer.java:123)
> at org.apache.pig.PigServer$Graph.execute(PigServer.java:1464)
> at org.apache.pig.PigServer.executeBatchEx(PigServer.java:350)
> at org.apache.pig.PigServer.executeBatch(PigServer.java:324)
> at
> org.apache.pig.tools.grunt.GruntParser.executeBatch(GruntParser.java:111)
> at
> org.apache.pig.tools.grunt.GruntParser.parseStopOnError(GruntParser.java:168)
> at
> org.apache.pig.tools.grunt.GruntParser.parseStopOnError(GruntParser.java:140)
> at org.apache.pig.tools.grunt.Grunt.exec(Grunt.java:90)
> at org.apache.pig.Main.run(Main.java:491)
> at org.apache.pig.Main.main(Main.java:107)
> at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
> at
> sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:39)
> at
> sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:25)
> at java.lang.reflect.Method.invoke(Method.java:597)
> at org.apache.hadoop.util.RunJar.main(RunJar.java:156)
> Caused by: java.lang.NullPointerException
> at org.apache.pig.EvalFunc.getSchemaName(EvalFunc.java:76)
> at
> org.apache.pig.piggybank.impl.ErrorCatchingBase.outputSchema(ErrorCatchingBase.java:76)
> at
> org.apache.pig.newplan.logical.expression.UserFuncExpression.getFieldSchema(UserFuncExpression.java:111)
> at
> org.apache.pig.newplan.logical.optimizer.FieldSchemaResetter.execute(SchemaResetter.java:175)
> at
> org.apache.pig.newplan.logical.expression.AllSameExpressionVisitor.visit(AllSameExpressionVisitor.java:143)
> at
> org.apache.pig.newplan.logical.expression.UserFuncExpression.accept(UserFuncExpression.java:55)
> at
> org.apache.pig.newplan.ReverseDependencyOrderWalker.walk(ReverseDependencyOrderWalker.java:69)
> at org.apache.pig.newplan.PlanVisitor.visit(PlanVisitor.java:50)
> at
> org.apache.pig.newplan.logical.optimizer.SchemaResetter.visit(SchemaResetter.java:87)
> at
> org.apache.pig.newplan.logical.relational.LOGenerate.accept(LOGenerate.java:149)
> at
> org.apache.pig.newplan.DependencyOrderWalker.walk(DependencyOrderWalker.java:74)
> at
> org.apache.pig.newplan.logical.optimizer.SchemaResetter.visit(SchemaResetter.java:76)
> at
> org.apache.pig.newplan.logical.relational.LOForEach.accept(LOForEach.java:71)
> at
> org.apache.pig.newplan.DependencyOrderWalker.walk(DependencyOrderWalker.java:74)
> at org.apache.pig.newplan.PlanVisitor.visit(PlanVisitor.java:50)
> at
> org.apache.pig.backend.hadoop.executionengine.HExecutionEngine.compile(HExecutionEngine.java:247)
> ... 18 more
> ================================================================================}}
> 1. http://developer.amazonwebservices.com/connect/entry.jspa?externalID=2729
--
This message is automatically generated by JIRA.
-
You can reply to this email to add a comment to the issue online.
