[
https://issues.apache.org/jira/browse/PIG-1835?page=com.atlassian.jira.plugin.system.issuetabpanels:all-tabpanel
]
Vivek Padmanabhan updated PIG-1835:
-----------------------------------
Description:
I have the below script which is throwing class cast exception while doing SUM.
Even though all the fields are properly typed, while computing sum in m_agg0
and m_agg02 the record from tuple is coming as java.lang.Long instead of Double.
The problem is happening in Pig 0.9. It works fine with 0.9 if I flag off new
logical plan by -Dpig.usenewlogicalplan=false.
{code}
A0 = load 'inputA' using PigStorage('\t') as ( group_id, r_id:long,
is_phase2:int, roi_value:double,roi_cost:double,ecpm, prob:double,pixel_id,
pixel_type,
val:long,f3, f4,type:long, amount:double,item_id:long);
A0 = foreach A0 generate r_id, is_phase2, ((val==257 or val==258)? 1: 0) as
imps,
((val==257 or val==258)? amount: 0.0) as a_out, ((val==257 or
val==258)? item_id: 0) as a_item_id,
((val==257 or val==258)? roi_value: 0.0) as roi_value,((val==257 or
val==258)? roi_cost: 0.0) as roi_cost,
((val==257 or val==513)? ecpm: 0.0) as ecpm, ((val==257 or val==513)?
prob: 0.0) as prob,
((val==257 or val==513)? amount: 0.0) as pub_rev, ((val==257 or
val==513)? item_id: 0) as pub_line_id,((val==257 or val==513)? type: 0) as
pub_pt;
-------------------------------------------------------------------------------------------------------------------------------------------------
B0 = load 'inputB' using PigStorage('\t') as ( group_id:long, r_id:long,
roi_value:double,roi_cost:double,receive_time,
host_name,site_id,rm_has_cookies,rm_pearl_id,
f1,f2,pixel_id:long,pixel_type:int, xcookie,val:long,f3,
f4,type:long,amount:double,item_id:long);
B0 = foreach B0 generate r_id, ((val==257 or val==258)? 1: 0) as B0,((val==257
or val==258)? amount: 0.0) as a_out,
((val==257 or val==258)? item_id: 0) as a_item_id,((val==257 or
val==258)? roi_value: 0.0) as roi_value,
((val==257 or val==258)? roi_cost: 0.0) as roi_cost, ((val==257 or
val==513)? amount: 0.0) as pub_rev,
((val==257 or val==513)? item_id: 0) as pub_line_id, ((val==257 or
val==513)? type: 0) as pub_pt;
------------------------------------------------------------------------------------------------------------------------------------------------
C0 = load 'inputC' using PigStorage('\t') as ( group_id:long, r_id:long,
roi_value:double, roi_cost:double, receive_time:long, host_name:chararray,
site_id:long, rm_has_cookies:int,rm_pearl_id:long,f1,f2, pixel_id:long,
pixel_type:int,rm_is_post_click:int,
rm_conversion_id,xcookie:chararray,val:long,f3:long,f4:long,type:long,amount:double,item_id:long);
C0 = foreach C0 generate r_id,((val==257 or val==258)? 1: 0) as C0,
((val==257 or val==258)? amount: 0.0) as a_out,
((val==257 or val==258)? item_id: 0) as a_item_id,((val==257 or
val==513)? amount: 0.0) as pub_rev,
((val==257 or val==513)? item_id: 0) as pub_line_id, ((val==257 or
val==513)? type: 0) as pub_pt;
------------------------------------------------------------------------------------------------------------------------------------------------
m_all = cogroup A0 by (r_id) outer, B0 by (r_id) outer, C0 by (r_id) outer ;
m_agg01 = foreach m_all generate (double)(IsEmpty(C0) ? 0.0 : SUM(C0.pub_rev))
as conv_pub_rev;
store m_agg01 into 'out1' USING PigStorage(',');
m_all = cogroup A0 by (r_id) outer, B0 by (r_id) outer, C0 by (r_id) outer ;
m_agg02 = foreach m_all generate (double)(IsEmpty(C0) ? 0.0 : SUM(C0.pub_rev))
as conv_pub_rev;
store m_agg02 into 'out2' USING PigStorage(',');
{code}
The below are the inputs to the script (all single record and tab seperated)
{code}
inputA
------
1111 1111 1 1.1 1.1 1.1 1.1 1111 1 1111
1111 1111 1111 1.1 1111
inputB
------
1111 1111 1.1 1.1 1111 a1 1111 1 1111 b1
1 1111 1 c1 1111 1111 1111 1111 1.1 1111
inputC
------
1111 1111 1.1 1.1 1111 a1 1111 1 1111 b1
1 1111 1 1 1111 c1 1111 1111 1111 1111
1.1 1111
{code}
Exception from reducers
_______________________
org.apache.pig.backend.executionengine.ExecException: ERROR 2103: Problem while
computing sum of doubles.
at org.apache.pig.builtin.DoubleSum.sum(DoubleSum.java:147)
at org.apache.pig.builtin.DoubleSum.exec(DoubleSum.java:46)
at org.apache.pig.builtin.DoubleSum.exec(DoubleSum.java:41)
at
org.apache.pig.backend.hadoop.executionengine.physicalLayer.expressionOperators.POUserFunc.getNext(POUserFunc.java:230)
at
org.apache.pig.backend.hadoop.executionengine.physicalLayer.expressionOperators.POUserFunc.getNext(POUserFunc.java:302)
at
org.apache.pig.backend.hadoop.executionengine.physicalLayer.expressionOperators.POBinCond.getNext(POBinCond.java:140)
at
org.apache.pig.backend.hadoop.executionengine.physicalLayer.expressionOperators.POCast.getNext(POCast.java:446)
at
org.apache.pig.backend.hadoop.executionengine.physicalLayer.relationalOperators.POForEach.processPlan(POForEach.java:346)
at
org.apache.pig.backend.hadoop.executionengine.physicalLayer.relationalOperators.POForEach.getNext(POForEach.java:289)
at
org.apache.pig.backend.hadoop.executionengine.mapReduceLayer.PigMapReduce$Reduce.runPipeline(PigMapReduce.java:455)
at
org.apache.pig.backend.hadoop.executionengine.mapReduceLayer.PigMapReduce$Reduce.processOnePackageOutput(PigMapReduce.java:423)
at
org.apache.pig.backend.hadoop.executionengine.mapReduceLayer.PigMapReduce$Reduce.reduce(PigMapReduce.java:403)
at
org.apache.pig.backend.hadoop.executionengine.mapReduceLayer.PigMapReduce$Reduce.reduce(PigMapReduce.java:258)
at org.apache.hadoop.mapreduce.Reducer.run(Reducer.java:176)
at
org.apache.hadoop.mapred.ReduceTask.runNewReducer(ReduceTask.java:572)
at org.apache.hadoop.mapred.ReduceTask.run(ReduceTask.java:414)
at org.apache.hadoop.mapred.Child$4.run(Child.java:242)
at java.security.AccessController.doPrivileged(Native Method)
at javax.security.auth.Subject.doAs(Subject.java:396)
at
org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1062)
at org.apache.hadoop.mapred.Child.main(Child.java:236)
Caused by: java.lang.ClassCastException: java.lang.Long cannot be cast to
java.lang.Double
at org.apache.pig.builtin.DoubleSum.sum(DoubleSum.java:140)
... 20 more
was:
I have the below script which is throwing class cast exception while doing SUM.
Even though all the fields are properly typed, while computing sum in m_agg0
and m_agg02 the record from tuple is coming as java.lang.Long instead of Double.
The problem is happening in Pig 0.9. It works fine with 0.9 if I flag off new
logical plan by -Dpig.usenewlogicalplan=false.
{code}
A0 = load 'inputA' using PigStorage('\t') as ( group_id, r_id:long,
is_phase2:int, roi_value:double,roi_cost:double,ecpm, prob:double,pixel_id,
pixel_type,
val:long,f3, f4,type:long, amount:double,item_id:long);
A0 = foreach A0 generate r_id, is_phase2, ((val==257 or val==258)? 1: 0) as
imps,
((val==257 or val==258)? amount: 0.0) as a_out, ((val==257 or
val==258)? item_id: 0) as a_item_id,
((val==257 or val==258)? roi_value: 0.0) as roi_value,((val==257 or
val==258)? roi_cost: 0.0) as roi_cost,
((val==257 or val==513)? ecpm: 0.0) as ecpm, ((val==257 or val==513)?
prob: 0.0) as prob,
((val==257 or val==513)? amount: 0.0) as pub_rev, ((val==257 or
val==513)? item_id: 0) as pub_line_id,((val==257 or val==513)? type: 0) as
pub_pt;
-------------------------------------------------------------------------------------------------------------------------------------------------
B0 = load 'inputB' using PigStorage('\t') as ( group_id:long, r_id:long,
roi_value:double,roi_cost:double,receive_time,
host_name,site_id,rm_has_cookies,rm_pearl_id,
f1,f2,pixel_id:long,pixel_type:int, xcookie,val:long,f3,
f4,type:long,amount:double,item_id:long);
B0 = foreach B0 generate r_id, ((val==257 or val==258)? 1: 0) as B0,((val==257
or val==258)? amount: 0.0) as a_out,
((val==257 or val==258)? item_id: 0) as a_item_id,((val==257 or
val==258)? roi_value: 0.0) as roi_value,
((val==257 or val==258)? roi_cost: 0.0) as roi_cost, ((val==257 or
val==513)? amount: 0.0) as pub_rev,
((val==257 or val==513)? item_id: 0) as pub_line_id, ((val==257 or
val==513)? type: 0) as pub_pt;
------------------------------------------------------------------------------------------------------------------------------------------------
C0 = load 'inputC' using PigStorage('\t') as ( group_id:long, r_id:long,
roi_value:double, roi_cost:double, receive_time:long, host_name:chararray,
site_id:long, rm_has_cookies:int,rm_pearl_id:long,f1,f2, pixel_id:long,
pixel_type:int,rm_is_post_click:int,
rm_conversion_id,xcookie:chararray,val:long,f3:long,f4:long,type:long,amount:double,item_id:long);
C0 = foreach C0 generate r_id,((val==257 or val==258)? 1: 0) as C0,
((val==257 or val==258)? amount: 0.0) as a_out,
((val==257 or val==258)? item_id: 0) as a_item_id,((val==257 or
val==513)? amount: 0.0) as pub_rev,
((val==257 or val==513)? item_id: 0) as pub_line_id, ((val==257 or
val==513)? type: 0) as pub_pt;
------------------------------------------------------------------------------------------------------------------------------------------------
m_all = cogroup A0 by (r_id) outer, B0 by (r_id) outer, C0 by (r_id) outer ;
m_agg01 = foreach m_all generate (double)(IsEmpty(C0) ? 0.0 : SUM(C0.pub_rev))
as conv_pub_rev;
store m_agg01 into 'out1' USING PigStorage(',');
m_all = cogroup A0 by (r_id) outer, B0 by (r_id) outer, C0 by (r_id) outer ;
m_agg02 = foreach m_all generate (double)(IsEmpty(C0) ? 0.0 : SUM(C0.pub_rev))
as conv_pub_rev;
store m_agg02 into 'out2' USING PigStorage(',');
{code}
The below are the inputs to the script (all single record and tab seperated)
inputA
------
1111 1111 1 1.1 1.1 1.1 1.1 1111 1 1111
1111 1111 1111 1.1 1111
inputB
------
1111 1111 1.1 1.1 1111 a1 1111 1 1111 b1
1 1111 1 c1 1111 1111 1111 1111 1.1 1111
inputC
------
1111 1111 1.1 1.1 1111 a1 1111 1 1111 b1
1 1111 1 1 1111 c1 1111 1111 1111 1111
1.1 1111
Exception from reducers
_______________________
org.apache.pig.backend.executionengine.ExecException: ERROR 2103: Problem while
computing sum of doubles.
at org.apache.pig.builtin.DoubleSum.sum(DoubleSum.java:147)
at org.apache.pig.builtin.DoubleSum.exec(DoubleSum.java:46)
at org.apache.pig.builtin.DoubleSum.exec(DoubleSum.java:41)
at
org.apache.pig.backend.hadoop.executionengine.physicalLayer.expressionOperators.POUserFunc.getNext(POUserFunc.java:230)
at
org.apache.pig.backend.hadoop.executionengine.physicalLayer.expressionOperators.POUserFunc.getNext(POUserFunc.java:302)
at
org.apache.pig.backend.hadoop.executionengine.physicalLayer.expressionOperators.POBinCond.getNext(POBinCond.java:140)
at
org.apache.pig.backend.hadoop.executionengine.physicalLayer.expressionOperators.POCast.getNext(POCast.java:446)
at
org.apache.pig.backend.hadoop.executionengine.physicalLayer.relationalOperators.POForEach.processPlan(POForEach.java:346)
at
org.apache.pig.backend.hadoop.executionengine.physicalLayer.relationalOperators.POForEach.getNext(POForEach.java:289)
at
org.apache.pig.backend.hadoop.executionengine.mapReduceLayer.PigMapReduce$Reduce.runPipeline(PigMapReduce.java:455)
at
org.apache.pig.backend.hadoop.executionengine.mapReduceLayer.PigMapReduce$Reduce.processOnePackageOutput(PigMapReduce.java:423)
at
org.apache.pig.backend.hadoop.executionengine.mapReduceLayer.PigMapReduce$Reduce.reduce(PigMapReduce.java:403)
at
org.apache.pig.backend.hadoop.executionengine.mapReduceLayer.PigMapReduce$Reduce.reduce(PigMapReduce.java:258)
at org.apache.hadoop.mapreduce.Reducer.run(Reducer.java:176)
at
org.apache.hadoop.mapred.ReduceTask.runNewReducer(ReduceTask.java:572)
at org.apache.hadoop.mapred.ReduceTask.run(ReduceTask.java:414)
at org.apache.hadoop.mapred.Child$4.run(Child.java:242)
at java.security.AccessController.doPrivileged(Native Method)
at javax.security.auth.Subject.doAs(Subject.java:396)
at
org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1062)
at org.apache.hadoop.mapred.Child.main(Child.java:236)
Caused by: java.lang.ClassCastException: java.lang.Long cannot be cast to
java.lang.Double
at org.apache.pig.builtin.DoubleSum.sum(DoubleSum.java:140)
... 20 more
> Pig 0.9 new logical plan throws class cast exception
> ----------------------------------------------------
>
> Key: PIG-1835
> URL: https://issues.apache.org/jira/browse/PIG-1835
> Project: Pig
> Issue Type: Bug
> Affects Versions: 0.9.0
> Reporter: Vivek Padmanabhan
>
> I have the below script which is throwing class cast exception while doing
> SUM. Even though all the fields are properly typed, while computing sum in
> m_agg0 and m_agg02 the record from tuple is coming as java.lang.Long instead
> of Double.
> The problem is happening in Pig 0.9. It works fine with 0.9 if I flag off new
> logical plan by -Dpig.usenewlogicalplan=false.
> {code}
> A0 = load 'inputA' using PigStorage('\t') as ( group_id, r_id:long,
> is_phase2:int, roi_value:double,roi_cost:double,ecpm, prob:double,pixel_id,
> pixel_type,
> val:long,f3, f4,type:long, amount:double,item_id:long);
> A0 = foreach A0 generate r_id, is_phase2, ((val==257 or val==258)? 1: 0) as
> imps,
> ((val==257 or val==258)? amount: 0.0) as a_out, ((val==257 or
> val==258)? item_id: 0) as a_item_id,
> ((val==257 or val==258)? roi_value: 0.0) as roi_value,((val==257 or
> val==258)? roi_cost: 0.0) as roi_cost,
> ((val==257 or val==513)? ecpm: 0.0) as ecpm, ((val==257 or val==513)?
> prob: 0.0) as prob,
> ((val==257 or val==513)? amount: 0.0) as pub_rev, ((val==257 or
> val==513)? item_id: 0) as pub_line_id,((val==257 or val==513)? type: 0) as
> pub_pt;
> -------------------------------------------------------------------------------------------------------------------------------------------------
> B0 = load 'inputB' using PigStorage('\t') as ( group_id:long, r_id:long,
> roi_value:double,roi_cost:double,receive_time,
> host_name,site_id,rm_has_cookies,rm_pearl_id,
> f1,f2,pixel_id:long,pixel_type:int, xcookie,val:long,f3,
> f4,type:long,amount:double,item_id:long);
> B0 = foreach B0 generate r_id, ((val==257 or val==258)? 1: 0) as
> B0,((val==257 or val==258)? amount: 0.0) as a_out,
> ((val==257 or val==258)? item_id: 0) as a_item_id,((val==257 or
> val==258)? roi_value: 0.0) as roi_value,
> ((val==257 or val==258)? roi_cost: 0.0) as roi_cost, ((val==257 or
> val==513)? amount: 0.0) as pub_rev,
> ((val==257 or val==513)? item_id: 0) as pub_line_id, ((val==257 or
> val==513)? type: 0) as pub_pt;
> ------------------------------------------------------------------------------------------------------------------------------------------------
> C0 = load 'inputC' using PigStorage('\t') as ( group_id:long, r_id:long,
> roi_value:double, roi_cost:double, receive_time:long, host_name:chararray,
> site_id:long, rm_has_cookies:int,rm_pearl_id:long,f1,f2, pixel_id:long,
> pixel_type:int,rm_is_post_click:int,
> rm_conversion_id,xcookie:chararray,val:long,f3:long,f4:long,type:long,amount:double,item_id:long);
> C0 = foreach C0 generate r_id,((val==257 or val==258)? 1: 0) as C0,
> ((val==257 or val==258)? amount: 0.0) as a_out,
> ((val==257 or val==258)? item_id: 0) as a_item_id,((val==257 or
> val==513)? amount: 0.0) as pub_rev,
> ((val==257 or val==513)? item_id: 0) as pub_line_id, ((val==257 or
> val==513)? type: 0) as pub_pt;
> ------------------------------------------------------------------------------------------------------------------------------------------------
> m_all = cogroup A0 by (r_id) outer, B0 by (r_id) outer, C0 by (r_id) outer
> ;
> m_agg01 = foreach m_all generate (double)(IsEmpty(C0) ? 0.0 :
> SUM(C0.pub_rev)) as conv_pub_rev;
> store m_agg01 into 'out1' USING PigStorage(',');
> m_all = cogroup A0 by (r_id) outer, B0 by (r_id) outer, C0 by (r_id) outer
> ;
> m_agg02 = foreach m_all generate (double)(IsEmpty(C0) ? 0.0 :
> SUM(C0.pub_rev)) as conv_pub_rev;
> store m_agg02 into 'out2' USING PigStorage(',');
> {code}
> The below are the inputs to the script (all single record and tab seperated)
> {code}
> inputA
> ------
> 1111 1111 1 1.1 1.1 1.1 1.1 1111 1 1111
> 1111 1111 1111 1.1 1111
> inputB
> ------
> 1111 1111 1.1 1.1 1111 a1 1111 1 1111 b1
> 1 1111 1 c1 1111 1111 1111 1111 1.1 1111
> inputC
> ------
> 1111 1111 1.1 1.1 1111 a1 1111 1 1111 b1
> 1 1111 1 1 1111 c1 1111 1111 1111
> 1111 1.1 1111
> {code}
> Exception from reducers
> _______________________
> org.apache.pig.backend.executionengine.ExecException: ERROR 2103: Problem
> while computing sum of doubles.
> at org.apache.pig.builtin.DoubleSum.sum(DoubleSum.java:147)
> at org.apache.pig.builtin.DoubleSum.exec(DoubleSum.java:46)
> at org.apache.pig.builtin.DoubleSum.exec(DoubleSum.java:41)
> at
> org.apache.pig.backend.hadoop.executionengine.physicalLayer.expressionOperators.POUserFunc.getNext(POUserFunc.java:230)
> at
> org.apache.pig.backend.hadoop.executionengine.physicalLayer.expressionOperators.POUserFunc.getNext(POUserFunc.java:302)
> at
> org.apache.pig.backend.hadoop.executionengine.physicalLayer.expressionOperators.POBinCond.getNext(POBinCond.java:140)
> at
> org.apache.pig.backend.hadoop.executionengine.physicalLayer.expressionOperators.POCast.getNext(POCast.java:446)
> at
> org.apache.pig.backend.hadoop.executionengine.physicalLayer.relationalOperators.POForEach.processPlan(POForEach.java:346)
> at
> org.apache.pig.backend.hadoop.executionengine.physicalLayer.relationalOperators.POForEach.getNext(POForEach.java:289)
> at
> org.apache.pig.backend.hadoop.executionengine.mapReduceLayer.PigMapReduce$Reduce.runPipeline(PigMapReduce.java:455)
> at
> org.apache.pig.backend.hadoop.executionengine.mapReduceLayer.PigMapReduce$Reduce.processOnePackageOutput(PigMapReduce.java:423)
> at
> org.apache.pig.backend.hadoop.executionengine.mapReduceLayer.PigMapReduce$Reduce.reduce(PigMapReduce.java:403)
> at
> org.apache.pig.backend.hadoop.executionengine.mapReduceLayer.PigMapReduce$Reduce.reduce(PigMapReduce.java:258)
> at org.apache.hadoop.mapreduce.Reducer.run(Reducer.java:176)
> at
> org.apache.hadoop.mapred.ReduceTask.runNewReducer(ReduceTask.java:572)
> at org.apache.hadoop.mapred.ReduceTask.run(ReduceTask.java:414)
> at org.apache.hadoop.mapred.Child$4.run(Child.java:242)
> at java.security.AccessController.doPrivileged(Native Method)
> at javax.security.auth.Subject.doAs(Subject.java:396)
> at
> org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1062)
> at org.apache.hadoop.mapred.Child.main(Child.java:236)
> Caused by: java.lang.ClassCastException: java.lang.Long cannot be cast to
> java.lang.Double
> at org.apache.pig.builtin.DoubleSum.sum(DoubleSum.java:140)
> ... 20 more
--
This message is automatically generated by JIRA.
-
For more information on JIRA, see: http://www.atlassian.com/software/jira
