[ https://issues.apache.org/jira/browse/SPARK-19418?page=com.atlassian.jira.plugin.system.issuetabpanels:all-tabpanel ]
Suresh Avadhanula updated SPARK-19418: -------------------------------------- Attachment: encodertest.zip Testcase to reproduce the error. {code:title=Compile|borderStyle=solid} $mvn -P local clean install {code} {code:title=Run|borderStyle=solid} $ java -cp target/encodertest-1.0-SNAPSHOT.jar com.example.spark.EncoderTest {code} > Dataset generated java code fails to compile as java.lang.Long does not > accept UTF8String in constructor > -------------------------------------------------------------------------------------------------------- > > Key: SPARK-19418 > URL: https://issues.apache.org/jira/browse/SPARK-19418 > Project: Spark > Issue Type: Bug > Components: SQL > Affects Versions: 2.1.0 > Reporter: Suresh Avadhanula > Attachments: encodertest.zip > > > I have the following in Java spark driver. > DealerPerson module object is > {code:title=DealerPerson.java|borderStyle=solid} > public class DealerPerson > { > Long schemaOrgUnitId ; > List<Person> personList > } > {code} > I populate it using group by as follows. > {code} > Dataset<DealerPerson> dps = persondds.groupByKey(new MapFunction<Person, > Long>() { > @Override > public Long call(Person person) throws Exception { > return person.getSchemaOrgUnitId(); > } > }, Encoders.LONG()). > mapGroups(new MapGroupsFunction<Long, Person, DealerPerson>() > { > @Override > public DealerPerson call(Long dp, > java.util.Iterator<Person> iterator) throws Exception { > DealerPerson retDp = new DealerPerson(); > retDp.setSchemaOrgUnitId(dp); > ArrayList<Person> persons = new ArrayList<Person>(); > while (iterator.hasNext()) > persons.add(iterator.next()); > retDp.setPersons(persons); > return retDp; > } > }, Encoders.bean(DealerPerson.class)); > {code} > The generated code throws compiler exception since UTF8String is > java.lang.Long() > {noformat} > 7/01/31 20:32:28 INFO TaskSetManager: Starting task 0.0 in stage 5.0 (TID 5, > localhost, executor driver, partition 0, PROCESS_LOCAL, 6442 bytes) > 17/01/31 20:32:28 INFO Executor: Running task 0.0 in stage 5.0 (TID 5) > 17/01/31 20:32:28 ERROR CodeGenerator: failed to compile: > org.codehaus.commons.compiler.CompileException: File 'generated.java', Line > 56, Column 58: No applicable constructor/method found for actual parameters > "org.apache.spark.unsafe.types.UTF8String"; candidates are: > "java.lang.Long(long)", "java.lang.Long(java.lang.String)" > /* 001 */ public java.lang.Object generate(Object[] references) { > /* 002 */ return new SpecificSafeProjection(references); > /* 003 */ } > /* 004 */ > /* 005 */ class SpecificSafeProjection extends > org.apache.spark.sql.catalyst.expressions.codegen.BaseProjection { > /* 006 */ > /* 007 */ private Object[] references; > /* 008 */ private InternalRow mutableRow; > /* 009 */ private com.xtime.spark.model.Person javaBean; > /* 010 */ private boolean resultIsNull; > /* 011 */ private UTF8String argValue; > /* 012 */ private boolean resultIsNull1; > /* 013 */ private UTF8String argValue1; > /* 014 */ private boolean resultIsNull2; > /* 015 */ private UTF8String argValue2; > /* 016 */ private boolean resultIsNull3; > /* 017 */ private UTF8String argValue3; > /* 018 */ private boolean resultIsNull4; > /* 019 */ private UTF8String argValue4; > /* 020 */ > /* 021 */ public SpecificSafeProjection(Object[] references) { > /* 022 */ this.references = references; > /* 023 */ mutableRow = (InternalRow) references[references.length - 1]; > /* 024 */ > /* 025 */ > /* 026 */ > /* 027 */ > /* 028 */ > /* 029 */ > /* 030 */ > /* 031 */ > /* 032 */ > /* 033 */ > /* 034 */ > /* 035 */ > /* 036 */ } > /* 037 */ > /* 038 */ public void initialize(int partitionIndex) { > /* 039 */ > /* 040 */ } > /* 041 */ > /* 042 */ > /* 043 */ private void apply_4(InternalRow i) { > /* 044 */ > /* 045 */ > /* 046 */ resultIsNull1 = false; > /* 047 */ if (!resultIsNull1) { > /* 048 */ > /* 049 */ boolean isNull21 = i.isNullAt(17); > /* 050 */ UTF8String value21 = isNull21 ? null : (i.getUTF8String(17)); > /* 051 */ resultIsNull1 = isNull21; > /* 052 */ argValue1 = value21; > /* 053 */ } > /* 054 */ > /* 055 */ > /* 056 */ final java.lang.Long value20 = resultIsNull1 ? null : new > java.lang.Long(argValue1); > /* 057 */ javaBean.setSchemaOrgUnitId(value20); > /* 058 */ > /* 059 */ > /* 060 */ resultIsNull2 = false; > /* 061 */ if (!resultIsNull2) { > /* 062 */ > /* 063 */ boolean isNull23 = i.isNullAt(0); > /* 064 */ UTF8String value23 = isNull23 ? null : (i.getUTF8String(0)); > /* 065 */ resultIsNull2 = isNull23; > /* 066 */ argValue2 = value23; > /* 067 */ } > /* 068 */ > /* 069 */ > /* 070 */ final java.lang.Long value22 = resultIsNull2 ? null : new > java.lang.Long(argValue2); > /* 071 */ javaBean.setPersonId(value22); > /* 072 */ > /* 073 */ > /* 074 */ boolean isNull25 = i.isNullAt(10); > /* 075 */ UTF8String value25 = isNull25 ? null : (i.getUTF8String(10)); > /* 076 */ boolean isNull24 = true; > /* 077 */ java.lang.String value24 = null; > /* 078 */ if (!isNull25) { > /* 079 */ > /* 080 */ isNull24 = false; > /* 081 */ if (!isNull24) { > /* 082 */ > /* 083 */ Object funcResult8 = null; > /* 084 */ funcResult8 = value25.toString(); > /* 085 */ if (funcResult8 == null) { > /* 086 */ isNull24 = true; > /* 087 */ } else { > /* 088 */ value24 = (java.lang.String) funcResult8; > /* 089 */ } > /* 090 */ > /* 091 */ } > /* 092 */ isNull24 = value24 == null; > /* 093 */ } > /* 094 */ javaBean.setCreateMethod(value24); > /* 095 */ > /* 096 */ } > /* 097 */ > /* 098 */ > /* > {noformat} -- This message was sent by Atlassian JIRA (v6.3.15#6346) --------------------------------------------------------------------- To unsubscribe, e-mail: issues-unsubscr...@spark.apache.org For additional commands, e-mail: issues-h...@spark.apache.org