[jira] [Commented] (SPARK-18125) Spark generated code causes CompileException when groupByKey, reduceGroups and map(_._2) are used
[ https://issues.apache.org/jira/browse/SPARK-18125?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel&focusedCommentId=15611142#comment-15611142 ] Zach Kull commented on SPARK-18125: --- Same here: After groupByKey.reduceGroups.map(_._2): java.util.concurrent.ExecutionException: java.lang.Exception: failed to compile: org.codehaus.commons.compiler.CompileException: File 'generated.java', Line 31, Column 69: Unknown variable or type "value4" > Spark generated code causes CompileException when groupByKey, reduceGroups > and map(_._2) are used > - > > Key: SPARK-18125 > URL: https://issues.apache.org/jira/browse/SPARK-18125 > Project: Spark > Issue Type: Bug >Affects Versions: 2.0.1 >Reporter: Ray Qiu > > Code logic looks like this: > .groupByKey > .reduceGroups > .map(_._2) > Works fine with 2.0.0. > 2.0.1 error Message: > Caused by: java.util.concurrent.ExecutionException: java.lang.Exception: > failed to compile: org.codehaus.commons.compiler.CompileException: File > 'generated.java', Line 206, Column 123: Unknown variable or type "value4" > /* 001 */ public java.lang.Object generate(Object[] references) { > /* 002 */ return new SpecificMutableProjection(references); > /* 003 */ } > /* 004 */ > /* 005 */ class SpecificMutableProjection extends > org.apache.spark.sql.catalyst.expressions.codegen.BaseMutableProjection { > /* 006 */ > /* 007 */ private Object[] references; > /* 008 */ private MutableRow mutableRow; > /* 009 */ private Object[] values; > /* 010 */ private java.lang.String errMsg; > /* 011 */ private java.lang.String errMsg1; > /* 012 */ private boolean MapObjects_loopIsNull1; > /* 013 */ private io.mistnet.analytics.lib.ConnLog MapObjects_loopValue0; > /* 014 */ private java.lang.String errMsg2; > /* 015 */ private Object[] values1; > /* 016 */ private boolean MapObjects_loopIsNull3; > /* 017 */ private java.lang.String MapObjects_loopValue2; > /* 018 */ private boolean isNull_0; > /* 019 */ private boolean value_0; > /* 020 */ private boolean isNull_1; > /* 021 */ private InternalRow value_1; > /* 022 */ > /* 023 */ private void apply_4(InternalRow i) { > /* 024 */ > /* 025 */ boolean isNull52 = MapObjects_loopIsNull1; > /* 026 */ final double value52 = isNull52 ? -1.0 : > MapObjects_loopValue0.ts(); > /* 027 */ if (isNull52) { > /* 028 */ values1[8] = null; > /* 029 */ } else { > /* 030 */ values1[8] = value52; > /* 031 */ } > /* 032 */ boolean isNull54 = MapObjects_loopIsNull1; > /* 033 */ final java.lang.String value54 = isNull54 ? null : > (java.lang.String) MapObjects_loopValue0.uid(); > /* 034 */ isNull54 = value54 == null; > /* 035 */ boolean isNull53 = isNull54; > /* 036 */ final UTF8String value53 = isNull53 ? null : > org.apache.spark.unsafe.types.UTF8String.fromString(value54); > /* 037 */ isNull53 = value53 == null; > /* 038 */ if (isNull53) { > /* 039 */ values1[9] = null; > /* 040 */ } else { > /* 041 */ values1[9] = value53; > /* 042 */ } > /* 043 */ boolean isNull56 = MapObjects_loopIsNull1; > /* 044 */ final java.lang.String value56 = isNull56 ? null : > (java.lang.String) MapObjects_loopValue0.src(); > /* 045 */ isNull56 = value56 == null; > /* 046 */ boolean isNull55 = isNull56; > /* 047 */ final UTF8String value55 = isNull55 ? null : > org.apache.spark.unsafe.types.UTF8String.fromString(value56); > /* 048 */ isNull55 = value55 == null; > /* 049 */ if (isNull55) { > /* 050 */ values1[10] = null; > /* 051 */ } else { > /* 052 */ values1[10] = value55; > /* 053 */ } > /* 054 */ } > /* 055 */ > /* 056 */ > /* 057 */ private void apply_7(InternalRow i) { > /* 058 */ > /* 059 */ boolean isNull69 = MapObjects_loopIsNull1; > /* 060 */ final scala.Option value69 = isNull69 ? null : (scala.Option) > MapObjects_loopValue0.orig_bytes(); > /* 061 */ isNull69 = value69 == null; > /* 062 */ > /* 063 */ final boolean isNull68 = isNull69 || value69.isEmpty(); > /* 064 */ long value68 = isNull68 ? > /* 065 */ -1L : (Long) value69.get(); > /* 066 */ if (isNull68) { > /* 067 */ values1[17] = null; > /* 068 */ } else { > /* 069 */ values1[17] = value68; > /* 070 */ } > /* 071 */ boolean isNull71 = MapObjects_loopIsNull1; > /* 072 */ final scala.Option value71 = isNull71 ? null : (scala.Option) > MapObjects_loopValue0.resp_bytes(); > /* 073 */ isNull71 = value71 == null; > /* 074 */ > /* 075 */ final boolean isNull70 = isNull71 || value71.isEmpty(); > /* 076 */ long value70 = isNull70 ? > /* 077 */ -1L : (Long) value71.get(); > /* 078 */ if (isNull70) { > /* 079 */ values1[18]
[jira] [Commented] (SPARK-18125) Spark generated code causes CompileException when groupByKey, reduceGroups and map(_._2) are used
[ https://issues.apache.org/jira/browse/SPARK-18125?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel&focusedCommentId=15613099#comment-15613099 ] Ray Qiu commented on SPARK-18125: - Move to Priority Critical unless a workaround is identified. This is a very basic functionality. > Spark generated code causes CompileException when groupByKey, reduceGroups > and map(_._2) are used > - > > Key: SPARK-18125 > URL: https://issues.apache.org/jira/browse/SPARK-18125 > Project: Spark > Issue Type: Bug >Affects Versions: 2.0.1 >Reporter: Ray Qiu >Priority: Critical > > Code logic looks like this: > .groupByKey > .reduceGroups > .map(_._2) > Works fine with 2.0.0. > 2.0.1 error Message: > Caused by: java.util.concurrent.ExecutionException: java.lang.Exception: > failed to compile: org.codehaus.commons.compiler.CompileException: File > 'generated.java', Line 206, Column 123: Unknown variable or type "value4" > /* 001 */ public java.lang.Object generate(Object[] references) { > /* 002 */ return new SpecificMutableProjection(references); > /* 003 */ } > /* 004 */ > /* 005 */ class SpecificMutableProjection extends > org.apache.spark.sql.catalyst.expressions.codegen.BaseMutableProjection { > /* 006 */ > /* 007 */ private Object[] references; > /* 008 */ private MutableRow mutableRow; > /* 009 */ private Object[] values; > /* 010 */ private java.lang.String errMsg; > /* 011 */ private java.lang.String errMsg1; > /* 012 */ private boolean MapObjects_loopIsNull1; > /* 013 */ private io.mistnet.analytics.lib.ConnLog MapObjects_loopValue0; > /* 014 */ private java.lang.String errMsg2; > /* 015 */ private Object[] values1; > /* 016 */ private boolean MapObjects_loopIsNull3; > /* 017 */ private java.lang.String MapObjects_loopValue2; > /* 018 */ private boolean isNull_0; > /* 019 */ private boolean value_0; > /* 020 */ private boolean isNull_1; > /* 021 */ private InternalRow value_1; > /* 022 */ > /* 023 */ private void apply_4(InternalRow i) { > /* 024 */ > /* 025 */ boolean isNull52 = MapObjects_loopIsNull1; > /* 026 */ final double value52 = isNull52 ? -1.0 : > MapObjects_loopValue0.ts(); > /* 027 */ if (isNull52) { > /* 028 */ values1[8] = null; > /* 029 */ } else { > /* 030 */ values1[8] = value52; > /* 031 */ } > /* 032 */ boolean isNull54 = MapObjects_loopIsNull1; > /* 033 */ final java.lang.String value54 = isNull54 ? null : > (java.lang.String) MapObjects_loopValue0.uid(); > /* 034 */ isNull54 = value54 == null; > /* 035 */ boolean isNull53 = isNull54; > /* 036 */ final UTF8String value53 = isNull53 ? null : > org.apache.spark.unsafe.types.UTF8String.fromString(value54); > /* 037 */ isNull53 = value53 == null; > /* 038 */ if (isNull53) { > /* 039 */ values1[9] = null; > /* 040 */ } else { > /* 041 */ values1[9] = value53; > /* 042 */ } > /* 043 */ boolean isNull56 = MapObjects_loopIsNull1; > /* 044 */ final java.lang.String value56 = isNull56 ? null : > (java.lang.String) MapObjects_loopValue0.src(); > /* 045 */ isNull56 = value56 == null; > /* 046 */ boolean isNull55 = isNull56; > /* 047 */ final UTF8String value55 = isNull55 ? null : > org.apache.spark.unsafe.types.UTF8String.fromString(value56); > /* 048 */ isNull55 = value55 == null; > /* 049 */ if (isNull55) { > /* 050 */ values1[10] = null; > /* 051 */ } else { > /* 052 */ values1[10] = value55; > /* 053 */ } > /* 054 */ } > /* 055 */ > /* 056 */ > /* 057 */ private void apply_7(InternalRow i) { > /* 058 */ > /* 059 */ boolean isNull69 = MapObjects_loopIsNull1; > /* 060 */ final scala.Option value69 = isNull69 ? null : (scala.Option) > MapObjects_loopValue0.orig_bytes(); > /* 061 */ isNull69 = value69 == null; > /* 062 */ > /* 063 */ final boolean isNull68 = isNull69 || value69.isEmpty(); > /* 064 */ long value68 = isNull68 ? > /* 065 */ -1L : (Long) value69.get(); > /* 066 */ if (isNull68) { > /* 067 */ values1[17] = null; > /* 068 */ } else { > /* 069 */ values1[17] = value68; > /* 070 */ } > /* 071 */ boolean isNull71 = MapObjects_loopIsNull1; > /* 072 */ final scala.Option value71 = isNull71 ? null : (scala.Option) > MapObjects_loopValue0.resp_bytes(); > /* 073 */ isNull71 = value71 == null; > /* 074 */ > /* 075 */ final boolean isNull70 = isNull71 || value71.isEmpty(); > /* 076 */ long value70 = isNull70 ? > /* 077 */ -1L : (Long) value71.get(); > /* 078 */ if (isNull70) { > /* 079 */ values1[18] = null; > /* 080 */ } else { > /* 081 */ values1[18] = value70; > /* 082 */ } > /* 083 */ boolean isNull74 = MapObje
[jira] [Commented] (SPARK-18125) Spark generated code causes CompileException when groupByKey, reduceGroups and map(_._2) are used
[ https://issues.apache.org/jira/browse/SPARK-18125?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel&focusedCommentId=15613497#comment-15613497 ] Herman van Hovell commented on SPARK-18125: --- Could one of you provide a reproducible example. > Spark generated code causes CompileException when groupByKey, reduceGroups > and map(_._2) are used > - > > Key: SPARK-18125 > URL: https://issues.apache.org/jira/browse/SPARK-18125 > Project: Spark > Issue Type: Bug >Affects Versions: 2.0.1 >Reporter: Ray Qiu >Priority: Critical > > Code logic looks like this: > {noformat} > .groupByKey > .reduceGroups > .map(_._2) > {noformat} > Works fine with 2.0.0. > 2.0.1 error Message: > {noformat} > Caused by: java.util.concurrent.ExecutionException: java.lang.Exception: > failed to compile: org.codehaus.commons.compiler.CompileException: File > 'generated.java', Line 206, Column 123: Unknown variable or type "value4" > /* 001 */ public java.lang.Object generate(Object[] references) { > /* 002 */ return new SpecificMutableProjection(references); > /* 003 */ } > /* 004 */ > /* 005 */ class SpecificMutableProjection extends > org.apache.spark.sql.catalyst.expressions.codegen.BaseMutableProjection { > /* 006 */ > /* 007 */ private Object[] references; > /* 008 */ private MutableRow mutableRow; > /* 009 */ private Object[] values; > /* 010 */ private java.lang.String errMsg; > /* 011 */ private java.lang.String errMsg1; > /* 012 */ private boolean MapObjects_loopIsNull1; > /* 013 */ private io.mistnet.analytics.lib.ConnLog MapObjects_loopValue0; > /* 014 */ private java.lang.String errMsg2; > /* 015 */ private Object[] values1; > /* 016 */ private boolean MapObjects_loopIsNull3; > /* 017 */ private java.lang.String MapObjects_loopValue2; > /* 018 */ private boolean isNull_0; > /* 019 */ private boolean value_0; > /* 020 */ private boolean isNull_1; > /* 021 */ private InternalRow value_1; > /* 022 */ > /* 023 */ private void apply_4(InternalRow i) { > /* 024 */ > /* 025 */ boolean isNull52 = MapObjects_loopIsNull1; > /* 026 */ final double value52 = isNull52 ? -1.0 : > MapObjects_loopValue0.ts(); > /* 027 */ if (isNull52) { > /* 028 */ values1[8] = null; > /* 029 */ } else { > /* 030 */ values1[8] = value52; > /* 031 */ } > /* 032 */ boolean isNull54 = MapObjects_loopIsNull1; > /* 033 */ final java.lang.String value54 = isNull54 ? null : > (java.lang.String) MapObjects_loopValue0.uid(); > /* 034 */ isNull54 = value54 == null; > /* 035 */ boolean isNull53 = isNull54; > /* 036 */ final UTF8String value53 = isNull53 ? null : > org.apache.spark.unsafe.types.UTF8String.fromString(value54); > /* 037 */ isNull53 = value53 == null; > /* 038 */ if (isNull53) { > /* 039 */ values1[9] = null; > /* 040 */ } else { > /* 041 */ values1[9] = value53; > /* 042 */ } > /* 043 */ boolean isNull56 = MapObjects_loopIsNull1; > /* 044 */ final java.lang.String value56 = isNull56 ? null : > (java.lang.String) MapObjects_loopValue0.src(); > /* 045 */ isNull56 = value56 == null; > /* 046 */ boolean isNull55 = isNull56; > /* 047 */ final UTF8String value55 = isNull55 ? null : > org.apache.spark.unsafe.types.UTF8String.fromString(value56); > /* 048 */ isNull55 = value55 == null; > /* 049 */ if (isNull55) { > /* 050 */ values1[10] = null; > /* 051 */ } else { > /* 052 */ values1[10] = value55; > /* 053 */ } > /* 054 */ } > /* 055 */ > /* 056 */ > /* 057 */ private void apply_7(InternalRow i) { > /* 058 */ > /* 059 */ boolean isNull69 = MapObjects_loopIsNull1; > /* 060 */ final scala.Option value69 = isNull69 ? null : (scala.Option) > MapObjects_loopValue0.orig_bytes(); > /* 061 */ isNull69 = value69 == null; > /* 062 */ > /* 063 */ final boolean isNull68 = isNull69 || value69.isEmpty(); > /* 064 */ long value68 = isNull68 ? > /* 065 */ -1L : (Long) value69.get(); > /* 066 */ if (isNull68) { > /* 067 */ values1[17] = null; > /* 068 */ } else { > /* 069 */ values1[17] = value68; > /* 070 */ } > /* 071 */ boolean isNull71 = MapObjects_loopIsNull1; > /* 072 */ final scala.Option value71 = isNull71 ? null : (scala.Option) > MapObjects_loopValue0.resp_bytes(); > /* 073 */ isNull71 = value71 == null; > /* 074 */ > /* 075 */ final boolean isNull70 = isNull71 || value71.isEmpty(); > /* 076 */ long value70 = isNull70 ? > /* 077 */ -1L : (Long) value71.get(); > /* 078 */ if (isNull70) { > /* 079 */ values1[18] = null; > /* 080 */ } else { > /* 081 */ values1[18] = value70; > /* 082 */ } > /* 083 */ boolean isNull74
[jira] [Commented] (SPARK-18125) Spark generated code causes CompileException when groupByKey, reduceGroups and map(_._2) are used
[ https://issues.apache.org/jira/browse/SPARK-18125?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel&focusedCommentId=15613580#comment-15613580 ] Herman van Hovell commented on SPARK-18125: --- I tried something like this on master and on branch-2.0: {noformat} val ds = spark.range(1).select($"id" % 100 as "grp_id", array($"id")).as[(Long, Seq[Long])] ds.groupByKey(_._1).reduceGroups((a, b) => (a._1, a._2 ++ b._2)).map(_._2) {noformat} > Spark generated code causes CompileException when groupByKey, reduceGroups > and map(_._2) are used > - > > Key: SPARK-18125 > URL: https://issues.apache.org/jira/browse/SPARK-18125 > Project: Spark > Issue Type: Bug >Affects Versions: 2.0.1 >Reporter: Ray Qiu >Priority: Critical > > Code logic looks like this: > {noformat} > .groupByKey > .reduceGroups > .map(_._2) > {noformat} > Works fine with 2.0.0. > 2.0.1 error Message: > {noformat} > Caused by: java.util.concurrent.ExecutionException: java.lang.Exception: > failed to compile: org.codehaus.commons.compiler.CompileException: File > 'generated.java', Line 206, Column 123: Unknown variable or type "value4" > /* 001 */ public java.lang.Object generate(Object[] references) { > /* 002 */ return new SpecificMutableProjection(references); > /* 003 */ } > /* 004 */ > /* 005 */ class SpecificMutableProjection extends > org.apache.spark.sql.catalyst.expressions.codegen.BaseMutableProjection { > /* 006 */ > /* 007 */ private Object[] references; > /* 008 */ private MutableRow mutableRow; > /* 009 */ private Object[] values; > /* 010 */ private java.lang.String errMsg; > /* 011 */ private java.lang.String errMsg1; > /* 012 */ private boolean MapObjects_loopIsNull1; > /* 013 */ private io.mistnet.analytics.lib.ConnLog MapObjects_loopValue0; > /* 014 */ private java.lang.String errMsg2; > /* 015 */ private Object[] values1; > /* 016 */ private boolean MapObjects_loopIsNull3; > /* 017 */ private java.lang.String MapObjects_loopValue2; > /* 018 */ private boolean isNull_0; > /* 019 */ private boolean value_0; > /* 020 */ private boolean isNull_1; > /* 021 */ private InternalRow value_1; > /* 022 */ > /* 023 */ private void apply_4(InternalRow i) { > /* 024 */ > /* 025 */ boolean isNull52 = MapObjects_loopIsNull1; > /* 026 */ final double value52 = isNull52 ? -1.0 : > MapObjects_loopValue0.ts(); > /* 027 */ if (isNull52) { > /* 028 */ values1[8] = null; > /* 029 */ } else { > /* 030 */ values1[8] = value52; > /* 031 */ } > /* 032 */ boolean isNull54 = MapObjects_loopIsNull1; > /* 033 */ final java.lang.String value54 = isNull54 ? null : > (java.lang.String) MapObjects_loopValue0.uid(); > /* 034 */ isNull54 = value54 == null; > /* 035 */ boolean isNull53 = isNull54; > /* 036 */ final UTF8String value53 = isNull53 ? null : > org.apache.spark.unsafe.types.UTF8String.fromString(value54); > /* 037 */ isNull53 = value53 == null; > /* 038 */ if (isNull53) { > /* 039 */ values1[9] = null; > /* 040 */ } else { > /* 041 */ values1[9] = value53; > /* 042 */ } > /* 043 */ boolean isNull56 = MapObjects_loopIsNull1; > /* 044 */ final java.lang.String value56 = isNull56 ? null : > (java.lang.String) MapObjects_loopValue0.src(); > /* 045 */ isNull56 = value56 == null; > /* 046 */ boolean isNull55 = isNull56; > /* 047 */ final UTF8String value55 = isNull55 ? null : > org.apache.spark.unsafe.types.UTF8String.fromString(value56); > /* 048 */ isNull55 = value55 == null; > /* 049 */ if (isNull55) { > /* 050 */ values1[10] = null; > /* 051 */ } else { > /* 052 */ values1[10] = value55; > /* 053 */ } > /* 054 */ } > /* 055 */ > /* 056 */ > /* 057 */ private void apply_7(InternalRow i) { > /* 058 */ > /* 059 */ boolean isNull69 = MapObjects_loopIsNull1; > /* 060 */ final scala.Option value69 = isNull69 ? null : (scala.Option) > MapObjects_loopValue0.orig_bytes(); > /* 061 */ isNull69 = value69 == null; > /* 062 */ > /* 063 */ final boolean isNull68 = isNull69 || value69.isEmpty(); > /* 064 */ long value68 = isNull68 ? > /* 065 */ -1L : (Long) value69.get(); > /* 066 */ if (isNull68) { > /* 067 */ values1[17] = null; > /* 068 */ } else { > /* 069 */ values1[17] = value68; > /* 070 */ } > /* 071 */ boolean isNull71 = MapObjects_loopIsNull1; > /* 072 */ final scala.Option value71 = isNull71 ? null : (scala.Option) > MapObjects_loopValue0.resp_bytes(); > /* 073 */ isNull71 = value71 == null; > /* 074 */ > /* 075 */ final boolean isNull70 = isNull71 || value71.isEmpty(); > /* 076 */ long value70 = isNull70 ? > /* 077 */ -1L : (Long)
[jira] [Commented] (SPARK-18125) Spark generated code causes CompileException when groupByKey, reduceGroups and map(_._2) are used
[ https://issues.apache.org/jira/browse/SPARK-18125?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel&focusedCommentId=15613682#comment-15613682 ] Ray Qiu commented on SPARK-18125: - Try this in spark-shell: case class Route(src: String, dest: String, cost: Int) case class GroupedRoutes(src: String, dest: String, routes: Seq[Route]) import spark.implicits._ val ds = sc.parallelize(Array( Route("a", "b", 1), Route("a", "b", 2), Route("a", "c", 2), Route("a", "d", 10), Route("b", "a", 1), Route("b", "a", 5), Route("b", "c", 6)) ).toDF.as[Route] val grped = ds.map(r => GroupedRoutes(r.src, r.dest, Seq(r))) .groupByKey(r => (r.src, r.dest)) .reduceGroups { (g1: GroupedRoutes, g2: GroupedRoutes) => GroupedRoutes(g1.src, g1.dest, g1.routes ++ g2.routes) }.map(_._2) On Thu, Oct 27, 2016 at 4:06 PM, Herman van Hovell (JIRA) -- Regards, Ray > Spark generated code causes CompileException when groupByKey, reduceGroups > and map(_._2) are used > - > > Key: SPARK-18125 > URL: https://issues.apache.org/jira/browse/SPARK-18125 > Project: Spark > Issue Type: Bug >Affects Versions: 2.0.1 >Reporter: Ray Qiu >Priority: Critical > > Code logic looks like this: > {noformat} > .groupByKey > .reduceGroups > .map(_._2) > {noformat} > Works fine with 2.0.0. > 2.0.1 error Message: > {noformat} > Caused by: java.util.concurrent.ExecutionException: java.lang.Exception: > failed to compile: org.codehaus.commons.compiler.CompileException: File > 'generated.java', Line 206, Column 123: Unknown variable or type "value4" > /* 001 */ public java.lang.Object generate(Object[] references) { > /* 002 */ return new SpecificMutableProjection(references); > /* 003 */ } > /* 004 */ > /* 005 */ class SpecificMutableProjection extends > org.apache.spark.sql.catalyst.expressions.codegen.BaseMutableProjection { > /* 006 */ > /* 007 */ private Object[] references; > /* 008 */ private MutableRow mutableRow; > /* 009 */ private Object[] values; > /* 010 */ private java.lang.String errMsg; > /* 011 */ private java.lang.String errMsg1; > /* 012 */ private boolean MapObjects_loopIsNull1; > /* 013 */ private io.mistnet.analytics.lib.ConnLog MapObjects_loopValue0; > /* 014 */ private java.lang.String errMsg2; > /* 015 */ private Object[] values1; > /* 016 */ private boolean MapObjects_loopIsNull3; > /* 017 */ private java.lang.String MapObjects_loopValue2; > /* 018 */ private boolean isNull_0; > /* 019 */ private boolean value_0; > /* 020 */ private boolean isNull_1; > /* 021 */ private InternalRow value_1; > /* 022 */ > /* 023 */ private void apply_4(InternalRow i) { > /* 024 */ > /* 025 */ boolean isNull52 = MapObjects_loopIsNull1; > /* 026 */ final double value52 = isNull52 ? -1.0 : > MapObjects_loopValue0.ts(); > /* 027 */ if (isNull52) { > /* 028 */ values1[8] = null; > /* 029 */ } else { > /* 030 */ values1[8] = value52; > /* 031 */ } > /* 032 */ boolean isNull54 = MapObjects_loopIsNull1; > /* 033 */ final java.lang.String value54 = isNull54 ? null : > (java.lang.String) MapObjects_loopValue0.uid(); > /* 034 */ isNull54 = value54 == null; > /* 035 */ boolean isNull53 = isNull54; > /* 036 */ final UTF8String value53 = isNull53 ? null : > org.apache.spark.unsafe.types.UTF8String.fromString(value54); > /* 037 */ isNull53 = value53 == null; > /* 038 */ if (isNull53) { > /* 039 */ values1[9] = null; > /* 040 */ } else { > /* 041 */ values1[9] = value53; > /* 042 */ } > /* 043 */ boolean isNull56 = MapObjects_loopIsNull1; > /* 044 */ final java.lang.String value56 = isNull56 ? null : > (java.lang.String) MapObjects_loopValue0.src(); > /* 045 */ isNull56 = value56 == null; > /* 046 */ boolean isNull55 = isNull56; > /* 047 */ final UTF8String value55 = isNull55 ? null : > org.apache.spark.unsafe.types.UTF8String.fromString(value56); > /* 048 */ isNull55 = value55 == null; > /* 049 */ if (isNull55) { > /* 050 */ values1[10] = null; > /* 051 */ } else { > /* 052 */ values1[10] = value55; > /* 053 */ } > /* 054 */ } > /* 055 */ > /* 056 */ > /* 057 */ private void apply_7(InternalRow i) { > /* 058 */ > /* 059 */ boolean isNull69 = MapObjects_loopIsNull1; > /* 060 */ final scala.Option value69 = isNull69 ? null : (scala.Option) > MapObjects_loopValue0.orig_bytes(); > /* 061 */ isNull69 = value69 == null; > /* 062 */ > /* 063 */ final boolean isNull68 = isNull69 || value69.isEmpty(); > /* 064 */ long value68 = isNull68 ? > /* 065 */ -1L : (Long) value69.get(); > /* 066 */ if (isNull68) { > /* 067 */ values1[17] = null; > /* 068 */
[jira] [Commented] (SPARK-18125) Spark generated code causes CompileException when groupByKey, reduceGroups and map(_._2) are used
[ https://issues.apache.org/jira/browse/SPARK-18125?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel&focusedCommentId=15613704#comment-15613704 ] Ray Qiu commented on SPARK-18125: - Same thing works fine in 2.0.0 -- Regards, Ray > Spark generated code causes CompileException when groupByKey, reduceGroups > and map(_._2) are used > - > > Key: SPARK-18125 > URL: https://issues.apache.org/jira/browse/SPARK-18125 > Project: Spark > Issue Type: Bug >Affects Versions: 2.0.1 >Reporter: Ray Qiu >Priority: Critical > > Code logic looks like this: > {noformat} > .groupByKey > .reduceGroups > .map(_._2) > {noformat} > Works fine with 2.0.0. > 2.0.1 error Message: > {noformat} > Caused by: java.util.concurrent.ExecutionException: java.lang.Exception: > failed to compile: org.codehaus.commons.compiler.CompileException: File > 'generated.java', Line 206, Column 123: Unknown variable or type "value4" > /* 001 */ public java.lang.Object generate(Object[] references) { > /* 002 */ return new SpecificMutableProjection(references); > /* 003 */ } > /* 004 */ > /* 005 */ class SpecificMutableProjection extends > org.apache.spark.sql.catalyst.expressions.codegen.BaseMutableProjection { > /* 006 */ > /* 007 */ private Object[] references; > /* 008 */ private MutableRow mutableRow; > /* 009 */ private Object[] values; > /* 010 */ private java.lang.String errMsg; > /* 011 */ private java.lang.String errMsg1; > /* 012 */ private boolean MapObjects_loopIsNull1; > /* 013 */ private io.mistnet.analytics.lib.ConnLog MapObjects_loopValue0; > /* 014 */ private java.lang.String errMsg2; > /* 015 */ private Object[] values1; > /* 016 */ private boolean MapObjects_loopIsNull3; > /* 017 */ private java.lang.String MapObjects_loopValue2; > /* 018 */ private boolean isNull_0; > /* 019 */ private boolean value_0; > /* 020 */ private boolean isNull_1; > /* 021 */ private InternalRow value_1; > /* 022 */ > /* 023 */ private void apply_4(InternalRow i) { > /* 024 */ > /* 025 */ boolean isNull52 = MapObjects_loopIsNull1; > /* 026 */ final double value52 = isNull52 ? -1.0 : > MapObjects_loopValue0.ts(); > /* 027 */ if (isNull52) { > /* 028 */ values1[8] = null; > /* 029 */ } else { > /* 030 */ values1[8] = value52; > /* 031 */ } > /* 032 */ boolean isNull54 = MapObjects_loopIsNull1; > /* 033 */ final java.lang.String value54 = isNull54 ? null : > (java.lang.String) MapObjects_loopValue0.uid(); > /* 034 */ isNull54 = value54 == null; > /* 035 */ boolean isNull53 = isNull54; > /* 036 */ final UTF8String value53 = isNull53 ? null : > org.apache.spark.unsafe.types.UTF8String.fromString(value54); > /* 037 */ isNull53 = value53 == null; > /* 038 */ if (isNull53) { > /* 039 */ values1[9] = null; > /* 040 */ } else { > /* 041 */ values1[9] = value53; > /* 042 */ } > /* 043 */ boolean isNull56 = MapObjects_loopIsNull1; > /* 044 */ final java.lang.String value56 = isNull56 ? null : > (java.lang.String) MapObjects_loopValue0.src(); > /* 045 */ isNull56 = value56 == null; > /* 046 */ boolean isNull55 = isNull56; > /* 047 */ final UTF8String value55 = isNull55 ? null : > org.apache.spark.unsafe.types.UTF8String.fromString(value56); > /* 048 */ isNull55 = value55 == null; > /* 049 */ if (isNull55) { > /* 050 */ values1[10] = null; > /* 051 */ } else { > /* 052 */ values1[10] = value55; > /* 053 */ } > /* 054 */ } > /* 055 */ > /* 056 */ > /* 057 */ private void apply_7(InternalRow i) { > /* 058 */ > /* 059 */ boolean isNull69 = MapObjects_loopIsNull1; > /* 060 */ final scala.Option value69 = isNull69 ? null : (scala.Option) > MapObjects_loopValue0.orig_bytes(); > /* 061 */ isNull69 = value69 == null; > /* 062 */ > /* 063 */ final boolean isNull68 = isNull69 || value69.isEmpty(); > /* 064 */ long value68 = isNull68 ? > /* 065 */ -1L : (Long) value69.get(); > /* 066 */ if (isNull68) { > /* 067 */ values1[17] = null; > /* 068 */ } else { > /* 069 */ values1[17] = value68; > /* 070 */ } > /* 071 */ boolean isNull71 = MapObjects_loopIsNull1; > /* 072 */ final scala.Option value71 = isNull71 ? null : (scala.Option) > MapObjects_loopValue0.resp_bytes(); > /* 073 */ isNull71 = value71 == null; > /* 074 */ > /* 075 */ final boolean isNull70 = isNull71 || value71.isEmpty(); > /* 076 */ long value70 = isNull70 ? > /* 077 */ -1L : (Long) value71.get(); > /* 078 */ if (isNull70) { > /* 079 */ values1[18] = null; > /* 080 */ } else { > /* 081 */ values1[18] = value70; > /* 082 */ } > /* 083 */ boolean isNull74 = MapObjects_lo
[jira] [Commented] (SPARK-18125) Spark generated code causes CompileException when groupByKey, reduceGroups and map(_._2) are used
[ https://issues.apache.org/jira/browse/SPARK-18125?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel&focusedCommentId=15615034#comment-15615034 ] Kazuaki Ishizaki commented on SPARK-18125: -- I confirmed this code can reproduce on 2.0.1. This problem occurs due to the similar reason in SPARK-18147 To call {{ctx.splitExpression}} in {{createStruct.doGenCode}} make *a variable* inaccesssible by splitting the original one function into multiple functions. SPARK-14793 seems to have introduced {{ctx.splitExpression}} here to fix other issues on April. Since Ray says this code works in 2.0.0, other changes may introduce this issue. > Spark generated code causes CompileException when groupByKey, reduceGroups > and map(_._2) are used > - > > Key: SPARK-18125 > URL: https://issues.apache.org/jira/browse/SPARK-18125 > Project: Spark > Issue Type: Bug >Affects Versions: 2.0.1 >Reporter: Ray Qiu >Priority: Critical > > Code logic looks like this: > {noformat} > .groupByKey > .reduceGroups > .map(_._2) > {noformat} > Works fine with 2.0.0. > 2.0.1 error Message: > {noformat} > Caused by: java.util.concurrent.ExecutionException: java.lang.Exception: > failed to compile: org.codehaus.commons.compiler.CompileException: File > 'generated.java', Line 206, Column 123: Unknown variable or type "value4" > /* 001 */ public java.lang.Object generate(Object[] references) { > /* 002 */ return new SpecificMutableProjection(references); > /* 003 */ } > /* 004 */ > /* 005 */ class SpecificMutableProjection extends > org.apache.spark.sql.catalyst.expressions.codegen.BaseMutableProjection { > /* 006 */ > /* 007 */ private Object[] references; > /* 008 */ private MutableRow mutableRow; > /* 009 */ private Object[] values; > /* 010 */ private java.lang.String errMsg; > /* 011 */ private java.lang.String errMsg1; > /* 012 */ private boolean MapObjects_loopIsNull1; > /* 013 */ private io.mistnet.analytics.lib.ConnLog MapObjects_loopValue0; > /* 014 */ private java.lang.String errMsg2; > /* 015 */ private Object[] values1; > /* 016 */ private boolean MapObjects_loopIsNull3; > /* 017 */ private java.lang.String MapObjects_loopValue2; > /* 018 */ private boolean isNull_0; > /* 019 */ private boolean value_0; > /* 020 */ private boolean isNull_1; > /* 021 */ private InternalRow value_1; > /* 022 */ > /* 023 */ private void apply_4(InternalRow i) { > /* 024 */ > /* 025 */ boolean isNull52 = MapObjects_loopIsNull1; > /* 026 */ final double value52 = isNull52 ? -1.0 : > MapObjects_loopValue0.ts(); > /* 027 */ if (isNull52) { > /* 028 */ values1[8] = null; > /* 029 */ } else { > /* 030 */ values1[8] = value52; > /* 031 */ } > /* 032 */ boolean isNull54 = MapObjects_loopIsNull1; > /* 033 */ final java.lang.String value54 = isNull54 ? null : > (java.lang.String) MapObjects_loopValue0.uid(); > /* 034 */ isNull54 = value54 == null; > /* 035 */ boolean isNull53 = isNull54; > /* 036 */ final UTF8String value53 = isNull53 ? null : > org.apache.spark.unsafe.types.UTF8String.fromString(value54); > /* 037 */ isNull53 = value53 == null; > /* 038 */ if (isNull53) { > /* 039 */ values1[9] = null; > /* 040 */ } else { > /* 041 */ values1[9] = value53; > /* 042 */ } > /* 043 */ boolean isNull56 = MapObjects_loopIsNull1; > /* 044 */ final java.lang.String value56 = isNull56 ? null : > (java.lang.String) MapObjects_loopValue0.src(); > /* 045 */ isNull56 = value56 == null; > /* 046 */ boolean isNull55 = isNull56; > /* 047 */ final UTF8String value55 = isNull55 ? null : > org.apache.spark.unsafe.types.UTF8String.fromString(value56); > /* 048 */ isNull55 = value55 == null; > /* 049 */ if (isNull55) { > /* 050 */ values1[10] = null; > /* 051 */ } else { > /* 052 */ values1[10] = value55; > /* 053 */ } > /* 054 */ } > /* 055 */ > /* 056 */ > /* 057 */ private void apply_7(InternalRow i) { > /* 058 */ > /* 059 */ boolean isNull69 = MapObjects_loopIsNull1; > /* 060 */ final scala.Option value69 = isNull69 ? null : (scala.Option) > MapObjects_loopValue0.orig_bytes(); > /* 061 */ isNull69 = value69 == null; > /* 062 */ > /* 063 */ final boolean isNull68 = isNull69 || value69.isEmpty(); > /* 064 */ long value68 = isNull68 ? > /* 065 */ -1L : (Long) value69.get(); > /* 066 */ if (isNull68) { > /* 067 */ values1[17] = null; > /* 068 */ } else { > /* 069 */ values1[17] = value68; > /* 070 */ } > /* 071 */ boolean isNull71 = MapObjects_loopIsNull1; > /* 072 */ final scala.Option value71 = isNull71 ? null : (scala.Option) > MapObjects_loopValue0.resp_bytes(); > /* 0
[jira] [Commented] (SPARK-18125) Spark generated code causes CompileException when groupByKey, reduceGroups and map(_._2) are used
[ https://issues.apache.org/jira/browse/SPARK-18125?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel&focusedCommentId=15621681#comment-15621681 ] Apache Spark commented on SPARK-18125: -- User 'viirya' has created a pull request for this issue: https://github.com/apache/spark/pull/15693 > Spark generated code causes CompileException when groupByKey, reduceGroups > and map(_._2) are used > - > > Key: SPARK-18125 > URL: https://issues.apache.org/jira/browse/SPARK-18125 > Project: Spark > Issue Type: Bug >Affects Versions: 2.0.1 >Reporter: Ray Qiu >Priority: Critical > > Code logic looks like this: > {noformat} > .groupByKey > .reduceGroups > .map(_._2) > {noformat} > Works fine with 2.0.0. > 2.0.1 error Message: > {noformat} > Caused by: java.util.concurrent.ExecutionException: java.lang.Exception: > failed to compile: org.codehaus.commons.compiler.CompileException: File > 'generated.java', Line 206, Column 123: Unknown variable or type "value4" > /* 001 */ public java.lang.Object generate(Object[] references) { > /* 002 */ return new SpecificMutableProjection(references); > /* 003 */ } > /* 004 */ > /* 005 */ class SpecificMutableProjection extends > org.apache.spark.sql.catalyst.expressions.codegen.BaseMutableProjection { > /* 006 */ > /* 007 */ private Object[] references; > /* 008 */ private MutableRow mutableRow; > /* 009 */ private Object[] values; > /* 010 */ private java.lang.String errMsg; > /* 011 */ private java.lang.String errMsg1; > /* 012 */ private boolean MapObjects_loopIsNull1; > /* 013 */ private io.mistnet.analytics.lib.ConnLog MapObjects_loopValue0; > /* 014 */ private java.lang.String errMsg2; > /* 015 */ private Object[] values1; > /* 016 */ private boolean MapObjects_loopIsNull3; > /* 017 */ private java.lang.String MapObjects_loopValue2; > /* 018 */ private boolean isNull_0; > /* 019 */ private boolean value_0; > /* 020 */ private boolean isNull_1; > /* 021 */ private InternalRow value_1; > /* 022 */ > /* 023 */ private void apply_4(InternalRow i) { > /* 024 */ > /* 025 */ boolean isNull52 = MapObjects_loopIsNull1; > /* 026 */ final double value52 = isNull52 ? -1.0 : > MapObjects_loopValue0.ts(); > /* 027 */ if (isNull52) { > /* 028 */ values1[8] = null; > /* 029 */ } else { > /* 030 */ values1[8] = value52; > /* 031 */ } > /* 032 */ boolean isNull54 = MapObjects_loopIsNull1; > /* 033 */ final java.lang.String value54 = isNull54 ? null : > (java.lang.String) MapObjects_loopValue0.uid(); > /* 034 */ isNull54 = value54 == null; > /* 035 */ boolean isNull53 = isNull54; > /* 036 */ final UTF8String value53 = isNull53 ? null : > org.apache.spark.unsafe.types.UTF8String.fromString(value54); > /* 037 */ isNull53 = value53 == null; > /* 038 */ if (isNull53) { > /* 039 */ values1[9] = null; > /* 040 */ } else { > /* 041 */ values1[9] = value53; > /* 042 */ } > /* 043 */ boolean isNull56 = MapObjects_loopIsNull1; > /* 044 */ final java.lang.String value56 = isNull56 ? null : > (java.lang.String) MapObjects_loopValue0.src(); > /* 045 */ isNull56 = value56 == null; > /* 046 */ boolean isNull55 = isNull56; > /* 047 */ final UTF8String value55 = isNull55 ? null : > org.apache.spark.unsafe.types.UTF8String.fromString(value56); > /* 048 */ isNull55 = value55 == null; > /* 049 */ if (isNull55) { > /* 050 */ values1[10] = null; > /* 051 */ } else { > /* 052 */ values1[10] = value55; > /* 053 */ } > /* 054 */ } > /* 055 */ > /* 056 */ > /* 057 */ private void apply_7(InternalRow i) { > /* 058 */ > /* 059 */ boolean isNull69 = MapObjects_loopIsNull1; > /* 060 */ final scala.Option value69 = isNull69 ? null : (scala.Option) > MapObjects_loopValue0.orig_bytes(); > /* 061 */ isNull69 = value69 == null; > /* 062 */ > /* 063 */ final boolean isNull68 = isNull69 || value69.isEmpty(); > /* 064 */ long value68 = isNull68 ? > /* 065 */ -1L : (Long) value69.get(); > /* 066 */ if (isNull68) { > /* 067 */ values1[17] = null; > /* 068 */ } else { > /* 069 */ values1[17] = value68; > /* 070 */ } > /* 071 */ boolean isNull71 = MapObjects_loopIsNull1; > /* 072 */ final scala.Option value71 = isNull71 ? null : (scala.Option) > MapObjects_loopValue0.resp_bytes(); > /* 073 */ isNull71 = value71 == null; > /* 074 */ > /* 075 */ final boolean isNull70 = isNull71 || value71.isEmpty(); > /* 076 */ long value70 = isNull70 ? > /* 077 */ -1L : (Long) value71.get(); > /* 078 */ if (isNull70) { > /* 079 */ values1[18] = null; > /* 080 */ } else { > /* 081 */ values1[18] = value70; > /* 082 *
[jira] [Commented] (SPARK-18125) Spark generated code causes CompileException when groupByKey, reduceGroups and map(_._2) are used
[ https://issues.apache.org/jira/browse/SPARK-18125?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel&focusedCommentId=15644458#comment-15644458 ] Apache Spark commented on SPARK-18125: -- User 'viirya' has created a pull request for this issue: https://github.com/apache/spark/pull/15796 > Spark generated code causes CompileException when groupByKey, reduceGroups > and map(_._2) are used > - > > Key: SPARK-18125 > URL: https://issues.apache.org/jira/browse/SPARK-18125 > Project: Spark > Issue Type: Bug >Affects Versions: 2.0.1 >Reporter: Ray Qiu >Assignee: Liang-Chi Hsieh >Priority: Critical > Fix For: 2.1.0 > > > Code logic looks like this: > {noformat} > .groupByKey > .reduceGroups > .map(_._2) > {noformat} > Works fine with 2.0.0. > 2.0.1 error Message: > {noformat} > Caused by: java.util.concurrent.ExecutionException: java.lang.Exception: > failed to compile: org.codehaus.commons.compiler.CompileException: File > 'generated.java', Line 206, Column 123: Unknown variable or type "value4" > /* 001 */ public java.lang.Object generate(Object[] references) { > /* 002 */ return new SpecificMutableProjection(references); > /* 003 */ } > /* 004 */ > /* 005 */ class SpecificMutableProjection extends > org.apache.spark.sql.catalyst.expressions.codegen.BaseMutableProjection { > /* 006 */ > /* 007 */ private Object[] references; > /* 008 */ private MutableRow mutableRow; > /* 009 */ private Object[] values; > /* 010 */ private java.lang.String errMsg; > /* 011 */ private java.lang.String errMsg1; > /* 012 */ private boolean MapObjects_loopIsNull1; > /* 013 */ private io.mistnet.analytics.lib.ConnLog MapObjects_loopValue0; > /* 014 */ private java.lang.String errMsg2; > /* 015 */ private Object[] values1; > /* 016 */ private boolean MapObjects_loopIsNull3; > /* 017 */ private java.lang.String MapObjects_loopValue2; > /* 018 */ private boolean isNull_0; > /* 019 */ private boolean value_0; > /* 020 */ private boolean isNull_1; > /* 021 */ private InternalRow value_1; > /* 022 */ > /* 023 */ private void apply_4(InternalRow i) { > /* 024 */ > /* 025 */ boolean isNull52 = MapObjects_loopIsNull1; > /* 026 */ final double value52 = isNull52 ? -1.0 : > MapObjects_loopValue0.ts(); > /* 027 */ if (isNull52) { > /* 028 */ values1[8] = null; > /* 029 */ } else { > /* 030 */ values1[8] = value52; > /* 031 */ } > /* 032 */ boolean isNull54 = MapObjects_loopIsNull1; > /* 033 */ final java.lang.String value54 = isNull54 ? null : > (java.lang.String) MapObjects_loopValue0.uid(); > /* 034 */ isNull54 = value54 == null; > /* 035 */ boolean isNull53 = isNull54; > /* 036 */ final UTF8String value53 = isNull53 ? null : > org.apache.spark.unsafe.types.UTF8String.fromString(value54); > /* 037 */ isNull53 = value53 == null; > /* 038 */ if (isNull53) { > /* 039 */ values1[9] = null; > /* 040 */ } else { > /* 041 */ values1[9] = value53; > /* 042 */ } > /* 043 */ boolean isNull56 = MapObjects_loopIsNull1; > /* 044 */ final java.lang.String value56 = isNull56 ? null : > (java.lang.String) MapObjects_loopValue0.src(); > /* 045 */ isNull56 = value56 == null; > /* 046 */ boolean isNull55 = isNull56; > /* 047 */ final UTF8String value55 = isNull55 ? null : > org.apache.spark.unsafe.types.UTF8String.fromString(value56); > /* 048 */ isNull55 = value55 == null; > /* 049 */ if (isNull55) { > /* 050 */ values1[10] = null; > /* 051 */ } else { > /* 052 */ values1[10] = value55; > /* 053 */ } > /* 054 */ } > /* 055 */ > /* 056 */ > /* 057 */ private void apply_7(InternalRow i) { > /* 058 */ > /* 059 */ boolean isNull69 = MapObjects_loopIsNull1; > /* 060 */ final scala.Option value69 = isNull69 ? null : (scala.Option) > MapObjects_loopValue0.orig_bytes(); > /* 061 */ isNull69 = value69 == null; > /* 062 */ > /* 063 */ final boolean isNull68 = isNull69 || value69.isEmpty(); > /* 064 */ long value68 = isNull68 ? > /* 065 */ -1L : (Long) value69.get(); > /* 066 */ if (isNull68) { > /* 067 */ values1[17] = null; > /* 068 */ } else { > /* 069 */ values1[17] = value68; > /* 070 */ } > /* 071 */ boolean isNull71 = MapObjects_loopIsNull1; > /* 072 */ final scala.Option value71 = isNull71 ? null : (scala.Option) > MapObjects_loopValue0.resp_bytes(); > /* 073 */ isNull71 = value71 == null; > /* 074 */ > /* 075 */ final boolean isNull70 = isNull71 || value71.isEmpty(); > /* 076 */ long value70 = isNull70 ? > /* 077 */ -1L : (Long) value71.get(); > /* 078 */ if (isNull70) { > /* 079 */ values1[18] = null; > /* 0