[
https://issues.apache.org/jira/browse/SPARK-18147?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel&focusedCommentId=16320159#comment-16320159
]
Alexander Chermenin commented on SPARK-18147:
---------------------------------------------
Is it resolved for the 2.2.1 version? I have a similar problem with PySpark and
Structured Streaming:
{noformat}
18/01/10 15:12:37 ERROR CodeGenerator: failed to compile:
org.codehaus.commons.compiler.CompileException: File 'generated.java', Line
174, Column 113: Expression "isNull3" is not an rvalue
/* 001 */ public java.lang.Object generate(Object[] references) {
/* 002 */ return new SpecificUnsafeProjection(references);
/* 003 */ }
/* 004 */
/* 005 */ class SpecificUnsafeProjection extends
org.apache.spark.sql.catalyst.expressions.UnsafeProjection {
/* 006 */
/* 007 */ private Object[] references;
/* 008 */ private UTF8String lastRegex;
/* 009 */ private java.util.regex.Pattern pattern;
/* 010 */ private String lastReplacement;
/* 011 */ private UTF8String lastReplacementInUTF8;
/* 012 */ private java.lang.StringBuffer result;
/* 013 */ private UTF8String lastRegex1;
/* 014 */ private java.util.regex.Pattern pattern1;
/* 015 */ private String lastReplacement1;
/* 016 */ private UTF8String lastReplacementInUTF81;
/* 017 */ private java.lang.StringBuffer result1;
/* 018 */ private UnsafeRow result2;
/* 019 */ private
org.apache.spark.sql.catalyst.expressions.codegen.BufferHolder holder;
/* 020 */ private
org.apache.spark.sql.catalyst.expressions.codegen.UnsafeRowWriter rowWriter;
/* 021 */
/* 022 */ public SpecificUnsafeProjection(Object[] references) {
/* 023 */ this.references = references;
/* 024 */ lastRegex = null;
/* 025 */ pattern = null;
/* 026 */ lastReplacement = null;
/* 027 */ lastReplacementInUTF8 = null;
/* 028 */ result = new java.lang.StringBuffer();
/* 029 */ lastRegex1 = null;
/* 030 */ pattern1 = null;
/* 031 */ lastReplacement1 = null;
/* 032 */ lastReplacementInUTF81 = null;
/* 033 */ result1 = new java.lang.StringBuffer();
/* 034 */ result2 = new UnsafeRow(3);
/* 035 */ this.holder = new
org.apache.spark.sql.catalyst.expressions.codegen.BufferHolder(result2, 64);
/* 036 */ this.rowWriter = new
org.apache.spark.sql.catalyst.expressions.codegen.UnsafeRowWriter(holder, 3);
/* 037 */
/* 038 */ }
/* 039 */
/* 040 */ public void initialize(int partitionIndex) {
/* 041 */
/* 042 */ }
/* 043 */
/* 044 */
/* 045 */ private void apply_1(InternalRow i) {
/* 046 */
/* 047 */ UTF8String[] args1 = new UTF8String[3];
/* 048 */
/* 049 */
/* 050 */ if (!false) {
/* 051 */ args1[0] = ((UTF8String) references[6]);
/* 052 */ }
/* 053 */
/* 054 */
/* 055 */ if (!false) {
/* 056 */ args1[1] = ((UTF8String) references[7]);
/* 057 */ }
/* 058 */
/* 059 */
/* 060 */ boolean isNull13 = false;
/* 061 */
/* 062 */ UTF8String value14 = i.getUTF8String(2);
/* 063 */
/* 064 */
/* 065 */ UTF8String value13 = null;
/* 066 */
/* 067 */ if (!((UTF8String) references[8]).equals(lastRegex1)) {
/* 068 */ // regex value changed
/* 069 */ lastRegex1 = ((UTF8String) references[8]).clone();
/* 070 */ pattern1 =
java.util.regex.Pattern.compile(lastRegex1.toString());
/* 071 */ }
/* 072 */ if (!((UTF8String) references[9]).equals(lastReplacementInUTF81))
{
/* 073 */ // replacement string changed
/* 074 */ lastReplacementInUTF81 = ((UTF8String) references[9]).clone();
/* 075 */ lastReplacement1 = lastReplacementInUTF81.toString();
/* 076 */ }
/* 077 */ result1.delete(0, result1.length());
/* 078 */ java.util.regex.Matcher matcher1 =
pattern1.matcher(value14.toString());
/* 079 */
/* 080 */ while (matcher1.find()) {
/* 081 */ matcher1.appendReplacement(result1, lastReplacement1);
/* 082 */ }
/* 083 */ matcher1.appendTail(result1);
/* 084 */ value13 = UTF8String.fromString(result1.toString());
/* 085 */ if (!false) {
/* 086 */ args1[2] = value13;
/* 087 */ }
/* 088 */
/* 089 */ UTF8String value10 = UTF8String.concat(args1);
/* 090 */ boolean isNull10 = value10 == null;
/* 091 */ }
/* 092 */
/* 093 */
/* 094 */ private void apply1_1(InternalRow i) {
/* 095 */
/* 096 */
/* 097 */ final Object value20 = null;
/* 098 */ if (true) {
/* 099 */ rowWriter.setNullAt(2);
/* 100 */ } else {
/* 101 */
/* 102 */ }
/* 103 */
/* 104 */ }
/* 105 */
/* 106 */
/* 107 */ private void apply_0(InternalRow i) {
/* 108 */
/* 109 */ UTF8String[] args = new UTF8String[3];
/* 110 */
/* 111 */
/* 112 */ if (!false) {
/* 113 */ args[0] = ((UTF8String) references[2]);
/* 114 */ }
/* 115 */
/* 116 */
/* 117 */ if (!false) {
/* 118 */ args[1] = ((UTF8String) references[3]);
/* 119 */ }
/* 120 */
/* 121 */
/* 122 */ boolean isNull6 = true;
/* 123 */ UTF8String value6 = null;
/* 124 */
/* 125 */ boolean isNull7 = i.isNullAt(1);
/* 126 */ UTF8String value7 = isNull7 ? null : (i.getUTF8String(1));
/* 127 */ if (!isNull7) {
/* 128 */
/* 129 */
/* 130 */
/* 131 */ isNull6 = false; // resultCode could change nullability.
/* 132 */
/* 133 */ if (!((UTF8String) references[4]).equals(lastRegex)) {
/* 134 */ // regex value changed
/* 135 */ lastRegex = ((UTF8String) references[4]).clone();
/* 136 */ pattern =
java.util.regex.Pattern.compile(lastRegex.toString());
/* 137 */ }
/* 138 */ if (!((UTF8String)
references[5]).equals(lastReplacementInUTF8)) {
/* 139 */ // replacement string changed
/* 140 */ lastReplacementInUTF8 = ((UTF8String) references[5]).clone();
/* 141 */ lastReplacement = lastReplacementInUTF8.toString();
/* 142 */ }
/* 143 */ result.delete(0, result.length());
/* 144 */ java.util.regex.Matcher matcher =
pattern.matcher(value7.toString());
/* 145 */
/* 146 */ while (matcher.find()) {
/* 147 */ matcher.appendReplacement(result, lastReplacement);
/* 148 */ }
/* 149 */ matcher.appendTail(result);
/* 150 */ value6 = UTF8String.fromString(result.toString());
/* 151 */ isNull6 = false;
/* 152 */
/* 153 */
/* 154 */ }
/* 155 */ if (!isNull6) {
/* 156 */ args[2] = value6;
/* 157 */ }
/* 158 */
/* 159 */ UTF8String value3 = UTF8String.concat(args);
/* 160 */ boolean isNull3 = value3 == null;
/* 161 */ }
/* 162 */
/* 163 */
/* 164 */ private void apply1_0(InternalRow i) {
/* 165 */
/* 166 */
/* 167 */ apply_0(i);
/* 168 */ apply_1(i);
/* 169 */ apply_2(i);
/* 170 */ int varargNum = 4;
/* 171 */ int idxInVararg = 0;
/* 172 */
/* 173 */ UTF8String[] array = new UTF8String[varargNum];
/* 174 */ array[idxInVararg ++] = false ? (UTF8String) null : ((UTF8String)
references[1]);array[idxInVararg ++] = isNull3 ? (UTF8String) null :
value3;array[idxInVararg ++] = isNull10 ? (UTF8String) null :
value10;array[idxInVararg ++] = false ? (UTF8String) null : ((UTF8String)
references[10]);
/* 175 */ UTF8String value = UTF8String.concatWs(((UTF8String)
references[0]), array);
/* 176 */ boolean isNull = value == null;
/* 177 */ if (isNull) {
/* 178 */ rowWriter.setNullAt(0);
/* 179 */ } else {
/* 180 */ rowWriter.write(0, value);
/* 181 */ }
/* 182 */
/* 183 */
/* 184 */ Object obj = ((Expression) references[11]).eval(i);
/* 185 */ boolean isNull19 = obj == null;
/* 186 */ UTF8String value19 = null;
/* 187 */ if (!isNull19) {
/* 188 */ value19 = (UTF8String) obj;
/* 189 */ }
/* 190 */ if (isNull19) {
/* 191 */ rowWriter.setNullAt(1);
/* 192 */ } else {
/* 193 */ rowWriter.write(1, value19);
/* 194 */ }
/* 195 */
/* 196 */ }
/* 197 */
/* 198 */
/* 199 */ private void apply_2(InternalRow i) {
/* 200 */
/* 201 */ final ArrayData value17 = null;
/* 202 */ }
/* 203 */
/* 204 */
/* 205 */ // Scala.Function1 need this
/* 206 */ public java.lang.Object apply(java.lang.Object row) {
/* 207 */ return apply((InternalRow) row);
/* 208 */ }
/* 209 */
/* 210 */ public UnsafeRow apply(InternalRow i) {
/* 211 */ holder.reset();
/* 212 */
/* 213 */ rowWriter.zeroOutNullBytes();
/* 214 */ apply1_0(i);
/* 215 */ apply1_1(i);
/* 216 */ result2.setTotalSize(holder.totalSize());
/* 217 */ return result2;
/* 218 */ }
/* 219 */ }
org.codehaus.commons.compiler.CompileException: File 'generated.java', Line
174, Column 113: Expression "isNull3" is not an rvalue
at
org.codehaus.janino.UnitCompiler.compileError(UnitCompiler.java:11004)
at
org.codehaus.janino.UnitCompiler.toRvalueOrCompileException(UnitCompiler.java:6639)
at
org.codehaus.janino.UnitCompiler.getConstantValue2(UnitCompiler.java:5001)
at org.codehaus.janino.UnitCompiler.access$10500(UnitCompiler.java:206)
at
org.codehaus.janino.UnitCompiler$13.visitAmbiguousName(UnitCompiler.java:4984)
at org.codehaus.janino.Java$AmbiguousName.accept(Java.java:3633)
at org.codehaus.janino.Java$Lvalue.accept(Java.java:3563)
at
org.codehaus.janino.UnitCompiler.getConstantValue(UnitCompiler.java:4956)
at
org.codehaus.janino.UnitCompiler.getConstantValue2(UnitCompiler.java:5027)
at org.codehaus.janino.UnitCompiler.access$9700(UnitCompiler.java:206)
at
org.codehaus.janino.UnitCompiler$13.visitConditionalExpression(UnitCompiler.java:4964)
at org.codehaus.janino.Java$ConditionalExpression.accept(Java.java:3877)
at
org.codehaus.janino.UnitCompiler.getConstantValue(UnitCompiler.java:4956)
at
org.codehaus.janino.UnitCompiler.compileGetValue(UnitCompiler.java:4925)
at org.codehaus.janino.UnitCompiler.compile2(UnitCompiler.java:3189)
at org.codehaus.janino.UnitCompiler.access$5100(UnitCompiler.java:206)
at
org.codehaus.janino.UnitCompiler$9.visitAssignment(UnitCompiler.java:3143)
at
org.codehaus.janino.UnitCompiler$9.visitAssignment(UnitCompiler.java:3139)
at org.codehaus.janino.Java$Assignment.accept(Java.java:3847)
at org.codehaus.janino.UnitCompiler.compile(UnitCompiler.java:3139)
at org.codehaus.janino.UnitCompiler.compile2(UnitCompiler.java:2112)
at org.codehaus.janino.UnitCompiler.access$1700(UnitCompiler.java:206)
at
org.codehaus.janino.UnitCompiler$6.visitExpressionStatement(UnitCompiler.java:1377)
at
org.codehaus.janino.UnitCompiler$6.visitExpressionStatement(UnitCompiler.java:1370)
at org.codehaus.janino.Java$ExpressionStatement.accept(Java.java:2558)
at org.codehaus.janino.UnitCompiler.compile(UnitCompiler.java:1370)
at
org.codehaus.janino.UnitCompiler.compileStatements(UnitCompiler.java:1450)
at org.codehaus.janino.UnitCompiler.compile(UnitCompiler.java:2811)
at
org.codehaus.janino.UnitCompiler.compileDeclaredMethods(UnitCompiler.java:1262)
at
org.codehaus.janino.UnitCompiler.compileDeclaredMethods(UnitCompiler.java:1234)
at org.codehaus.janino.UnitCompiler.compile2(UnitCompiler.java:538)
at org.codehaus.janino.UnitCompiler.compile2(UnitCompiler.java:890)
at org.codehaus.janino.UnitCompiler.compile2(UnitCompiler.java:894)
at org.codehaus.janino.UnitCompiler.access$600(UnitCompiler.java:206)
at
org.codehaus.janino.UnitCompiler$2.visitMemberClassDeclaration(UnitCompiler.java:377)
at
org.codehaus.janino.UnitCompiler$2.visitMemberClassDeclaration(UnitCompiler.java:369)
at
org.codehaus.janino.Java$MemberClassDeclaration.accept(Java.java:1128)
at org.codehaus.janino.UnitCompiler.compile(UnitCompiler.java:369)
at
org.codehaus.janino.UnitCompiler.compileDeclaredMemberTypes(UnitCompiler.java:1209)
at org.codehaus.janino.UnitCompiler.compile2(UnitCompiler.java:564)
at org.codehaus.janino.UnitCompiler.compile2(UnitCompiler.java:420)
at org.codehaus.janino.UnitCompiler.access$400(UnitCompiler.java:206)
at
org.codehaus.janino.UnitCompiler$2.visitPackageMemberClassDeclaration(UnitCompiler.java:374)
at
org.codehaus.janino.UnitCompiler$2.visitPackageMemberClassDeclaration(UnitCompiler.java:369)
at
org.codehaus.janino.Java$AbstractPackageMemberClassDeclaration.accept(Java.java:1309)
at org.codehaus.janino.UnitCompiler.compile(UnitCompiler.java:369)
at org.codehaus.janino.UnitCompiler.compileUnit(UnitCompiler.java:345)
at
org.codehaus.janino.SimpleCompiler.compileToClassLoader(SimpleCompiler.java:396)
at
org.codehaus.janino.ClassBodyEvaluator.compileToClass(ClassBodyEvaluator.java:311)
at
org.codehaus.janino.ClassBodyEvaluator.cook(ClassBodyEvaluator.java:229)
at org.codehaus.janino.SimpleCompiler.cook(SimpleCompiler.java:196)
at org.codehaus.commons.compiler.Cookable.cook(Cookable.java:91)
at
org.apache.spark.sql.catalyst.expressions.codegen.CodeGenerator$.org$apache$spark$sql$catalyst$expressions$codegen$CodeGenerator$$doCompile(CodeGenerator.scala:1000)
at
org.apache.spark.sql.catalyst.expressions.codegen.CodeGenerator$$anon$1.load(CodeGenerator.scala:1067)
at
org.apache.spark.sql.catalyst.expressions.codegen.CodeGenerator$$anon$1.load(CodeGenerator.scala:1064)
at
org.spark_project.guava.cache.LocalCache$LoadingValueReference.loadFuture(LocalCache.java:3599)
at
org.spark_project.guava.cache.LocalCache$Segment.loadSync(LocalCache.java:2379)
at
org.spark_project.guava.cache.LocalCache$Segment.lockedGetOrLoad(LocalCache.java:2342)
at
org.spark_project.guava.cache.LocalCache$Segment.get(LocalCache.java:2257)
at org.spark_project.guava.cache.LocalCache.get(LocalCache.java:4000)
at
org.spark_project.guava.cache.LocalCache.getOrLoad(LocalCache.java:4004)
at
org.spark_project.guava.cache.LocalCache$LocalLoadingCache.get(LocalCache.java:4874)
at
org.apache.spark.sql.catalyst.expressions.codegen.CodeGenerator$.compile(CodeGenerator.scala:946)
at
org.apache.spark.sql.catalyst.expressions.codegen.GenerateUnsafeProjection$.create(GenerateUnsafeProjection.scala:412)
at
org.apache.spark.sql.catalyst.expressions.codegen.GenerateUnsafeProjection$.create(GenerateUnsafeProjection.scala:366)
at
org.apache.spark.sql.catalyst.expressions.codegen.GenerateUnsafeProjection$.create(GenerateUnsafeProjection.scala:32)
at
org.apache.spark.sql.catalyst.expressions.codegen.CodeGenerator.generate(CodeGenerator.scala:930)
at
org.apache.spark.sql.catalyst.expressions.UnsafeProjection$.create(Projection.scala:130)
at
org.apache.spark.sql.catalyst.expressions.UnsafeProjection$.create(Projection.scala:140)
at
org.apache.spark.sql.execution.aggregate.AggregationIterator.generateResultProjection(AggregationIterator.scala:219)
at
org.apache.spark.sql.execution.aggregate.TungstenAggregationIterator.generateResultProjection(TungstenAggregationIterator.scala:144)
at
org.apache.spark.sql.execution.aggregate.AggregationIterator.<init>(AggregationIterator.scala:266)
at
org.apache.spark.sql.execution.aggregate.TungstenAggregationIterator.<init>(TungstenAggregationIterator.scala:92)
at
org.apache.spark.sql.execution.aggregate.HashAggregateExec$$anonfun$doExecute$1$$anonfun$4.apply(HashAggregateExec.scala:106)
at
org.apache.spark.sql.execution.aggregate.HashAggregateExec$$anonfun$doExecute$1$$anonfun$4.apply(HashAggregateExec.scala:97)
at
org.apache.spark.rdd.RDD$$anonfun$mapPartitions$1$$anonfun$apply$23.apply(RDD.scala:797)
at
org.apache.spark.rdd.RDD$$anonfun$mapPartitions$1$$anonfun$apply$23.apply(RDD.scala:797)
at
org.apache.spark.rdd.MapPartitionsRDD.compute(MapPartitionsRDD.scala:38)
at org.apache.spark.rdd.RDD.computeOrReadCheckpoint(RDD.scala:323)
at org.apache.spark.rdd.RDD.iterator(RDD.scala:287)
at
org.apache.spark.rdd.MapPartitionsRDD.compute(MapPartitionsRDD.scala:38)
at org.apache.spark.rdd.RDD.computeOrReadCheckpoint(RDD.scala:323)
at org.apache.spark.rdd.RDD.iterator(RDD.scala:287)
at org.apache.spark.scheduler.ResultTask.runTask(ResultTask.scala:87)
at org.apache.spark.scheduler.Task.run(Task.scala:108)
at org.apache.spark.executor.Executor$TaskRunner.run(Executor.scala:338)
at
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
at
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
at java.lang.Thread.run(Thread.java:745)
18/01/10 15:12:37 ERROR Executor: Exception in task 0.0 in stage 1.0 (TID 1)
{noformat}
> Broken Spark SQL Codegen
> ------------------------
>
> Key: SPARK-18147
> URL: https://issues.apache.org/jira/browse/SPARK-18147
> Project: Spark
> Issue Type: Bug
> Components: SQL
> Affects Versions: 2.0.1
> Reporter: koert kuipers
> Assignee: Liang-Chi Hsieh
> Priority: Critical
> Fix For: 2.1.0
>
>
> this is me on purpose trying to break spark sql codegen to uncover potential
> issues, by creating arbitrately complex data structures using primitives,
> strings, basic collections (map, seq, option), tuples, and case classes.
> first example: nested case classes
> code:
> {noformat}
> class ComplexResultAgg[B: TypeTag, C: TypeTag](val zero: B, result: C)
> extends Aggregator[Row, B, C] {
> override def reduce(b: B, input: Row): B = b
> override def merge(b1: B, b2: B): B = b1
> override def finish(reduction: B): C = result
> override def bufferEncoder: Encoder[B] = ExpressionEncoder[B]()
> override def outputEncoder: Encoder[C] = ExpressionEncoder[C]()
> }
> case class Struct2(d: Double = 0.0, s1: Seq[Double] = Seq.empty, s2:
> Seq[Long] = Seq.empty)
> case class Struct3(a: Struct2 = Struct2(), b: Struct2 = Struct2())
> val df1 = Seq(("a", "aa"), ("a", "aa"), ("b", "b"), ("b", null)).toDF("x",
> "y").groupBy("x").agg(
> new ComplexResultAgg("boo", Struct3()).toColumn
> )
> df1.printSchema
> df1.show
> {noformat}
> the result is:
> {noformat}
> [info] Cause: java.util.concurrent.ExecutionException: java.lang.Exception:
> failed to compile: org.codehaus.commons.compiler.CompileException: File
> 'generated.java', Line 33, Column 12: Expression "isNull1" is not an rvalue
> [info] /* 001 */ public java.lang.Object generate(Object[] references) {
> [info] /* 002 */ return new SpecificMutableProjection(references);
> [info] /* 003 */ }
> [info] /* 004 */
> [info] /* 005 */ class SpecificMutableProjection extends
> org.apache.spark.sql.catalyst.expressions.codegen.BaseMutableProjection {
> [info] /* 006 */
> [info] /* 007 */ private Object[] references;
> [info] /* 008 */ private MutableRow mutableRow;
> [info] /* 009 */ private Object[] values;
> [info] /* 010 */ private java.lang.String errMsg;
> [info] /* 011 */ private Object[] values1;
> [info] /* 012 */ private java.lang.String errMsg1;
> [info] /* 013 */ private boolean[] argIsNulls;
> [info] /* 014 */ private scala.collection.Seq argValue;
> [info] /* 015 */ private java.lang.String errMsg2;
> [info] /* 016 */ private boolean[] argIsNulls1;
> [info] /* 017 */ private scala.collection.Seq argValue1;
> [info] /* 018 */ private java.lang.String errMsg3;
> [info] /* 019 */ private java.lang.String errMsg4;
> [info] /* 020 */ private Object[] values2;
> [info] /* 021 */ private java.lang.String errMsg5;
> [info] /* 022 */ private boolean[] argIsNulls2;
> [info] /* 023 */ private scala.collection.Seq argValue2;
> [info] /* 024 */ private java.lang.String errMsg6;
> [info] /* 025 */ private boolean[] argIsNulls3;
> [info] /* 026 */ private scala.collection.Seq argValue3;
> [info] /* 027 */ private java.lang.String errMsg7;
> [info] /* 028 */ private boolean isNull_0;
> [info] /* 029 */ private InternalRow value_0;
> [info] /* 030 */
> [info] /* 031 */ private void apply_1(InternalRow i) {
> [info] /* 032 */
> [info] /* 033 */ if (isNull1) {
> [info] /* 034 */ throw new RuntimeException(errMsg3);
> [info] /* 035 */ }
> [info] /* 036 */
> [info] /* 037 */ boolean isNull24 = false;
> [info] /* 038 */ final com.tresata.spark.sql.Struct2 value24 = isNull24 ?
> null : (com.tresata.spark.sql.Struct2) value1.a();
> [info] /* 039 */ isNull24 = value24 == null;
> [info] /* 040 */
> [info] /* 041 */ boolean isNull23 = isNull24;
> [info] /* 042 */ final scala.collection.Seq value23 = isNull23 ? null :
> (scala.collection.Seq) value24.s2();
> [info] /* 043 */ isNull23 = value23 == null;
> [info] /* 044 */ argIsNulls1[0] = isNull23;
> [info] /* 045 */ argValue1 = value23;
> [info] /* 046 */
> [info] /* 047 */
> [info] /* 048 */
> [info] /* 049 */ boolean isNull22 = false;
> [info] /* 050 */ for (int idx = 0; idx < 1; idx++) {
> [info] /* 051 */ if (argIsNulls1[idx]) { isNull22 = true; break; }
> [info] /* 052 */ }
> [info] /* 053 */
> [info] /* 054 */ final ArrayData value22 = isNull22 ? null : new
> org.apache.spark.sql.catalyst.util.GenericArrayData(argValue1);
> [info] /* 055 */ if (isNull22) {
> [info] /* 056 */ values1[2] = null;
> [info] /* 057 */ } else {
> [info] /* 058 */ values1[2] = value22;
> [info] /* 059 */ }
> [info] /* 060 */ }
> [info] /* 061 */
> [info] /* 062 */
> [info] /* 063 */ private void apply1_1(InternalRow i) {
> [info] /* 064 */
> [info] /* 065 */ if (isNull1) {
> [info] /* 066 */ throw new RuntimeException(errMsg7);
> [info] /* 067 */ }
> [info] /* 068 */
> [info] /* 069 */ boolean isNull41 = false;
> [info] /* 070 */ final com.tresata.spark.sql.Struct2 value41 = isNull41 ?
> null : (com.tresata.spark.sql.Struct2) value1.b();
> [info] /* 071 */ isNull41 = value41 == null;
> [info] /* 072 */
> [info] /* 073 */ boolean isNull40 = isNull41;
> [info] /* 074 */ final scala.collection.Seq value40 = isNull40 ? null :
> (scala.collection.Seq) value41.s2();
> [info] /* 075 */ isNull40 = value40 == null;
> [info] /* 076 */ argIsNulls3[0] = isNull40;
> [info] /* 077 */ argValue3 = value40;
> [info] /* 078 */
> [info] /* 079 */
> [info] /* 080 */
> [info] /* 081 */ boolean isNull39 = false;
> [info] /* 082 */ for (int idx = 0; idx < 1; idx++) {
> [info] /* 083 */ if (argIsNulls3[idx]) { isNull39 = true; break; }
> [info] /* 084 */ }
> [info] /* 085 */
> [info] /* 086 */ final ArrayData value39 = isNull39 ? null : new
> org.apache.spark.sql.catalyst.util.GenericArrayData(argValue3);
> [info] /* 087 */ if (isNull39) {
> [info] /* 088 */ values2[2] = null;
> [info] /* 089 */ } else {
> [info] /* 090 */ values2[2] = value39;
> [info] /* 091 */ }
> [info] /* 092 */ }
> [info] /* 093 */
> [info] /* 094 */
> [info] /* 095 */ private void apply_0(InternalRow i) {
> [info] /* 096 */
> [info] /* 097 */ if (isNull1) {
> [info] /* 098 */ throw new RuntimeException(errMsg1);
> [info] /* 099 */ }
> [info] /* 100 */
> [info] /* 101 */ boolean isNull16 = false;
> [info] /* 102 */ final com.tresata.spark.sql.Struct2 value16 = isNull16 ?
> null : (com.tresata.spark.sql.Struct2) value1.a();
> [info] /* 103 */ isNull16 = value16 == null;
> [info] /* 104 */
> [info] /* 105 */ boolean isNull15 = isNull16;
> [info] /* 106 */ final double value15 = isNull15 ? -1.0 : value16.d();
> [info] /* 107 */ if (isNull15) {
> [info] /* 108 */ values1[0] = null;
> [info] /* 109 */ } else {
> [info] /* 110 */ values1[0] = value15;
> [info] /* 111 */ }
> [info] /* 112 */ if (isNull1) {
> [info] /* 113 */ throw new RuntimeException(errMsg2);
> [info] /* 114 */ }
> [info] /* 115 */
> [info] /* 116 */ boolean isNull20 = false;
> [info] /* 117 */ final com.tresata.spark.sql.Struct2 value20 = isNull20 ?
> null : (com.tresata.spark.sql.Struct2) value1.a();
> [info] /* 118 */ isNull20 = value20 == null;
> [info] /* 119 */
> [info] /* 120 */ boolean isNull19 = isNull20;
> [info] /* 121 */ final scala.collection.Seq value19 = isNull19 ? null :
> (scala.collection.Seq) value20.s1();
> [info] /* 122 */ isNull19 = value19 == null;
> [info] /* 123 */ argIsNulls[0] = isNull19;
> [info] /* 124 */ argValue = value19;
> [info] /* 125 */
> [info] /* 126 */
> [info] /* 127 */
> [info] /* 128 */ boolean isNull18 = false;
> [info] /* 129 */ for (int idx = 0; idx < 1; idx++) {
> [info] /* 130 */ if (argIsNulls[idx]) { isNull18 = true; break; }
> [info] /* 131 */ }
> [info] /* 132 */
> [info] /* 133 */ final ArrayData value18 = isNull18 ? null : new
> org.apache.spark.sql.catalyst.util.GenericArrayData(argValue);
> [info] /* 134 */ if (isNull18) {
> [info] /* 135 */ values1[1] = null;
> [info] /* 136 */ } else {
> [info] /* 137 */ values1[1] = value18;
> [info] /* 138 */ }
> [info] /* 139 */ }
> [info] /* 140 */
> [info] /* 141 */
> [info] /* 142 */ private void apply1_0(InternalRow i) {
> [info] /* 143 */
> [info] /* 144 */ if (isNull1) {
> [info] /* 145 */ throw new RuntimeException(errMsg5);
> [info] /* 146 */ }
> [info] /* 147 */
> [info] /* 148 */ boolean isNull33 = false;
> [info] /* 149 */ final com.tresata.spark.sql.Struct2 value33 = isNull33 ?
> null : (com.tresata.spark.sql.Struct2) value1.b();
> [info] /* 150 */ isNull33 = value33 == null;
> [info] /* 151 */
> [info] /* 152 */ boolean isNull32 = isNull33;
> [info] /* 153 */ final double value32 = isNull32 ? -1.0 : value33.d();
> [info] /* 154 */ if (isNull32) {
> [info] /* 155 */ values2[0] = null;
> [info] /* 156 */ } else {
> [info] /* 157 */ values2[0] = value32;
> [info] /* 158 */ }
> [info] /* 159 */ if (isNull1) {
> [info] /* 160 */ throw new RuntimeException(errMsg6);
> [info] /* 161 */ }
> [info] /* 162 */
> [info] /* 163 */ boolean isNull37 = false;
> [info] /* 164 */ final com.tresata.spark.sql.Struct2 value37 = isNull37 ?
> null : (com.tresata.spark.sql.Struct2) value1.b();
> [info] /* 165 */ isNull37 = value37 == null;
> [info] /* 166 */
> [info] /* 167 */ boolean isNull36 = isNull37;
> [info] /* 168 */ final scala.collection.Seq value36 = isNull36 ? null :
> (scala.collection.Seq) value37.s1();
> [info] /* 169 */ isNull36 = value36 == null;
> [info] /* 170 */ argIsNulls2[0] = isNull36;
> [info] /* 171 */ argValue2 = value36;
> [info] /* 172 */
> [info] /* 173 */
> [info] /* 174 */
> [info] /* 175 */ boolean isNull35 = false;
> [info] /* 176 */ for (int idx = 0; idx < 1; idx++) {
> [info] /* 177 */ if (argIsNulls2[idx]) { isNull35 = true; break; }
> [info] /* 178 */ }
> [info] /* 179 */
> [info] /* 180 */ final ArrayData value35 = isNull35 ? null : new
> org.apache.spark.sql.catalyst.util.GenericArrayData(argValue2);
> [info] /* 181 */ if (isNull35) {
> [info] /* 182 */ values2[1] = null;
> [info] /* 183 */ } else {
> [info] /* 184 */ values2[1] = value35;
> [info] /* 185 */ }
> [info] /* 186 */ }
> [info] /* 187 */
> [info] /* 188 */
> [info] /* 189 */ public SpecificMutableProjection(Object[] references) {
> [info] /* 190 */ this.references = references;
> [info] /* 191 */ mutableRow = new
> org.apache.spark.sql.catalyst.expressions.GenericMutableRow(1);
> [info] /* 192 */ this.values = null;
> [info] /* 193 */ this.errMsg = (java.lang.String) references[1];
> [info] /* 194 */ this.values1 = null;
> [info] /* 195 */ this.errMsg1 = (java.lang.String) references[2];
> [info] /* 196 */ argIsNulls = new boolean[1];
> [info] /* 197 */
> [info] /* 198 */ this.errMsg2 = (java.lang.String) references[3];
> [info] /* 199 */ argIsNulls1 = new boolean[1];
> [info] /* 200 */
> [info] /* 201 */ this.errMsg3 = (java.lang.String) references[4];
> [info] /* 202 */ this.errMsg4 = (java.lang.String) references[5];
> [info] /* 203 */ this.values2 = null;
> [info] /* 204 */ this.errMsg5 = (java.lang.String) references[6];
> [info] /* 205 */ argIsNulls2 = new boolean[1];
> [info] /* 206 */
> [info] /* 207 */ this.errMsg6 = (java.lang.String) references[7];
> [info] /* 208 */ argIsNulls3 = new boolean[1];
> [info] /* 209 */
> [info] /* 210 */ this.errMsg7 = (java.lang.String) references[8];
> [info] /* 211 */ this.isNull_0 = true;
> [info] /* 212 */ this.value_0 = null;
> [info] /* 213 */ }
> [info] /* 214 */
> [info] /* 215 */ public
> org.apache.spark.sql.catalyst.expressions.codegen.BaseMutableProjection
> target(MutableRow row) {
> [info] /* 216 */ mutableRow = row;
> [info] /* 217 */ return this;
> [info] /* 218 */ }
> [info] /* 219 */
> [info] /* 220 */ /* Provide immutable access to the last projected row. */
> [info] /* 221 */ public InternalRow currentValue() {
> [info] /* 222 */ return (InternalRow) mutableRow;
> [info] /* 223 */ }
> [info] /* 224 */
> [info] /* 225 */ public java.lang.Object apply(java.lang.Object _i) {
> [info] /* 226 */ InternalRow i = (InternalRow) _i;
> [info] /* 227 */
> [info] /* 228 */
> [info] /* 229 */
> [info] /* 230 */ Object obj = ((Expression) references[0]).eval(null);
> [info] /* 231 */ org.apache.spark.sql.expressions.Aggregator value2 =
> (org.apache.spark.sql.expressions.Aggregator) obj;
> [info] /* 232 */
> [info] /* 233 */ boolean isNull4 = i.isNullAt(0);
> [info] /* 234 */ UTF8String value4 = isNull4 ? null :
> (i.getUTF8String(0));
> [info] /* 235 */
> [info] /* 236 */ boolean isNull3 = isNull4;
> [info] /* 237 */ final java.lang.String value3 = isNull3 ? null :
> (java.lang.String) value4.toString();
> [info] /* 238 */ isNull3 = value3 == null;
> [info] /* 239 */ boolean isNull1 = false || isNull3;
> [info] /* 240 */ final com.tresata.spark.sql.Struct3 value1 = isNull1 ?
> null : (com.tresata.spark.sql.Struct3) value2.finish(value3);
> [info] /* 241 */ isNull1 = value1 == null;
> [info] /* 242 */
> [info] /* 243 */ boolean isNull5 = false;
> [info] /* 244 */ InternalRow value5 = null;
> [info] /* 245 */ if (!false && isNull1) {
> [info] /* 246 */
> [info] /* 247 */ final InternalRow value7 = null;
> [info] /* 248 */ isNull5 = true;
> [info] /* 249 */ value5 = value7;
> [info] /* 250 */ } else {
> [info] /* 251 */
> [info] /* 252 */ boolean isNull8 = false;
> [info] /* 253 */ this.values = new Object[2];
> [info] /* 254 */ if (isNull1) {
> [info] /* 255 */ throw new RuntimeException(errMsg);
> [info] /* 256 */ }
> [info] /* 257 */
> [info] /* 258 */ boolean isNull11 = false;
> [info] /* 259 */ final com.tresata.spark.sql.Struct2 value11 = isNull11
> ? null : (com.tresata.spark.sql.Struct2) value1.a();
> [info] /* 260 */ isNull11 = value11 == null;
> [info] /* 261 */ boolean isNull9 = false;
> [info] /* 262 */ InternalRow value9 = null;
> [info] /* 263 */ if (!false && isNull11) {
> [info] /* 264 */
> [info] /* 265 */ final InternalRow value13 = null;
> [info] /* 266 */ isNull9 = true;
> [info] /* 267 */ value9 = value13;
> [info] /* 268 */ } else {
> [info] /* 269 */
> [info] /* 270 */ boolean isNull14 = false;
> [info] /* 271 */ values1 = new Object[3];apply_0(i);
> [info] /* 272 */ apply_1(i);
> [info] /* 273 */ final InternalRow value14 = new
> org.apache.spark.sql.catalyst.expressions.GenericInternalRow(values1);
> [info] /* 274 */ this.values1 = null;
> [info] /* 275 */ isNull9 = isNull14;
> [info] /* 276 */ value9 = value14;
> [info] /* 277 */ }
> [info] /* 278 */ if (isNull9) {
> [info] /* 279 */ values[0] = null;
> [info] /* 280 */ } else {
> [info] /* 281 */ values[0] = value9;
> [info] /* 282 */ }
> [info] /* 283 */ if (isNull1) {
> [info] /* 284 */ throw new RuntimeException(errMsg4);
> [info] /* 285 */ }
> [info] /* 286 */
> [info] /* 287 */ boolean isNull28 = false;
> [info] /* 288 */ final com.tresata.spark.sql.Struct2 value28 = isNull28
> ? null : (com.tresata.spark.sql.Struct2) value1.b();
> [info] /* 289 */ isNull28 = value28 == null;
> [info] /* 290 */ boolean isNull26 = false;
> [info] /* 291 */ InternalRow value26 = null;
> [info] /* 292 */ if (!false && isNull28) {
> [info] /* 293 */
> [info] /* 294 */ final InternalRow value30 = null;
> [info] /* 295 */ isNull26 = true;
> [info] /* 296 */ value26 = value30;
> [info] /* 297 */ } else {
> [info] /* 298 */
> [info] /* 299 */ boolean isNull31 = false;
> [info] /* 300 */ values2 = new Object[3];apply1_0(i);
> [info] /* 301 */ apply1_1(i);
> [info] /* 302 */ final InternalRow value31 = new
> org.apache.spark.sql.catalyst.expressions.GenericInternalRow(values2);
> [info] /* 303 */ this.values2 = null;
> [info] /* 304 */ isNull26 = isNull31;
> [info] /* 305 */ value26 = value31;
> [info] /* 306 */ }
> [info] /* 307 */ if (isNull26) {
> [info] /* 308 */ values[1] = null;
> [info] /* 309 */ } else {
> [info] /* 310 */ values[1] = value26;
> [info] /* 311 */ }
> [info] /* 312 */ final InternalRow value8 = new
> org.apache.spark.sql.catalyst.expressions.GenericInternalRow(values);
> [info] /* 313 */ this.values = null;
> [info] /* 314 */ isNull5 = isNull8;
> [info] /* 315 */ value5 = value8;
> [info] /* 316 */ }
> [info] /* 317 */ this.isNull_0 = isNull5;
> [info] /* 318 */ this.value_0 = value5;
> [info] /* 319 */
> [info] /* 320 */ // copy all the results into MutableRow
> [info] /* 321 */
> [info] /* 322 */ if (!this.isNull_0) {
> [info] /* 323 */ mutableRow.update(0, this.value_0);
> [info] /* 324 */ } else {
> [info] /* 325 */ mutableRow.setNullAt(0);
> [info] /* 326 */ }
> [info] /* 327 */
> [info] /* 328 */ return mutableRow;
> [info] /* 329 */ }
> [info] /* 330 */ }
> {noformat}
--
This message was sent by Atlassian JIRA
(v6.4.14#64029)
---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscr...@spark.apache.org
For additional commands, e-mail: issues-h...@spark.apache.org
