[ https://issues.apache.org/jira/browse/SPARK-21391?page=com.atlassian.jira.plugin.system.issuetabpanels:all-tabpanel ]
Hyukjin Kwon reopened SPARK-21391: ---------------------------------- > Cannot convert a Seq of Map whose value type is again a seq, into a dataset > ---------------------------------------------------------------------------- > > Key: SPARK-21391 > URL: https://issues.apache.org/jira/browse/SPARK-21391 > Project: Spark > Issue Type: Bug > Components: SQL > Affects Versions: 2.1.0 > Environment: Seen on mac OSX, scala 2.11, java 8 > Reporter: indraneel rao > > There is an error while trying to create a dataset from a sequence of Maps, > whose values have any kind of collections. Even when they are wrapped in a > case class. > Eg : The following piece of code throws an error: > > {code:java} > case class Values(values: Seq[Double]) > case class ItemProperties(properties:Map[String,Values]) > val l1 = List(ItemProperties( > Map( > "A1" -> Values(Seq(1.0,2.0)), > "B1" -> Values(Seq(44.0,55.0)) > ) > ), > ItemProperties( > Map( > "A2" -> Values(Seq(123.0,25.0)), > "B2" -> Values(Seq(445.0,35.0)) > ) > ) > ) > l1.toDS().show() > {code} > Heres the error: > 17/07/12 21:59:35 ERROR CodeGenerator: failed to compile: > org.codehaus.commons.compiler.CompileException: File 'generated.java', Line > 65, Column 46: Expression "ExternalMapToCatalyst_value_isNull0" is not an > rvalue > /* 001 */ public java.lang.Object generate(Object[] references) { > /* 002 */ return new SpecificUnsafeProjection(references); > /* 003 */ } > /* 004 */ > /* 005 */ class SpecificUnsafeProjection extends > org.apache.spark.sql.catalyst.expressions.UnsafeProjection { > /* 006 */ > /* 007 */ private Object[] references; > /* 008 */ private boolean resultIsNull; > /* 009 */ private java.lang.String argValue; > /* 010 */ private Object[] values; > /* 011 */ private boolean resultIsNull1; > /* 012 */ private scala.collection.Seq argValue1; > /* 013 */ private boolean isNull11; > /* 014 */ private boolean value11; > /* 015 */ private boolean isNull12; > /* 016 */ private InternalRow value12; > /* 017 */ private boolean isNull13; > /* 018 */ private InternalRow value13; > /* 019 */ private UnsafeRow result; > /* 020 */ private > org.apache.spark.sql.catalyst.expressions.codegen.BufferHolder holder; > /* 021 */ private > org.apache.spark.sql.catalyst.expressions.codegen.UnsafeRowWriter rowWriter; > /* 022 */ private > org.apache.spark.sql.catalyst.expressions.codegen.UnsafeArrayWriter > arrayWriter; > /* 023 */ private > org.apache.spark.sql.catalyst.expressions.codegen.UnsafeArrayWriter > arrayWriter1; > /* 024 */ private > org.apache.spark.sql.catalyst.expressions.codegen.UnsafeRowWriter rowWriter1; > /* 025 */ private > org.apache.spark.sql.catalyst.expressions.codegen.UnsafeArrayWriter > arrayWriter2; > /* 026 */ > /* 027 */ public SpecificUnsafeProjection(Object[] references) { > /* 028 */ this.references = references; > /* 029 */ > /* 030 */ > /* 031 */ this.values = null; > /* 032 */ > /* 033 */ > /* 034 */ isNull11 = false; > /* 035 */ value11 = false; > /* 036 */ isNull12 = false; > /* 037 */ value12 = null; > /* 038 */ isNull13 = false; > /* 039 */ value13 = null; > /* 040 */ result = new UnsafeRow(1); > /* 041 */ this.holder = new > org.apache.spark.sql.catalyst.expressions.codegen.BufferHolder(result, 32); > /* 042 */ this.rowWriter = new > org.apache.spark.sql.catalyst.expressions.codegen.UnsafeRowWriter(holder, 1); > /* 043 */ this.arrayWriter = new > org.apache.spark.sql.catalyst.expressions.codegen.UnsafeArrayWriter(); > /* 044 */ this.arrayWriter1 = new > org.apache.spark.sql.catalyst.expressions.codegen.UnsafeArrayWriter(); > /* 045 */ this.rowWriter1 = new > org.apache.spark.sql.catalyst.expressions.codegen.UnsafeRowWriter(holder, 1); > /* 046 */ this.arrayWriter2 = new > org.apache.spark.sql.catalyst.expressions.codegen.UnsafeArrayWriter(); > /* 047 */ > /* 048 */ } > /* 049 */ > /* 050 */ public void initialize(int partitionIndex) { > /* 051 */ > /* 052 */ } > /* 053 */ > /* 054 */ > /* 055 */ private void evalIfTrueExpr(InternalRow i) { > /* 056 */ final InternalRow value7 = null; > /* 057 */ isNull12 = true; > /* 058 */ value12 = value7; > /* 059 */ } > /* 060 */ > /* 061 */ > /* 062 */ private void evalIfCondExpr(InternalRow i) { > /* 063 */ > /* 064 */ isNull11 = false; > /* 065 */ value11 = ExternalMapToCatalyst_value_isNull0; > /* 066 */ } > /* 067 */ > /* 068 */ > /* 069 */ private void evalIfFalseExpr(InternalRow i) { > /* 070 */ values = new Object[1]; > /* 071 */ resultIsNull1 = false; > /* 072 */ if (!resultIsNull1) { > /* 073 */ > /* 074 */ boolean isNull10 = true; > /* 075 */ scala.collection.Seq value10 = null; > /* 076 */ if (!ExternalMapToCatalyst_value_isNull0) { > /* 077 */ > /* 078 */ isNull10 = false; > /* 079 */ if (!isNull10) { > /* 080 */ > /* 081 */ Object funcResult1 = null; > /* 082 */ funcResult1 = ExternalMapToCatalyst_value0.values(); > /* 083 */ if (funcResult1 == null) { > /* 084 */ isNull10 = true; > /* 085 */ } else { > /* 086 */ value10 = (scala.collection.Seq) funcResult1; > /* 087 */ } > /* 088 */ > /* 089 */ } > /* 090 */ isNull10 = value10 == null; > /* 091 */ } > /* 092 */ resultIsNull1 = isNull10; > /* 093 */ argValue1 = value10; > /* 094 */ } > /* 095 */ > /* 096 */ > /* 097 */ final ArrayData value9 = resultIsNull1 ? null : new > org.apache.spark.sql.catalyst.util.GenericArrayData(argValue1); > /* 098 */ if (resultIsNull1) { > /* 099 */ values[0] = null; > /* 100 */ } else { > /* 101 */ values[0] = value9; > /* 102 */ } > /* 103 */ final InternalRow value8 = new > org.apache.spark.sql.catalyst.expressions.GenericInternalRow(values); > /* 104 */ this.values = null; > /* 105 */ isNull13 = false; > /* 106 */ value13 = value8; > /* 107 */ } > /* 108 */ > /* 109 */ > /* 110 */ // Scala.Function1 need this > /* 111 */ public java.lang.Object apply(java.lang.Object row) { > /* 112 */ return apply((InternalRow) row); > /* 113 */ } > /* 114 */ > /* 115 */ public UnsafeRow apply(InternalRow i) { > /* 116 */ holder.reset(); > /* 117 */ > /* 118 */ rowWriter.zeroOutNullBytes(); > /* 119 */ > /* 120 */ > /* 121 */ boolean isNull3 = i.isNullAt(0); > /* 122 */ local.ItemProperties value3 = isNull3 ? null : > ((local.ItemProperties)i.get(0, null)); > /* 123 */ > /* 124 */ if (isNull3) { > /* 125 */ throw new RuntimeException(((java.lang.String) > references[0])); > /* 126 */ } > /* 127 */ boolean isNull1 = true; > /* 128 */ scala.collection.immutable.Map value1 = null; > /* 129 */ if (!false) { > /* 130 */ > /* 131 */ isNull1 = false; > /* 132 */ if (!isNull1) { > /* 133 */ > /* 134 */ Object funcResult = null; > /* 135 */ funcResult = value3.properties(); > /* 136 */ if (funcResult == null) { > /* 137 */ isNull1 = true; > /* 138 */ } else { > /* 139 */ value1 = (scala.collection.immutable.Map) funcResult; > /* 140 */ } > /* 141 */ > /* 142 */ } > /* 143 */ isNull1 = value1 == null; > /* 144 */ } > /* 145 */ MapData value = null; > /* 146 */ if (!isNull1) { > /* 147 */ final int length = value1.size(); > /* 148 */ final Object[] convertedKeys = new Object[length]; > /* 149 */ final Object[] convertedValues = new Object[length]; > /* 150 */ int index = 0; > /* 151 */ final scala.collection.Iterator entries = value1.iterator(); > /* 152 */ while(entries.hasNext()) { > /* 153 */ > /* 154 */ final scala.Tuple2 entry = (scala.Tuple2) entries.next(); > /* 155 */ java.lang.String ExternalMapToCatalyst_key0 = > (java.lang.String) entry._1(); > /* 156 */ local.Values ExternalMapToCatalyst_value0 = (local.Values) > entry._2(); > /* 157 */ > /* 158 */ boolean ExternalMapToCatalyst_value_isNull0 = > ExternalMapToCatalyst_value0 == null; > /* 159 */ > /* 160 */ > /* 161 */ resultIsNull = false; > /* 162 */ if (!resultIsNull) { > /* 163 */ > /* 164 */ resultIsNull = false; > /* 165 */ argValue = ExternalMapToCatalyst_key0; > /* 166 */ } > /* 167 */ > /* 168 */ boolean isNull4 = resultIsNull; > /* 169 */ final UTF8String value4 = resultIsNull ? null : > org.apache.spark.unsafe.types.UTF8String.fromString(argValue); > /* 170 */ isNull4 = value4 == null; > /* 171 */ if (isNull4) { > /* 172 */ throw new RuntimeException("Cannot use null as map key!"); > /* 173 */ } else { > /* 174 */ convertedKeys[index] = (UTF8String) value4; > /* 175 */ } > /* 176 */ > /* 177 */ > /* 178 */ evalIfCondExpr(i); > /* 179 */ boolean isNull5 = false; > /* 180 */ InternalRow value5 = null; > /* 181 */ if (!isNull11 && value11) { > /* 182 */ evalIfTrueExpr(i); > /* 183 */ isNull5 = isNull12; > /* 184 */ value5 = value12; > /* 185 */ } else { > /* 186 */ evalIfFalseExpr(i); > /* 187 */ isNull5 = isNull13; > /* 188 */ value5 = value13; > /* 189 */ } > /* 190 */ if (isNull5) { > /* 191 */ convertedValues[index] = null; > /* 192 */ } else { > /* 193 */ convertedValues[index] = (InternalRow) value5; > /* 194 */ } > /* 195 */ > /* 196 */ index++; > /* 197 */ } > /* 198 */ > /* 199 */ value = new > org.apache.spark.sql.catalyst.util.ArrayBasedMapData(new > org.apache.spark.sql.catalyst.util.GenericArrayData(convertedKeys), new > org.apache.spark.sql.catalyst.util.GenericArrayData(convertedValues)); > /* 200 */ } > /* 201 */ if (isNull1) { > /* 202 */ rowWriter.setNullAt(0); > /* 203 */ } else { > /* 204 */ // Remember the current cursor so that we can calculate how > many bytes are > /* 205 */ // written later. > /* 206 */ final int tmpCursor = holder.cursor; > /* 207 */ > /* 208 */ if (value instanceof UnsafeMapData) { > /* 209 */ > /* 210 */ final int sizeInBytes = ((UnsafeMapData) > value).getSizeInBytes(); > /* 211 */ // grow the global buffer before writing data. > /* 212 */ holder.grow(sizeInBytes); > /* 213 */ ((UnsafeMapData) value).writeToMemory(holder.buffer, > holder.cursor); > /* 214 */ holder.cursor += sizeInBytes; > /* 215 */ > /* 216 */ } else { > /* 217 */ final ArrayData keys = value.keyArray(); > /* 218 */ final ArrayData values1 = value.valueArray(); > /* 219 */ > /* 220 */ // preserve 8 bytes to write the key array numBytes later. > /* 221 */ holder.grow(8); > /* 222 */ holder.cursor += 8; > /* 223 */ > /* 224 */ // Remember the current cursor so that we can write > numBytes of key array later. > /* 225 */ final int tmpCursor1 = holder.cursor; > /* 226 */ > /* 227 */ > /* 228 */ if (keys instanceof UnsafeArrayData) { > /* 229 */ > /* 230 */ final int sizeInBytes1 = ((UnsafeArrayData) > keys).getSizeInBytes(); > /* 231 */ // grow the global buffer before writing data. > /* 232 */ holder.grow(sizeInBytes1); > /* 233 */ ((UnsafeArrayData) keys).writeToMemory(holder.buffer, > holder.cursor); > /* 234 */ holder.cursor += sizeInBytes1; > /* 235 */ > /* 236 */ } else { > /* 237 */ final int numElements = keys.numElements(); > /* 238 */ arrayWriter.initialize(holder, numElements, 8); > /* 239 */ > /* 240 */ for (int index1 = 0; index1 < numElements; index1++) { > /* 241 */ if (keys.isNullAt(index1)) { > /* 242 */ arrayWriter.setNull(index1); > /* 243 */ } else { > /* 244 */ final UTF8String element = keys.getUTF8String(index1); > /* 245 */ arrayWriter.write(index1, element); > /* 246 */ } > /* 247 */ } > /* 248 */ } > /* 249 */ > /* 250 */ // Write the numBytes of key array into the first 8 bytes. > /* 251 */ Platform.putLong(holder.buffer, tmpCursor1 - 8, > holder.cursor - tmpCursor1); > /* 252 */ > /* 253 */ > /* 254 */ if (values1 instanceof UnsafeArrayData) { > /* 255 */ > /* 256 */ final int sizeInBytes4 = ((UnsafeArrayData) > values1).getSizeInBytes(); > /* 257 */ // grow the global buffer before writing data. > /* 258 */ holder.grow(sizeInBytes4); > /* 259 */ ((UnsafeArrayData) values1).writeToMemory(holder.buffer, > holder.cursor); > /* 260 */ holder.cursor += sizeInBytes4; > /* 261 */ > /* 262 */ } else { > /* 263 */ final int numElements1 = values1.numElements(); > /* 264 */ arrayWriter1.initialize(holder, numElements1, 8); > /* 265 */ > /* 266 */ for (int index2 = 0; index2 < numElements1; index2++) { > /* 267 */ if (values1.isNullAt(index2)) { > /* 268 */ arrayWriter1.setNull(index2); > /* 269 */ } else { > /* 270 */ final InternalRow element1 = > values1.getStruct(index2, 1); > /* 271 */ > /* 272 */ final int tmpCursor3 = holder.cursor; > /* 273 */ > /* 274 */ if (element1 instanceof UnsafeRow) { > /* 275 */ > /* 276 */ final int sizeInBytes2 = ((UnsafeRow) > element1).getSizeInBytes(); > /* 277 */ // grow the global buffer before writing data. > /* 278 */ holder.grow(sizeInBytes2); > /* 279 */ ((UnsafeRow) element1).writeToMemory(holder.buffer, > holder.cursor); > /* 280 */ holder.cursor += sizeInBytes2; > /* 281 */ > /* 282 */ } else { > /* 283 */ rowWriter1.reset(); > /* 284 */ > /* 285 */ final ArrayData fieldName = element1.getArray(0); > /* 286 */ if (element1.isNullAt(0)) { > /* 287 */ rowWriter1.setNullAt(0); > /* 288 */ } else { > /* 289 */ // Remember the current cursor so that we can > calculate how many bytes are > /* 290 */ // written later. > /* 291 */ final int tmpCursor4 = holder.cursor; > /* 292 */ > /* 293 */ if (fieldName instanceof UnsafeArrayData) { > /* 294 */ > /* 295 */ final int sizeInBytes3 = ((UnsafeArrayData) > fieldName).getSizeInBytes(); > /* 296 */ // grow the global buffer before writing data. > /* 297 */ holder.grow(sizeInBytes3); > /* 298 */ ((UnsafeArrayData) > fieldName).writeToMemory(holder.buffer, holder.cursor); > /* 299 */ holder.cursor += sizeInBytes3; > /* 300 */ > /* 301 */ } else { > /* 302 */ final int numElements2 = > fieldName.numElements(); > /* 303 */ arrayWriter2.initialize(holder, numElements2, > 8); > /* 304 */ > /* 305 */ for (int index3 = 0; index3 < numElements2; > index3++) { > /* 306 */ if (fieldName.isNullAt(index3)) { > /* 307 */ arrayWriter2.setNullDouble(index3); > /* 308 */ } else { > /* 309 */ final double element2 = > fieldName.getDouble(index3); > /* 310 */ arrayWriter2.write(index3, element2); > /* 311 */ } > /* 312 */ } > /* 313 */ } > /* 314 */ > /* 315 */ rowWriter1.setOffsetAndSize(0, tmpCursor4, > holder.cursor - tmpCursor4); > /* 316 */ } > /* 317 */ } > /* 318 */ > /* 319 */ arrayWriter1.setOffsetAndSize(index2, tmpCursor3, > holder.cursor - tmpCursor3); > /* 320 */ > /* 321 */ } > /* 322 */ } > /* 323 */ } > /* 324 */ > /* 325 */ } > /* 326 */ > /* 327 */ rowWriter.setOffsetAndSize(0, tmpCursor, holder.cursor - > tmpCursor); > /* 328 */ } > /* 329 */ result.setTotalSize(holder.totalSize()); > /* 330 */ return result; > /* 331 */ } > /* 332 */ } > org.codehaus.commons.compiler.CompileException: File 'generated.java', Line > 65, Column 46: Expression "ExternalMapToCatalyst_value_isNull0" is not an > rvalue > at > org.codehaus.janino.UnitCompiler.compileError(UnitCompiler.java:11004) > at > org.codehaus.janino.UnitCompiler.toRvalueOrCompileException(UnitCompiler.java:6639) > at > org.codehaus.janino.UnitCompiler.getConstantValue2(UnitCompiler.java:5001) > at org.codehaus.janino.UnitCompiler.access$10500(UnitCompiler.java:206) > at > org.codehaus.janino.UnitCompiler$13.visitAmbiguousName(UnitCompiler.java:4984) > at org.codehaus.janino.Java$AmbiguousName.accept(Java.java:3633) > at org.codehaus.janino.Java$Lvalue.accept(Java.java:3563) > at > org.codehaus.janino.UnitCompiler.getConstantValue(UnitCompiler.java:4956) > at > org.codehaus.janino.UnitCompiler.compileGetValue(UnitCompiler.java:4925) > at org.codehaus.janino.UnitCompiler.compile2(UnitCompiler.java:3189) > at org.codehaus.janino.UnitCompiler.access$5100(UnitCompiler.java:206) > at > org.codehaus.janino.UnitCompiler$9.visitAssignment(UnitCompiler.java:3143) > at > org.codehaus.janino.UnitCompiler$9.visitAssignment(UnitCompiler.java:3139) > at org.codehaus.janino.Java$Assignment.accept(Java.java:3847) > at org.codehaus.janino.UnitCompiler.compile(UnitCompiler.java:3139) > at org.codehaus.janino.UnitCompiler.compile2(UnitCompiler.java:2112) > at org.codehaus.janino.UnitCompiler.access$1700(UnitCompiler.java:206) > at > org.codehaus.janino.UnitCompiler$6.visitExpressionStatement(UnitCompiler.java:1377) > at > org.codehaus.janino.UnitCompiler$6.visitExpressionStatement(UnitCompiler.java:1370) > at org.codehaus.janino.Java$ExpressionStatement.accept(Java.java:2558) > at org.codehaus.janino.UnitCompiler.compile(UnitCompiler.java:1370) > at > org.codehaus.janino.UnitCompiler.compileStatements(UnitCompiler.java:1450) > at org.codehaus.janino.UnitCompiler.compile(UnitCompiler.java:2811) > at > org.codehaus.janino.UnitCompiler.compileDeclaredMethods(UnitCompiler.java:1262) > at > org.codehaus.janino.UnitCompiler.compileDeclaredMethods(UnitCompiler.java:1234) > at org.codehaus.janino.UnitCompiler.compile2(UnitCompiler.java:538) > at org.codehaus.janino.UnitCompiler.compile2(UnitCompiler.java:890) > at org.codehaus.janino.UnitCompiler.compile2(UnitCompiler.java:894) > at org.codehaus.janino.UnitCompiler.access$600(UnitCompiler.java:206) > at > org.codehaus.janino.UnitCompiler$2.visitMemberClassDeclaration(UnitCompiler.java:377) > at > org.codehaus.janino.UnitCompiler$2.visitMemberClassDeclaration(UnitCompiler.java:369) > at > org.codehaus.janino.Java$MemberClassDeclaration.accept(Java.java:1128) > at org.codehaus.janino.UnitCompiler.compile(UnitCompiler.java:369) > at > org.codehaus.janino.UnitCompiler.compileDeclaredMemberTypes(UnitCompiler.java:1209) > at org.codehaus.janino.UnitCompiler.compile2(UnitCompiler.java:564) > at org.codehaus.janino.UnitCompiler.compile2(UnitCompiler.java:420) > at org.codehaus.janino.UnitCompiler.access$400(UnitCompiler.java:206) > at > org.codehaus.janino.UnitCompiler$2.visitPackageMemberClassDeclaration(UnitCompiler.java:374) > at > org.codehaus.janino.UnitCompiler$2.visitPackageMemberClassDeclaration(UnitCompiler.java:369) > at > org.codehaus.janino.Java$AbstractPackageMemberClassDeclaration.accept(Java.java:1309) > at org.codehaus.janino.UnitCompiler.compile(UnitCompiler.java:369) > at org.codehaus.janino.UnitCompiler.compileUnit(UnitCompiler.java:345) > at > org.codehaus.janino.SimpleCompiler.compileToClassLoader(SimpleCompiler.java:396) > at > org.codehaus.janino.ClassBodyEvaluator.compileToClass(ClassBodyEvaluator.java:311) > at > org.codehaus.janino.ClassBodyEvaluator.cook(ClassBodyEvaluator.java:229) > at org.codehaus.janino.SimpleCompiler.cook(SimpleCompiler.java:196) > at org.codehaus.commons.compiler.Cookable.cook(Cookable.java:91) > at > org.apache.spark.sql.catalyst.expressions.codegen.CodeGenerator$.org$apache$spark$sql$catalyst$expressions$codegen$CodeGenerator$$doCompile(CodeGenerator.scala:935) > at > org.apache.spark.sql.catalyst.expressions.codegen.CodeGenerator$$anon$1.load(CodeGenerator.scala:998) > at > org.apache.spark.sql.catalyst.expressions.codegen.CodeGenerator$$anon$1.load(CodeGenerator.scala:995) > at > org.spark_project.guava.cache.LocalCache$LoadingValueReference.loadFuture(LocalCache.java:3599) > at > org.spark_project.guava.cache.LocalCache$Segment.loadSync(LocalCache.java:2379) > at > org.spark_project.guava.cache.LocalCache$Segment.lockedGetOrLoad(LocalCache.java:2342) > at > org.spark_project.guava.cache.LocalCache$Segment.get(LocalCache.java:2257) > at org.spark_project.guava.cache.LocalCache.get(LocalCache.java:4000) > at > org.spark_project.guava.cache.LocalCache.getOrLoad(LocalCache.java:4004) > at > org.spark_project.guava.cache.LocalCache$LocalLoadingCache.get(LocalCache.java:4874) > at > org.apache.spark.sql.catalyst.expressions.codegen.CodeGenerator$.compile(CodeGenerator.scala:890) > at > org.apache.spark.sql.catalyst.expressions.codegen.GenerateUnsafeProjection$.create(GenerateUnsafeProjection.scala:405) > at > org.apache.spark.sql.catalyst.expressions.codegen.GenerateUnsafeProjection$.create(GenerateUnsafeProjection.scala:359) > at > org.apache.spark.sql.catalyst.expressions.codegen.GenerateUnsafeProjection$.create(GenerateUnsafeProjection.scala:32) > at > org.apache.spark.sql.catalyst.expressions.codegen.CodeGenerator.generate(CodeGenerator.scala:874) > at > org.apache.spark.sql.catalyst.encoders.ExpressionEncoder.extractProjection$lzycompute(ExpressionEncoder.scala:266) > at > org.apache.spark.sql.catalyst.encoders.ExpressionEncoder.extractProjection(ExpressionEncoder.scala:266) > at > org.apache.spark.sql.catalyst.encoders.ExpressionEncoder.toRow(ExpressionEncoder.scala:290) > at > org.apache.spark.sql.SparkSession$$anonfun$2.apply(SparkSession.scala:429) > at > org.apache.spark.sql.SparkSession$$anonfun$2.apply(SparkSession.scala:429) > at > scala.collection.TraversableLike$$anonfun$map$1.apply(TraversableLike.scala:245) > at > scala.collection.TraversableLike$$anonfun$map$1.apply(TraversableLike.scala:245) > at scala.collection.immutable.List.foreach(List.scala:383) > at scala.collection.TraversableLike$class.map(TraversableLike.scala:245) > at scala.collection.immutable.List.map(List.scala:286) > at > org.apache.spark.sql.SparkSession.createDataset(SparkSession.scala:429) > at org.apache.spark.sql.SQLContext.createDataset(SQLContext.scala:389) > at > org.apache.spark.sql.SQLImplicits.localSeqToDatasetHolder(SQLImplicits.scala:173) > at local.TestApp$.main(TestApp.scala:63) > at local.TestApp.main(TestApp.scala) > at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) > at > sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) > at > sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) > at java.lang.reflect.Method.invoke(Method.java:498) > at com.intellij.rt.execution.application.AppMain.main(AppMain.java:147) > Exception in thread "main" java.lang.RuntimeException: Error while encoding: > java.util.concurrent.ExecutionException: java.lang.Exception: failed to > compile: org.codehaus.commons.compiler.CompileException: File > 'generated.java', Line 65, Column 46: Expression > "ExternalMapToCatalyst_value_isNull0" is not an rvalue > /* 001 */ public java.lang.Object generate(Object[] references) { > /* 002 */ return new SpecificUnsafeProjection(references); > /* 003 */ } > /* 004 */ > /* 005 */ class SpecificUnsafeProjection extends > org.apache.spark.sql.catalyst.expressions.UnsafeProjection { > /* 006 */ > /* 007 */ private Object[] references; > /* 008 */ private boolean resultIsNull; > /* 009 */ private java.lang.String argValue; > /* 010 */ private Object[] values; > /* 011 */ private boolean resultIsNull1; > /* 012 */ private scala.collection.Seq argValue1; > /* 013 */ private boolean isNull11; > /* 014 */ private boolean value11; > /* 015 */ private boolean isNull12; > /* 016 */ private InternalRow value12; > /* 017 */ private boolean isNull13; > /* 018 */ private InternalRow value13; > /* 019 */ private UnsafeRow result; > /* 020 */ private > org.apache.spark.sql.catalyst.expressions.codegen.BufferHolder holder; > /* 021 */ private > org.apache.spark.sql.catalyst.expressions.codegen.UnsafeRowWriter rowWriter; > /* 022 */ private > org.apache.spark.sql.catalyst.expressions.codegen.UnsafeArrayWriter > arrayWriter; > /* 023 */ private > org.apache.spark.sql.catalyst.expressions.codegen.UnsafeArrayWriter > arrayWriter1; > /* 024 */ private > org.apache.spark.sql.catalyst.expressions.codegen.UnsafeRowWriter rowWriter1; > /* 025 */ private > org.apache.spark.sql.catalyst.expressions.codegen.UnsafeArrayWriter > arrayWriter2; > /* 026 */ > /* 027 */ public SpecificUnsafeProjection(Object[] references) { > /* 028 */ this.references = references; > /* 029 */ > /* 030 */ > /* 031 */ this.values = null; > /* 032 */ > /* 033 */ > /* 034 */ isNull11 = false; > /* 035 */ value11 = false; > /* 036 */ isNull12 = false; > /* 037 */ value12 = null; > /* 038 */ isNull13 = false; > /* 039 */ value13 = null; > /* 040 */ result = new UnsafeRow(1); > /* 041 */ this.holder = new > org.apache.spark.sql.catalyst.expressions.codegen.BufferHolder(result, 32); > /* 042 */ this.rowWriter = new > org.apache.spark.sql.catalyst.expressions.codegen.UnsafeRowWriter(holder, 1); > /* 043 */ this.arrayWriter = new > org.apache.spark.sql.catalyst.expressions.codegen.UnsafeArrayWriter(); > /* 044 */ this.arrayWriter1 = new > org.apache.spark.sql.catalyst.expressions.codegen.UnsafeArrayWriter(); > /* 045 */ this.rowWriter1 = new > org.apache.spark.sql.catalyst.expressions.codegen.UnsafeRowWriter(holder, 1); > /* 046 */ this.arrayWriter2 = new > org.apache.spark.sql.catalyst.expressions.codegen.UnsafeArrayWriter(); > /* 047 */ > /* 048 */ } > /* 049 */ > /* 050 */ public void initialize(int partitionIndex) { > /* 051 */ > /* 052 */ } > /* 053 */ > /* 054 */ > /* 055 */ private void evalIfTrueExpr(InternalRow i) { > /* 056 */ final InternalRow value7 = null; > /* 057 */ isNull12 = true; > /* 058 */ value12 = value7; > /* 059 */ } > /* 060 */ > /* 061 */ > /* 062 */ private void evalIfCondExpr(InternalRow i) { > /* 063 */ > /* 064 */ isNull11 = false; > /* 065 */ value11 = ExternalMapToCatalyst_value_isNull0; > /* 066 */ } > /* 067 */ > /* 068 */ > /* 069 */ private void evalIfFalseExpr(InternalRow i) { > /* 070 */ values = new Object[1]; > /* 071 */ resultIsNull1 = false; > /* 072 */ if (!resultIsNull1) { > /* 073 */ > /* 074 */ boolean isNull10 = true; > /* 075 */ scala.collection.Seq value10 = null; > /* 076 */ if (!ExternalMapToCatalyst_value_isNull0) { > /* 077 */ > /* 078 */ isNull10 = false; > /* 079 */ if (!isNull10) { > /* 080 */ > /* 081 */ Object funcResult1 = null; > /* 082 */ funcResult1 = ExternalMapToCatalyst_value0.values(); > /* 083 */ if (funcResult1 == null) { > /* 084 */ isNull10 = true; > /* 085 */ } else { > /* 086 */ value10 = (scala.collection.Seq) funcResult1; > /* 087 */ } > /* 088 */ > /* 089 */ } > /* 090 */ isNull10 = value10 == null; > /* 091 */ } > /* 092 */ resultIsNull1 = isNull10; > /* 093 */ argValue1 = value10; > /* 094 */ } > /* 095 */ > /* 096 */ > /* 097 */ final ArrayData value9 = resultIsNull1 ? null : new > org.apache.spark.sql.catalyst.util.GenericArrayData(argValue1); > /* 098 */ if (resultIsNull1) { > /* 099 */ values[0] = null; > /* 100 */ } else { > /* 101 */ values[0] = value9; > /* 102 */ } > /* 103 */ final InternalRow value8 = new > org.apache.spark.sql.catalyst.expressions.GenericInternalRow(values); > /* 104 */ this.values = null; > /* 105 */ isNull13 = false; > /* 106 */ value13 = value8; > /* 107 */ } > /* 108 */ > /* 109 */ > /* 110 */ // Scala.Function1 need this > /* 111 */ public java.lang.Object apply(java.lang.Object row) { > /* 112 */ return apply((InternalRow) row); > /* 113 */ } > /* 114 */ > /* 115 */ public UnsafeRow apply(InternalRow i) { > /* 116 */ holder.reset(); > /* 117 */ > /* 118 */ rowWriter.zeroOutNullBytes(); > /* 119 */ > /* 120 */ > /* 121 */ boolean isNull3 = i.isNullAt(0); > /* 122 */ local.ItemProperties value3 = isNull3 ? null : > ((local.ItemProperties)i.get(0, null)); > /* 123 */ > /* 124 */ if (isNull3) { > /* 125 */ throw new RuntimeException(((java.lang.String) > references[0])); > /* 126 */ } > /* 127 */ boolean isNull1 = true; > /* 128 */ scala.collection.immutable.Map value1 = null; > /* 129 */ if (!false) { > /* 130 */ > /* 131 */ isNull1 = false; > /* 132 */ if (!isNull1) { > /* 133 */ > /* 134 */ Object funcResult = null; > /* 135 */ funcResult = value3.properties(); > /* 136 */ if (funcResult == null) { > /* 137 */ isNull1 = true; > /* 138 */ } else { > /* 139 */ value1 = (scala.collection.immutable.Map) funcResult; > /* 140 */ } > /* 141 */ > /* 142 */ } > /* 143 */ isNull1 = value1 == null; > /* 144 */ } > /* 145 */ MapData value = null; > /* 146 */ if (!isNull1) { > /* 147 */ final int length = value1.size(); > /* 148 */ final Object[] convertedKeys = new Object[length]; > /* 149 */ final Object[] convertedValues = new Object[length]; > /* 150 */ int index = 0; > /* 151 */ final scala.collection.Iterator entries = value1.iterator(); > /* 152 */ while(entries.hasNext()) { > /* 153 */ > /* 154 */ final scala.Tuple2 entry = (scala.Tuple2) entries.next(); > /* 155 */ java.lang.String ExternalMapToCatalyst_key0 = > (java.lang.String) entry._1(); > /* 156 */ local.Values ExternalMapToCatalyst_value0 = (local.Values) > entry._2(); > /* 157 */ > /* 158 */ boolean ExternalMapToCatalyst_value_isNull0 = > ExternalMapToCatalyst_value0 == null; > /* 159 */ > /* 160 */ > /* 161 */ resultIsNull = false; > /* 162 */ if (!resultIsNull) { > /* 163 */ > /* 164 */ resultIsNull = false; > /* 165 */ argValue = ExternalMapToCatalyst_key0; > /* 166 */ } > /* 167 */ > /* 168 */ boolean isNull4 = resultIsNull; > /* 169 */ final UTF8String value4 = resultIsNull ? null : > org.apache.spark.unsafe.types.UTF8String.fromString(argValue); > /* 170 */ isNull4 = value4 == null; > /* 171 */ if (isNull4) { > /* 172 */ throw new RuntimeException("Cannot use null as map key!"); > /* 173 */ } else { > /* 174 */ convertedKeys[index] = (UTF8String) value4; > /* 175 */ } > /* 176 */ > /* 177 */ > /* 178 */ evalIfCondExpr(i); > /* 179 */ boolean isNull5 = false; > /* 180 */ InternalRow value5 = null; > /* 181 */ if (!isNull11 && value11) { > /* 182 */ evalIfTrueExpr(i); > /* 183 */ isNull5 = isNull12; > /* 184 */ value5 = value12; > /* 185 */ } else { > /* 186 */ evalIfFalseExpr(i); > /* 187 */ isNull5 = isNull13; > /* 188 */ value5 = value13; > /* 189 */ } > /* 190 */ if (isNull5) { > /* 191 */ convertedValues[index] = null; > /* 192 */ } else { > /* 193 */ convertedValues[index] = (InternalRow) value5; > /* 194 */ } > /* 195 */ > /* 196 */ index++; > /* 197 */ } > /* 198 */ > /* 199 */ value = new > org.apache.spark.sql.catalyst.util.ArrayBasedMapData(new > org.apache.spark.sql.catalyst.util.GenericArrayData(convertedKeys), new > org.apache.spark.sql.catalyst.util.GenericArrayData(convertedValues)); > /* 200 */ } > /* 201 */ if (isNull1) { > /* 202 */ rowWriter.setNullAt(0); > /* 203 */ } else { > /* 204 */ // Remember the current cursor so that we can calculate how > many bytes are > /* 205 */ // written later. > /* 206 */ final int tmpCursor = holder.cursor; > /* 207 */ > /* 208 */ if (value instanceof UnsafeMapData) { > /* 209 */ > /* 210 */ final int sizeInBytes = ((UnsafeMapData) > value).getSizeInBytes(); > /* 211 */ // grow the global buffer before writing data. > /* 212 */ holder.grow(sizeInBytes); > /* 213 */ ((UnsafeMapData) value).writeToMemory(holder.buffer, > holder.cursor); > /* 214 */ holder.cursor += sizeInBytes; > /* 215 */ > /* 216 */ } else { > /* 217 */ final ArrayData keys = value.keyArray(); > /* 218 */ final ArrayData values1 = value.valueArray(); > /* 219 */ > /* 220 */ // preserve 8 bytes to write the key array numBytes later. > /* 221 */ holder.grow(8); > /* 222 */ holder.cursor += 8; > /* 223 */ > /* 224 */ // Remember the current cursor so that we can write > numBytes of key array later. > /* 225 */ final int tmpCursor1 = holder.cursor; > /* 226 */ > /* 227 */ > /* 228 */ if (keys instanceof UnsafeArrayData) { > /* 229 */ > /* 230 */ final int sizeInBytes1 = ((UnsafeArrayData) > keys).getSizeInBytes(); > /* 231 */ // grow the global buffer before writing data. > /* 232 */ holder.grow(sizeInBytes1); > /* 233 */ ((UnsafeArrayData) keys).writeToMemory(holder.buffer, > holder.cursor); > /* 234 */ holder.cursor += sizeInBytes1; > /* 235 */ > /* 236 */ } else { > /* 237 */ final int numElements = keys.numElements(); > /* 238 */ arrayWriter.initialize(holder, numElements, 8); > /* 239 */ > /* 240 */ for (int index1 = 0; index1 < numElements; index1++) { > /* 241 */ if (keys.isNullAt(index1)) { > /* 242 */ arrayWriter.setNull(index1); > /* 243 */ } else { > /* 244 */ final UTF8String element = keys.getUTF8String(index1); > /* 245 */ arrayWriter.write(index1, element); > /* 246 */ } > /* 247 */ } > /* 248 */ } > /* 249 */ > /* 250 */ // Write the numBytes of key array into the first 8 bytes. > /* 251 */ Platform.putLong(holder.buffer, tmpCursor1 - 8, > holder.cursor - tmpCursor1); > /* 252 */ > /* 253 */ > /* 254 */ if (values1 instanceof UnsafeArrayData) { > /* 255 */ > /* 256 */ final int sizeInBytes4 = ((UnsafeArrayData) > values1).getSizeInBytes(); > /* 257 */ // grow the global buffer before writing data. > /* 258 */ holder.grow(sizeInBytes4); > /* 259 */ ((UnsafeArrayData) values1).writeToMemory(holder.buffer, > holder.cursor); > /* 260 */ holder.cursor += sizeInBytes4; > /* 261 */ > /* 262 */ } else { > /* 263 */ final int numElements1 = values1.numElements(); > /* 264 */ arrayWriter1.initialize(holder, numElements1, 8); > /* 265 */ > /* 266 */ for (int index2 = 0; index2 < numElements1; index2++) { > /* 267 */ if (values1.isNullAt(index2)) { > /* 268 */ arrayWriter1.setNull(index2); > /* 269 */ } else { > /* 270 */ final InternalRow element1 = > values1.getStruct(index2, 1); > /* 271 */ > /* 272 */ final int tmpCursor3 = holder.cursor; > /* 273 */ > /* 274 */ if (element1 instanceof UnsafeRow) { > /* 275 */ > /* 276 */ final int sizeInBytes2 = ((UnsafeRow) > element1).getSizeInBytes(); > /* 277 */ // grow the global buffer before writing data. > /* 278 */ holder.grow(sizeInBytes2); > /* 279 */ ((UnsafeRow) element1).writeToMemory(holder.buffer, > holder.cursor); > /* 280 */ holder.cursor += sizeInBytes2; > /* 281 */ > /* 282 */ } else { > /* 283 */ rowWriter1.reset(); > /* 284 */ > /* 285 */ final ArrayData fieldName = element1.getArray(0); > /* 286 */ if (element1.isNullAt(0)) { > /* 287 */ rowWriter1.setNullAt(0); > /* 288 */ } else { > /* 289 */ // Remember the current cursor so that we can > calculate how many bytes are > /* 290 */ // written later. > /* 291 */ final int tmpCursor4 = holder.cursor; > /* 292 */ > /* 293 */ if (fieldName instanceof UnsafeArrayData) { > /* 294 */ > /* 295 */ final int sizeInBytes3 = ((UnsafeArrayData) > fieldName).getSizeInBytes(); > /* 296 */ // grow the global buffer before writing data. > /* 297 */ holder.grow(sizeInBytes3); > /* 298 */ ((UnsafeArrayData) > fieldName).writeToMemory(holder.buffer, holder.cursor); > /* 299 */ holder.cursor += sizeInBytes3; > /* 300 */ > /* 301 */ } else { > /* 302 */ final int numElements2 = > fieldName.numElements(); > /* 303 */ arrayWriter2.initialize(holder, numElements2, > 8); > /* 304 */ > /* 305 */ for (int index3 = 0; index3 < numElements2; > index3++) { > /* 306 */ if (fieldName.isNullAt(index3)) { > /* 307 */ arrayWriter2.setNullDouble(index3); > /* 308 */ } else { > /* 309 */ final double element2 = > fieldName.getDouble(index3); > /* 310 */ arrayWriter2.write(index3, element2); > /* 311 */ } > /* 312 */ } > /* 313 */ } > /* 314 */ > /* 315 */ rowWriter1.setOffsetAndSize(0, tmpCursor4, > holder.cursor - tmpCursor4); > /* 316 */ } > /* 317 */ } > /* 318 */ > /* 319 */ arrayWriter1.setOffsetAndSize(index2, tmpCursor3, > holder.cursor - tmpCursor3); > /* 320 */ > /* 321 */ } > /* 322 */ } > /* 323 */ } > /* 324 */ > /* 325 */ } > /* 326 */ > /* 327 */ rowWriter.setOffsetAndSize(0, tmpCursor, holder.cursor - > tmpCursor); > /* 328 */ } > /* 329 */ result.setTotalSize(holder.totalSize()); > /* 330 */ return result; > /* 331 */ } > /* 332 */ } > externalmaptocatalyst(ExternalMapToCatalyst_key0, ObjectType(class > java.lang.String), staticinvoke(class > org.apache.spark.unsafe.types.UTF8String, StringType, fromString, > lambdavariable(ExternalMapToCatalyst_key0, false, ObjectType(class > java.lang.String)), true), ExternalMapToCatalyst_value0, > ExternalMapToCatalyst_value_isNull0, ObjectType(class local.Values), if > (isnull(lambdavariable(ExternalMapToCatalyst_value0, > ExternalMapToCatalyst_value_isNull0, ObjectType(class local.Values)))) null > else named_struct(values, newInstance(class > org.apache.spark.sql.catalyst.util.GenericArrayData)), assertnotnull(input[0, > local.ItemProperties, true], top level Product input object).properties) AS > properties#0 > +- externalmaptocatalyst(ExternalMapToCatalyst_key0, ObjectType(class > java.lang.String), staticinvoke(class > org.apache.spark.unsafe.types.UTF8String, StringType, fromString, > lambdavariable(ExternalMapToCatalyst_key0, false, ObjectType(class > java.lang.String)), true), ExternalMapToCatalyst_value0, > ExternalMapToCatalyst_value_isNull0, ObjectType(class local.Values), if > (isnull(lambdavariable(ExternalMapToCatalyst_value0, > ExternalMapToCatalyst_value_isNull0, ObjectType(class local.Values)))) null > else named_struct(values, newInstance(class > org.apache.spark.sql.catalyst.util.GenericArrayData)), assertnotnull(input[0, > local.ItemProperties, true], top level Product input object).properties) > +- assertnotnull(input[0, local.ItemProperties, true], top level Product > input object).properties > +- assertnotnull(input[0, local.ItemProperties, true], top level > Product input object) > +- input[0, local.ItemProperties, true] > at > org.apache.spark.sql.catalyst.encoders.ExpressionEncoder.toRow(ExpressionEncoder.scala:293) > at > org.apache.spark.sql.SparkSession$$anonfun$2.apply(SparkSession.scala:429) > at > org.apache.spark.sql.SparkSession$$anonfun$2.apply(SparkSession.scala:429) > at > scala.collection.TraversableLike$$anonfun$map$1.apply(TraversableLike.scala:245) > at > scala.collection.TraversableLike$$anonfun$map$1.apply(TraversableLike.scala:245) > at scala.collection.immutable.List.foreach(List.scala:383) > at scala.collection.TraversableLike$class.map(TraversableLike.scala:245) > at scala.collection.immutable.List.map(List.scala:286) > at > org.apache.spark.sql.SparkSession.createDataset(SparkSession.scala:429) > at org.apache.spark.sql.SQLContext.createDataset(SQLContext.scala:389) > at > org.apache.spark.sql.SQLImplicits.localSeqToDatasetHolder(SQLImplicits.scala:173) > at local.TestApp$.main(TestApp.scala:63) > at local.TestApp.main(TestApp.scala) > at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) > at > sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) > at > sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) > at java.lang.reflect.Method.invoke(Method.java:498) > at com.intellij.rt.execution.application.AppMain.main(AppMain.java:147) > Caused by: java.util.concurrent.ExecutionException: java.lang.Exception: > failed to compile: org.codehaus.commons.compiler.CompileException: File > 'generated.java', Line 65, Column 46: Expression > "ExternalMapToCatalyst_value_isNull0" is not an rvalue > /* 001 */ public java.lang.Object generate(Object[] references) { > /* 002 */ return new SpecificUnsafeProjection(references); > /* 003 */ } > /* 004 */ > /* 005 */ class SpecificUnsafeProjection extends > org.apache.spark.sql.catalyst.expressions.UnsafeProjection { > /* 006 */ > /* 007 */ private Object[] references; > /* 008 */ private boolean resultIsNull; > /* 009 */ private java.lang.String argValue; > /* 010 */ private Object[] values; > /* 011 */ private boolean resultIsNull1; > /* 012 */ private scala.collection.Seq argValue1; > /* 013 */ private boolean isNull11; > /* 014 */ private boolean value11; > /* 015 */ private boolean isNull12; > /* 016 */ private InternalRow value12; > /* 017 */ private boolean isNull13; > /* 018 */ private InternalRow value13; > /* 019 */ private UnsafeRow result; > /* 020 */ private > org.apache.spark.sql.catalyst.expressions.codegen.BufferHolder holder; > /* 021 */ private > org.apache.spark.sql.catalyst.expressions.codegen.UnsafeRowWriter rowWriter; > /* 022 */ private > org.apache.spark.sql.catalyst.expressions.codegen.UnsafeArrayWriter > arrayWriter; > /* 023 */ private > org.apache.spark.sql.catalyst.expressions.codegen.UnsafeArrayWriter > arrayWriter1; > /* 024 */ private > org.apache.spark.sql.catalyst.expressions.codegen.UnsafeRowWriter rowWriter1; > /* 025 */ private > org.apache.spark.sql.catalyst.expressions.codegen.UnsafeArrayWriter > arrayWriter2; > /* 026 */ > /* 027 */ public SpecificUnsafeProjection(Object[] references) { > /* 028 */ this.references = references; > /* 029 */ > /* 030 */ > /* 031 */ this.values = null; > /* 032 */ > /* 033 */ > /* 034 */ isNull11 = false; > /* 035 */ value11 = false; > /* 036 */ isNull12 = false; > /* 037 */ value12 = null; > /* 038 */ isNull13 = false; > /* 039 */ value13 = null; > /* 040 */ result = new UnsafeRow(1); > /* 041 */ this.holder = new > org.apache.spark.sql.catalyst.expressions.codegen.BufferHolder(result, 32); > /* 042 */ this.rowWriter = new > org.apache.spark.sql.catalyst.expressions.codegen.UnsafeRowWriter(holder, 1); > /* 043 */ this.arrayWriter = new > org.apache.spark.sql.catalyst.expressions.codegen.UnsafeArrayWriter(); > /* 044 */ this.arrayWriter1 = new > org.apache.spark.sql.catalyst.expressions.codegen.UnsafeArrayWriter(); > /* 045 */ this.rowWriter1 = new > org.apache.spark.sql.catalyst.expressions.codegen.UnsafeRowWriter(holder, 1); > /* 046 */ this.arrayWriter2 = new > org.apache.spark.sql.catalyst.expressions.codegen.UnsafeArrayWriter(); > /* 047 */ > /* 048 */ } > /* 049 */ > /* 050 */ public void initialize(int partitionIndex) { > /* 051 */ > /* 052 */ } > /* 053 */ > /* 054 */ > /* 055 */ private void evalIfTrueExpr(InternalRow i) { > /* 056 */ final InternalRow value7 = null; > /* 057 */ isNull12 = true; > /* 058 */ value12 = value7; > /* 059 */ } > /* 060 */ > /* 061 */ > /* 062 */ private void evalIfCondExpr(InternalRow i) { > /* 063 */ > /* 064 */ isNull11 = false; > /* 065 */ value11 = ExternalMapToCatalyst_value_isNull0; > /* 066 */ } > /* 067 */ > /* 068 */ > /* 069 */ private void evalIfFalseExpr(InternalRow i) { > /* 070 */ values = new Object[1]; > /* 071 */ resultIsNull1 = false; > /* 072 */ if (!resultIsNull1) { > /* 073 */ > /* 074 */ boolean isNull10 = true; > /* 075 */ scala.collection.Seq value10 = null; > /* 076 */ if (!ExternalMapToCatalyst_value_isNull0) { > /* 077 */ > /* 078 */ isNull10 = false; > /* 079 */ if (!isNull10) { > /* 080 */ > /* 081 */ Object funcResult1 = null; > /* 082 */ funcResult1 = ExternalMapToCatalyst_value0.values(); > /* 083 */ if (funcResult1 == null) { > /* 084 */ isNull10 = true; > /* 085 */ } else { > /* 086 */ value10 = (scala.collection.Seq) funcResult1; > /* 087 */ } > /* 088 */ > /* 089 */ } > /* 090 */ isNull10 = value10 == null; > /* 091 */ } > /* 092 */ resultIsNull1 = isNull10; > /* 093 */ argValue1 = value10; > /* 094 */ } > /* 095 */ > /* 096 */ > /* 097 */ final ArrayData value9 = resultIsNull1 ? null : new > org.apache.spark.sql.catalyst.util.GenericArrayData(argValue1); > /* 098 */ if (resultIsNull1) { > /* 099 */ values[0] = null; > /* 100 */ } else { > /* 101 */ values[0] = value9; > /* 102 */ } > /* 103 */ final InternalRow value8 = new > org.apache.spark.sql.catalyst.expressions.GenericInternalRow(values); > /* 104 */ this.values = null; > /* 105 */ isNull13 = false; > /* 106 */ value13 = value8; > /* 107 */ } > /* 108 */ > /* 109 */ > /* 110 */ // Scala.Function1 need this > /* 111 */ public java.lang.Object apply(java.lang.Object row) { > /* 112 */ return apply((InternalRow) row); > /* 113 */ } > /* 114 */ > /* 115 */ public UnsafeRow apply(InternalRow i) { > /* 116 */ holder.reset(); > /* 117 */ > /* 118 */ rowWriter.zeroOutNullBytes(); > /* 119 */ > /* 120 */ > /* 121 */ boolean isNull3 = i.isNullAt(0); > /* 122 */ local.ItemProperties value3 = isNull3 ? null : > ((local.ItemProperties)i.get(0, null)); > /* 123 */ > /* 124 */ if (isNull3) { > /* 125 */ throw new RuntimeException(((java.lang.String) > references[0])); > /* 126 */ } > /* 127 */ boolean isNull1 = true; > /* 128 */ scala.collection.immutable.Map value1 = null; > /* 129 */ if (!false) { > /* 130 */ > /* 131 */ isNull1 = false; > /* 132 */ if (!isNull1) { > /* 133 */ > /* 134 */ Object funcResult = null; > /* 135 */ funcResult = value3.properties(); > /* 136 */ if (funcResult == null) { > /* 137 */ isNull1 = true; > /* 138 */ } else { > /* 139 */ value1 = (scala.collection.immutable.Map) funcResult; > /* 140 */ } > /* 141 */ > /* 142 */ } > /* 143 */ isNull1 = value1 == null; > /* 144 */ } > /* 145 */ MapData value = null; > /* 146 */ if (!isNull1) { > /* 147 */ final int length = value1.size(); > /* 148 */ final Object[] convertedKeys = new Object[length]; > /* 149 */ final Object[] convertedValues = new Object[length]; > /* 150 */ int index = 0; > /* 151 */ final scala.collection.Iterator entries = value1.iterator(); > /* 152 */ while(entries.hasNext()) { > /* 153 */ > /* 154 */ final scala.Tuple2 entry = (scala.Tuple2) entries.next(); > /* 155 */ java.lang.String ExternalMapToCatalyst_key0 = > (java.lang.String) entry._1(); > /* 156 */ local.Values ExternalMapToCatalyst_value0 = (local.Values) > entry._2(); > /* 157 */ > /* 158 */ boolean ExternalMapToCatalyst_value_isNull0 = > ExternalMapToCatalyst_value0 == null; > /* 159 */ > /* 160 */ > /* 161 */ resultIsNull = false; > /* 162 */ if (!resultIsNull) { > /* 163 */ > /* 164 */ resultIsNull = false; > /* 165 */ argValue = ExternalMapToCatalyst_key0; > /* 166 */ } > /* 167 */ > /* 168 */ boolean isNull4 = resultIsNull; > /* 169 */ final UTF8String value4 = resultIsNull ? null : > org.apache.spark.unsafe.types.UTF8String.fromString(argValue); > /* 170 */ isNull4 = value4 == null; > /* 171 */ if (isNull4) { > /* 172 */ throw new RuntimeException("Cannot use null as map key!"); > /* 173 */ } else { > /* 174 */ convertedKeys[index] = (UTF8String) value4; > /* 175 */ } > /* 176 */ > /* 177 */ > /* 178 */ evalIfCondExpr(i); > /* 179 */ boolean isNull5 = false; > /* 180 */ InternalRow value5 = null; > /* 181 */ if (!isNull11 && value11) { > /* 182 */ evalIfTrueExpr(i); > /* 183 */ isNull5 = isNull12; > /* 184 */ value5 = value12; > /* 185 */ } else { > /* 186 */ evalIfFalseExpr(i); > /* 187 */ isNull5 = isNull13; > /* 188 */ value5 = value13; > /* 189 */ } > /* 190 */ if (isNull5) { > /* 191 */ convertedValues[index] = null; > /* 192 */ } else { > /* 193 */ convertedValues[index] = (InternalRow) value5; > /* 194 */ } > /* 195 */ > /* 196 */ index++; > /* 197 */ } > /* 198 */ > /* 199 */ value = new > org.apache.spark.sql.catalyst.util.ArrayBasedMapData(new > org.apache.spark.sql.catalyst.util.GenericArrayData(convertedKeys), new > org.apache.spark.sql.catalyst.util.GenericArrayData(convertedValues)); > /* 200 */ } > /* 201 */ if (isNull1) { > /* 202 */ rowWriter.setNullAt(0); > /* 203 */ } else { > /* 204 */ // Remember the current cursor so that we can calculate how > many bytes are > /* 205 */ // written later. > /* 206 */ final int tmpCursor = holder.cursor; > /* 207 */ > /* 208 */ if (value instanceof UnsafeMapData) { > /* 209 */ > /* 210 */ final int sizeInBytes = ((UnsafeMapData) > value).getSizeInBytes(); > /* 211 */ // grow the global buffer before writing data. > /* 212 */ holder.grow(sizeInBytes); > /* 213 */ ((UnsafeMapData) value).writeToMemory(holder.buffer, > holder.cursor); > /* 214 */ holder.cursor += sizeInBytes; > /* 215 */ > /* 216 */ } else { > /* 217 */ final ArrayData keys = value.keyArray(); > /* 218 */ final ArrayData values1 = value.valueArray(); > /* 219 */ > /* 220 */ // preserve 8 bytes to write the key array numBytes later. > /* 221 */ holder.grow(8); > /* 222 */ holder.cursor += 8; > /* 223 */ > /* 224 */ // Remember the current cursor so that we can write > numBytes of key array later. > /* 225 */ final int tmpCursor1 = holder.cursor; > /* 226 */ > /* 227 */ > /* 228 */ if (keys instanceof UnsafeArrayData) { > /* 229 */ > /* 230 */ final int sizeInBytes1 = ((UnsafeArrayData) > keys).getSizeInBytes(); > /* 231 */ // grow the global buffer before writing data. > /* 232 */ holder.grow(sizeInBytes1); > /* 233 */ ((UnsafeArrayData) keys).writeToMemory(holder.buffer, > holder.cursor); > /* 234 */ holder.cursor += sizeInBytes1; > /* 235 */ > /* 236 */ } else { > /* 237 */ final int numElements = keys.numElements(); > /* 238 */ arrayWriter.initialize(holder, numElements, 8); > /* 239 */ > /* 240 */ for (int index1 = 0; index1 < numElements; index1++) { > /* 241 */ if (keys.isNullAt(index1)) { > /* 242 */ arrayWriter.setNull(index1); > /* 243 */ } else { > /* 244 */ final UTF8String element = keys.getUTF8String(index1); > /* 245 */ arrayWriter.write(index1, element); > /* 246 */ } > /* 247 */ } > /* 248 */ } > /* 249 */ > /* 250 */ // Write the numBytes of key array into the first 8 bytes. > /* 251 */ Platform.putLong(holder.buffer, tmpCursor1 - 8, > holder.cursor - tmpCursor1); > /* 252 */ > /* 253 */ > /* 254 */ if (values1 instanceof UnsafeArrayData) { > /* 255 */ > /* 256 */ final int sizeInBytes4 = ((UnsafeArrayData) > values1).getSizeInBytes(); > /* 257 */ // grow the global buffer before writing data. > /* 258 */ holder.grow(sizeInBytes4); > /* 259 */ ((UnsafeArrayData) values1).writeToMemory(holder.buffer, > holder.cursor); > /* 260 */ holder.cursor += sizeInBytes4; > /* 261 */ > /* 262 */ } else { > /* 263 */ final int numElements1 = values1.numElements(); > /* 264 */ arrayWriter1.initialize(holder, numElements1, 8); > /* 265 */ > /* 266 */ for (int index2 = 0; index2 < numElements1; index2++) { > /* 267 */ if (values1.isNullAt(index2)) { > /* 268 */ arrayWriter1.setNull(index2); > /* 269 */ } else { > /* 270 */ final InternalRow element1 = > values1.getStruct(index2, 1); > /* 271 */ > /* 272 */ final int tmpCursor3 = holder.cursor; > /* 273 */ > /* 274 */ if (element1 instanceof UnsafeRow) { > /* 275 */ > /* 276 */ final int sizeInBytes2 = ((UnsafeRow) > element1).getSizeInBytes(); > /* 277 */ // grow the global buffer before writing data. > /* 278 */ holder.grow(sizeInBytes2); > /* 279 */ ((UnsafeRow) element1).writeToMemory(holder.buffer, > holder.cursor); > /* 280 */ holder.cursor += sizeInBytes2; > /* 281 */ > /* 282 */ } else { > /* 283 */ rowWriter1.reset(); > /* 284 */ > /* 285 */ final ArrayData fieldName = element1.getArray(0); > /* 286 */ if (element1.isNullAt(0)) { > /* 287 */ rowWriter1.setNullAt(0); > /* 288 */ } else { > /* 289 */ // Remember the current cursor so that we can > calculate how many bytes are > /* 290 */ // written later. > /* 291 */ final int tmpCursor4 = holder.cursor; > /* 292 */ > /* 293 */ if (fieldName instanceof UnsafeArrayData) { > /* 294 */ > /* 295 */ final int sizeInBytes3 = ((UnsafeArrayData) > fieldName).getSizeInBytes(); > /* 296 */ // grow the global buffer before writing data. > /* 297 */ holder.grow(sizeInBytes3); > /* 298 */ ((UnsafeArrayData) > fieldName).writeToMemory(holder.buffer, holder.cursor); > /* 299 */ holder.cursor += sizeInBytes3; > /* 300 */ > /* 301 */ } else { > /* 302 */ final int numElements2 = > fieldName.numElements(); > /* 303 */ arrayWriter2.initialize(holder, numElements2, > 8); > /* 304 */ > /* 305 */ for (int index3 = 0; index3 < numElements2; > index3++) { > /* 306 */ if (fieldName.isNullAt(index3)) { > /* 307 */ arrayWriter2.setNullDouble(index3); > /* 308 */ } else { > /* 309 */ final double element2 = > fieldName.getDouble(index3); > /* 310 */ arrayWriter2.write(index3, element2); > /* 311 */ } > /* 312 */ } > /* 313 */ } > /* 314 */ > /* 315 */ rowWriter1.setOffsetAndSize(0, tmpCursor4, > holder.cursor - tmpCursor4); > /* 316 */ } > /* 317 */ } > /* 318 */ > /* 319 */ arrayWriter1.setOffsetAndSize(index2, tmpCursor3, > holder.cursor - tmpCursor3); > /* 320 */ > /* 321 */ } > /* 322 */ } > /* 323 */ } > /* 324 */ > /* 325 */ } > /* 326 */ > /* 327 */ rowWriter.setOffsetAndSize(0, tmpCursor, holder.cursor - > tmpCursor); > /* 328 */ } > /* 329 */ result.setTotalSize(holder.totalSize()); > /* 330 */ return result; > /* 331 */ } > /* 332 */ } > at > org.spark_project.guava.util.concurrent.AbstractFuture$Sync.getValue(AbstractFuture.java:306) > at > org.spark_project.guava.util.concurrent.AbstractFuture$Sync.get(AbstractFuture.java:293) > at > org.spark_project.guava.util.concurrent.AbstractFuture.get(AbstractFuture.java:116) > at > org.spark_project.guava.util.concurrent.Uninterruptibles.getUninterruptibly(Uninterruptibles.java:135) > at > org.spark_project.guava.cache.LocalCache$Segment.getAndRecordStats(LocalCache.java:2410) > at > org.spark_project.guava.cache.LocalCache$Segment.loadSync(LocalCache.java:2380) > at > org.spark_project.guava.cache.LocalCache$Segment.lockedGetOrLoad(LocalCache.java:2342) > at > org.spark_project.guava.cache.LocalCache$Segment.get(LocalCache.java:2257) > at org.spark_project.guava.cache.LocalCache.get(LocalCache.java:4000) > at > org.spark_project.guava.cache.LocalCache.getOrLoad(LocalCache.java:4004) > at > org.spark_project.guava.cache.LocalCache$LocalLoadingCache.get(LocalCache.java:4874) > at > org.apache.spark.sql.catalyst.expressions.codegen.CodeGenerator$.compile(CodeGenerator.scala:890) > at > org.apache.spark.sql.catalyst.expressions.codegen.GenerateUnsafeProjection$.create(GenerateUnsafeProjection.scala:405) > at > org.apache.spark.sql.catalyst.expressions.codegen.GenerateUnsafeProjection$.create(GenerateUnsafeProjection.scala:359) > at > org.apache.spark.sql.catalyst.expressions.codegen.GenerateUnsafeProjection$.create(GenerateUnsafeProjection.scala:32) > at > org.apache.spark.sql.catalyst.expressions.codegen.CodeGenerator.generate(CodeGenerator.scala:874) > at > org.apache.spark.sql.catalyst.encoders.ExpressionEncoder.extractProjection$lzycompute(ExpressionEncoder.scala:266) > at > org.apache.spark.sql.catalyst.encoders.ExpressionEncoder.extractProjection(ExpressionEncoder.scala:266) > at > org.apache.spark.sql.catalyst.encoders.ExpressionEncoder.toRow(ExpressionEncoder.scala:290) > ... 17 more > Caused by: java.lang.Exception: failed to compile: > org.codehaus.commons.compiler.CompileException: File 'generated.java', Line > 65, Column 46: Expression "ExternalMapToCatalyst_value_isNull0" is not an > rvalue > /* 001 */ public java.lang.Object generate(Object[] references) { > /* 002 */ return new SpecificUnsafeProjection(references); > /* 003 */ } > /* 004 */ > /* 005 */ class SpecificUnsafeProjection extends > org.apache.spark.sql.catalyst.expressions.UnsafeProjection { > /* 006 */ > /* 007 */ private Object[] references; > /* 008 */ private boolean resultIsNull; > /* 009 */ private java.lang.String argValue; > /* 010 */ private Object[] values; > /* 011 */ private boolean resultIsNull1; > /* 012 */ private scala.collection.Seq argValue1; > /* 013 */ private boolean isNull11; > /* 014 */ private boolean value11; > /* 015 */ private boolean isNull12; > /* 016 */ private InternalRow value12; > /* 017 */ private boolean isNull13; > /* 018 */ private InternalRow value13; > /* 019 */ private UnsafeRow result; > /* 020 */ private > org.apache.spark.sql.catalyst.expressions.codegen.BufferHolder holder; > /* 021 */ private > org.apache.spark.sql.catalyst.expressions.codegen.UnsafeRowWriter rowWriter; > /* 022 */ private > org.apache.spark.sql.catalyst.expressions.codegen.UnsafeArrayWriter > arrayWriter; > /* 023 */ private > org.apache.spark.sql.catalyst.expressions.codegen.UnsafeArrayWriter > arrayWriter1; > /* 024 */ private > org.apache.spark.sql.catalyst.expressions.codegen.UnsafeRowWriter rowWriter1; > /* 025 */ private > org.apache.spark.sql.catalyst.expressions.codegen.UnsafeArrayWriter > arrayWriter2; > /* 026 */ > /* 027 */ public SpecificUnsafeProjection(Object[] references) { > /* 028 */ this.references = references; > /* 029 */ > /* 030 */ > /* 031 */ this.values = null; > /* 032 */ > /* 033 */ > /* 034 */ isNull11 = false; > /* 035 */ value11 = false; > /* 036 */ isNull12 = false; > /* 037 */ value12 = null; > /* 038 */ isNull13 = false; > /* 039 */ value13 = null; > /* 040 */ result = new UnsafeRow(1); > /* 041 */ this.holder = new > org.apache.spark.sql.catalyst.expressions.codegen.BufferHolder(result, 32); > /* 042 */ this.rowWriter = new > org.apache.spark.sql.catalyst.expressions.codegen.UnsafeRowWriter(holder, 1); > /* 043 */ this.arrayWriter = new > org.apache.spark.sql.catalyst.expressions.codegen.UnsafeArrayWriter(); > /* 044 */ this.arrayWriter1 = new > org.apache.spark.sql.catalyst.expressions.codegen.UnsafeArrayWriter(); > /* 045 */ this.rowWriter1 = new > org.apache.spark.sql.catalyst.expressions.codegen.UnsafeRowWriter(holder, 1); > /* 046 */ this.arrayWriter2 = new > org.apache.spark.sql.catalyst.expressions.codegen.UnsafeArrayWriter(); > /* 047 */ > /* 048 */ } > /* 049 */ > /* 050 */ public void initialize(int partitionIndex) { > /* 051 */ > /* 052 */ } > /* 053 */ > /* 054 */ > /* 055 */ private void evalIfTrueExpr(InternalRow i) { > /* 056 */ final InternalRow value7 = null; > /* 057 */ isNull12 = true; > /* 058 */ value12 = value7; > /* 059 */ } > /* 060 */ > /* 061 */ > /* 062 */ private void evalIfCondExpr(InternalRow i) { > /* 063 */ > /* 064 */ isNull11 = false; > /* 065 */ value11 = ExternalMapToCatalyst_value_isNull0; > /* 066 */ } > /* 067 */ > /* 068 */ > /* 069 */ private void evalIfFalseExpr(InternalRow i) { > /* 070 */ values = new Object[1]; > /* 071 */ resultIsNull1 = false; > /* 072 */ if (!resultIsNull1) { > /* 073 */ > /* 074 */ boolean isNull10 = true; > /* 075 */ scala.collection.Seq value10 = null; > /* 076 */ if (!ExternalMapToCatalyst_value_isNull0) { > /* 077 */ > /* 078 */ isNull10 = false; > /* 079 */ if (!isNull10) { > /* 080 */ > /* 081 */ Object funcResult1 = null; > /* 082 */ funcResult1 = ExternalMapToCatalyst_value0.values(); > /* 083 */ if (funcResult1 == null) { > /* 084 */ isNull10 = true; > /* 085 */ } else { > /* 086 */ value10 = (scala.collection.Seq) funcResult1; > /* 087 */ } > /* 088 */ > /* 089 */ } > /* 090 */ isNull10 = value10 == null; > /* 091 */ } > /* 092 */ resultIsNull1 = isNull10; > /* 093 */ argValue1 = value10; > /* 094 */ } > /* 095 */ > /* 096 */ > /* 097 */ final ArrayData value9 = resultIsNull1 ? null : new > org.apache.spark.sql.catalyst.util.GenericArrayData(argValue1); > /* 098 */ if (resultIsNull1) { > /* 099 */ values[0] = null; > /* 100 */ } else { > /* 101 */ values[0] = value9; > /* 102 */ } > /* 103 */ final InternalRow value8 = new > org.apache.spark.sql.catalyst.expressions.GenericInternalRow(values); > /* 104 */ this.values = null; > /* 105 */ isNull13 = false; > /* 106 */ value13 = value8; > /* 107 */ } > /* 108 */ > /* 109 */ > /* 110 */ // Scala.Function1 need this > /* 111 */ public java.lang.Object apply(java.lang.Object row) { > /* 112 */ return apply((InternalRow) row); > /* 113 */ } > /* 114 */ > /* 115 */ public UnsafeRow apply(InternalRow i) { > /* 116 */ holder.reset(); > /* 117 */ > /* 118 */ rowWriter.zeroOutNullBytes(); > /* 119 */ > /* 120 */ > /* 121 */ boolean isNull3 = i.isNullAt(0); > /* 122 */ local.ItemProperties value3 = isNull3 ? null : > ((local.ItemProperties)i.get(0, null)); > /* 123 */ > /* 124 */ if (isNull3) { > /* 125 */ throw new RuntimeException(((java.lang.String) > references[0])); > /* 126 */ } > /* 127 */ boolean isNull1 = true; > /* 128 */ scala.collection.immutable.Map value1 = null; > /* 129 */ if (!false) { > /* 130 */ > /* 131 */ isNull1 = false; > /* 132 */ if (!isNull1) { > /* 133 */ > /* 134 */ Object funcResult = null; > /* 135 */ funcResult = value3.properties(); > /* 136 */ if (funcResult == null) { > /* 137 */ isNull1 = true; > /* 138 */ } else { > /* 139 */ value1 = (scala.collection.immutable.Map) funcResult; > /* 140 */ } > /* 141 */ > /* 142 */ } > /* 143 */ isNull1 = value1 == null; > /* 144 */ } > /* 145 */ MapData value = null; > /* 146 */ if (!isNull1) { > /* 147 */ final int length = value1.size(); > /* 148 */ final Object[] convertedKeys = new Object[length]; > /* 149 */ final Object[] convertedValues = new Object[length]; > /* 150 */ int index = 0; > /* 151 */ final scala.collection.Iterator entries = value1.iterator(); > /* 152 */ while(entries.hasNext()) { > /* 153 */ > /* 154 */ final scala.Tuple2 entry = (scala.Tuple2) entries.next(); > /* 155 */ java.lang.String ExternalMapToCatalyst_key0 = > (java.lang.String) entry._1(); > /* 156 */ local.Values ExternalMapToCatalyst_value0 = (local.Values) > entry._2(); > /* 157 */ > /* 158 */ boolean ExternalMapToCatalyst_value_isNull0 = > ExternalMapToCatalyst_value0 == null; > /* 159 */ > /* 160 */ > /* 161 */ resultIsNull = false; > /* 162 */ if (!resultIsNull) { > /* 163 */ > /* 164 */ resultIsNull = false; > /* 165 */ argValue = ExternalMapToCatalyst_key0; > /* 166 */ } > /* 167 */ > /* 168 */ boolean isNull4 = resultIsNull; > /* 169 */ final UTF8String value4 = resultIsNull ? null : > org.apache.spark.unsafe.types.UTF8String.fromString(argValue); > /* 170 */ isNull4 = value4 == null; > /* 171 */ if (isNull4) { > /* 172 */ throw new RuntimeException("Cannot use null as map key!"); > /* 173 */ } else { > /* 174 */ convertedKeys[index] = (UTF8String) value4; > /* 175 */ } > /* 176 */ > /* 177 */ > /* 178 */ evalIfCondExpr(i); > /* 179 */ boolean isNull5 = false; > /* 180 */ InternalRow value5 = null; > /* 181 */ if (!isNull11 && value11) { > /* 182 */ evalIfTrueExpr(i); > /* 183 */ isNull5 = isNull12; > /* 184 */ value5 = value12; > /* 185 */ } else { > /* 186 */ evalIfFalseExpr(i); > /* 187 */ isNull5 = isNull13; > /* 188 */ value5 = value13; > /* 189 */ } > /* 190 */ if (isNull5) { > /* 191 */ convertedValues[index] = null; > /* 192 */ } else { > /* 193 */ convertedValues[index] = (InternalRow) value5; > /* 194 */ } > /* 195 */ > /* 196 */ index++; > /* 197 */ } > /* 198 */ > /* 199 */ value = new > org.apache.spark.sql.catalyst.util.ArrayBasedMapData(new > org.apache.spark.sql.catalyst.util.GenericArrayData(convertedKeys), new > org.apache.spark.sql.catalyst.util.GenericArrayData(convertedValues)); > /* 200 */ } > /* 201 */ if (isNull1) { > /* 202 */ rowWriter.setNullAt(0); > /* 203 */ } else { > /* 204 */ // Remember the current cursor so that we can calculate how > many bytes are > /* 205 */ // written later. > /* 206 */ final int tmpCursor = holder.cursor; > /* 207 */ > /* 208 */ if (value instanceof UnsafeMapData) { > /* 209 */ > /* 210 */ final int sizeInBytes = ((UnsafeMapData) > value).getSizeInBytes(); > /* 211 */ // grow the global buffer before writing data. > /* 212 */ holder.grow(sizeInBytes); > /* 213 */ ((UnsafeMapData) value).writeToMemory(holder.buffer, > holder.cursor); > /* 214 */ holder.cursor += sizeInBytes; > /* 215 */ > /* 216 */ } else { > /* 217 */ final ArrayData keys = value.keyArray(); > /* 218 */ final ArrayData values1 = value.valueArray(); > /* 219 */ > /* 220 */ // preserve 8 bytes to write the key array numBytes later. > /* 221 */ holder.grow(8); > /* 222 */ holder.cursor += 8; > /* 223 */ > /* 224 */ // Remember the current cursor so that we can write > numBytes of key array later. > /* 225 */ final int tmpCursor1 = holder.cursor; > /* 226 */ > /* 227 */ > /* 228 */ if (keys instanceof UnsafeArrayData) { > /* 229 */ > /* 230 */ final int sizeInBytes1 = ((UnsafeArrayData) > keys).getSizeInBytes(); > /* 231 */ // grow the global buffer before writing data. > /* 232 */ holder.grow(sizeInBytes1); > /* 233 */ ((UnsafeArrayData) keys).writeToMemory(holder.buffer, > holder.cursor); > /* 234 */ holder.cursor += sizeInBytes1; > /* 235 */ > /* 236 */ } else { > /* 237 */ final int numElements = keys.numElements(); > /* 238 */ arrayWriter.initialize(holder, numElements, 8); > /* 239 */ > /* 240 */ for (int index1 = 0; index1 < numElements; index1++) { > /* 241 */ if (keys.isNullAt(index1)) { > /* 242 */ arrayWriter.setNull(index1); > /* 243 */ } else { > /* 244 */ final UTF8String element = keys.getUTF8String(index1); > /* 245 */ arrayWriter.write(index1, element); > /* 246 */ } > /* 247 */ } > /* 248 */ } > /* 249 */ > /* 250 */ // Write the numBytes of key array into the first 8 bytes. > /* 251 */ Platform.putLong(holder.buffer, tmpCursor1 - 8, > holder.cursor - tmpCursor1); > /* 252 */ > /* 253 */ > /* 254 */ if (values1 instanceof UnsafeArrayData) { > /* 255 */ > /* 256 */ final int sizeInBytes4 = ((UnsafeArrayData) > values1).getSizeInBytes(); > /* 257 */ // grow the global buffer before writing data. > /* 258 */ holder.grow(sizeInBytes4); > /* 259 */ ((UnsafeArrayData) values1).writeToMemory(holder.buffer, > holder.cursor); > /* 260 */ holder.cursor += sizeInBytes4; > /* 261 */ > /* 262 */ } else { > /* 263 */ final int numElements1 = values1.numElements(); > /* 264 */ arrayWriter1.initialize(holder, numElements1, 8); > /* 265 */ > /* 266 */ for (int index2 = 0; index2 < numElements1; index2++) { > /* 267 */ if (values1.isNullAt(index2)) { > /* 268 */ arrayWriter1.setNull(index2); > /* 269 */ } else { > /* 270 */ final InternalRow element1 = > values1.getStruct(index2, 1); > /* 271 */ > /* 272 */ final int tmpCursor3 = holder.cursor; > /* 273 */ > /* 274 */ if (element1 instanceof UnsafeRow) { > /* 275 */ > /* 276 */ final int sizeInBytes2 = ((UnsafeRow) > element1).getSizeInBytes(); > /* 277 */ // grow the global buffer before writing data. > /* 278 */ holder.grow(sizeInBytes2); > /* 279 */ ((UnsafeRow) element1).writeToMemory(holder.buffer, > holder.cursor); > /* 280 */ holder.cursor += sizeInBytes2; > /* 281 */ > /* 282 */ } else { > /* 283 */ rowWriter1.reset(); > /* 284 */ > /* 285 */ final ArrayData fieldName = element1.getArray(0); > /* 286 */ if (element1.isNullAt(0)) { > /* 287 */ rowWriter1.setNullAt(0); > /* 288 */ } else { > /* 289 */ // Remember the current cursor so that we can > calculate how many bytes are > /* 290 */ // written later. > /* 291 */ final int tmpCursor4 = holder.cursor; > /* 292 */ > /* 293 */ if (fieldName instanceof UnsafeArrayData) { > /* 294 */ > /* 295 */ final int sizeInBytes3 = ((UnsafeArrayData) > fieldName).getSizeInBytes(); > /* 296 */ // grow the global buffer before writing data. > /* 297 */ holder.grow(sizeInBytes3); > /* 298 */ ((UnsafeArrayData) > fieldName).writeToMemory(holder.buffer, holder.cursor); > /* 299 */ holder.cursor += sizeInBytes3; > /* 300 */ > /* 301 */ } else { > /* 302 */ final int numElements2 = > fieldName.numElements(); > /* 303 */ arrayWriter2.initialize(holder, numElements2, > 8); > /* 304 */ > /* 305 */ for (int index3 = 0; index3 < numElements2; > index3++) { > /* 306 */ if (fieldName.isNullAt(index3)) { > /* 307 */ arrayWriter2.setNullDouble(index3); > /* 308 */ } else { > /* 309 */ final double element2 = > fieldName.getDouble(index3); > /* 310 */ arrayWriter2.write(index3, element2); > /* 311 */ } > /* 312 */ } > /* 313 */ } > /* 314 */ > /* 315 */ rowWriter1.setOffsetAndSize(0, tmpCursor4, > holder.cursor - tmpCursor4); > /* 316 */ } > /* 317 */ } > /* 318 */ > /* 319 */ arrayWriter1.setOffsetAndSize(index2, tmpCursor3, > holder.cursor - tmpCursor3); > /* 320 */ > /* 321 */ } > /* 322 */ } > /* 323 */ } > /* 324 */ > /* 325 */ } > /* 326 */ > /* 327 */ rowWriter.setOffsetAndSize(0, tmpCursor, holder.cursor - > tmpCursor); > /* 328 */ } > /* 329 */ result.setTotalSize(holder.totalSize()); > /* 330 */ return result; > /* 331 */ } > /* 332 */ } > at > org.apache.spark.sql.catalyst.expressions.codegen.CodeGenerator$.org$apache$spark$sql$catalyst$expressions$codegen$CodeGenerator$$doCompile(CodeGenerator.scala:941) > at > org.apache.spark.sql.catalyst.expressions.codegen.CodeGenerator$$anon$1.load(CodeGenerator.scala:998) > at > org.apache.spark.sql.catalyst.expressions.codegen.CodeGenerator$$anon$1.load(CodeGenerator.scala:995) > at > org.spark_project.guava.cache.LocalCache$LoadingValueReference.loadFuture(LocalCache.java:3599) > at > org.spark_project.guava.cache.LocalCache$Segment.loadSync(LocalCache.java:2379) > ... 30 more > Caused by: org.codehaus.commons.compiler.CompileException: File > 'generated.java', Line 65, Column 46: Expression > "ExternalMapToCatalyst_value_isNull0" is not an rvalue > at > org.codehaus.janino.UnitCompiler.compileError(UnitCompiler.java:11004) > at > org.codehaus.janino.UnitCompiler.toRvalueOrCompileException(UnitCompiler.java:6639) > at > org.codehaus.janino.UnitCompiler.getConstantValue2(UnitCompiler.java:5001) > at org.codehaus.janino.UnitCompiler.access$10500(UnitCompiler.java:206) > at > org.codehaus.janino.UnitCompiler$13.visitAmbiguousName(UnitCompiler.java:4984) > at org.codehaus.janino.Java$AmbiguousName.accept(Java.java:3633) > at org.codehaus.janino.Java$Lvalue.accept(Java.java:3563) > at > org.codehaus.janino.UnitCompiler.getConstantValue(UnitCompiler.java:4956) > at > org.codehaus.janino.UnitCompiler.compileGetValue(UnitCompiler.java:4925) > at org.codehaus.janino.UnitCompiler.compile2(UnitCompiler.java:3189) > at org.codehaus.janino.UnitCompiler.access$5100(UnitCompiler.java:206) > at > org.codehaus.janino.UnitCompiler$9.visitAssignment(UnitCompiler.java:3143) > at > org.codehaus.janino.UnitCompiler$9.visitAssignment(UnitCompiler.java:3139) > at org.codehaus.janino.Java$Assignment.accept(Java.java:3847) > at org.codehaus.janino.UnitCompiler.compile(UnitCompiler.java:3139) > at org.codehaus.janino.UnitCompiler.compile2(UnitCompiler.java:2112) > at org.codehaus.janino.UnitCompiler.access$1700(UnitCompiler.java:206) > at > org.codehaus.janino.UnitCompiler$6.visitExpressionStatement(UnitCompiler.java:1377) > at > org.codehaus.janino.UnitCompiler$6.visitExpressionStatement(UnitCompiler.java:1370) > at org.codehaus.janino.Java$ExpressionStatement.accept(Java.java:2558) > at org.codehaus.janino.UnitCompiler.compile(UnitCompiler.java:1370) > at > org.codehaus.janino.UnitCompiler.compileStatements(UnitCompiler.java:1450) > at org.codehaus.janino.UnitCompiler.compile(UnitCompiler.java:2811) > at > org.codehaus.janino.UnitCompiler.compileDeclaredMethods(UnitCompiler.java:1262) > at > org.codehaus.janino.UnitCompiler.compileDeclaredMethods(UnitCompiler.java:1234) > at org.codehaus.janino.UnitCompiler.compile2(UnitCompiler.java:538) > at org.codehaus.janino.UnitCompiler.compile2(UnitCompiler.java:890) > at org.codehaus.janino.UnitCompiler.compile2(UnitCompiler.java:894) > at org.codehaus.janino.UnitCompiler.access$600(UnitCompiler.java:206) > at > org.codehaus.janino.UnitCompiler$2.visitMemberClassDeclaration(UnitCompiler.java:377) > at > org.codehaus.janino.UnitCompiler$2.visitMemberClassDeclaration(UnitCompiler.java:369) > at > org.codehaus.janino.Java$MemberClassDeclaration.accept(Java.java:1128) > at org.codehaus.janino.UnitCompiler.compile(UnitCompiler.java:369) > at > org.codehaus.janino.UnitCompiler.compileDeclaredMemberTypes(UnitCompiler.java:1209) > at org.codehaus.janino.UnitCompiler.compile2(UnitCompiler.java:564) > at org.codehaus.janino.UnitCompiler.compile2(UnitCompiler.java:420) > at org.codehaus.janino.UnitCompiler.access$400(UnitCompiler.java:206) > at > org.codehaus.janino.UnitCompiler$2.visitPackageMemberClassDeclaration(UnitCompiler.java:374) > at > org.codehaus.janino.UnitCompiler$2.visitPackageMemberClassDeclaration(UnitCompiler.java:369) > at > org.codehaus.janino.Java$AbstractPackageMemberClassDeclaration.accept(Java.java:1309) > at org.codehaus.janino.UnitCompiler.compile(UnitCompiler.java:369) > at org.codehaus.janino.UnitCompiler.compileUnit(UnitCompiler.java:345) > at > org.codehaus.janino.SimpleCompiler.compileToClassLoader(SimpleCompiler.java:396) > at > org.codehaus.janino.ClassBodyEvaluator.compileToClass(ClassBodyEvaluator.java:311) > at > org.codehaus.janino.ClassBodyEvaluator.cook(ClassBodyEvaluator.java:229) > at org.codehaus.janino.SimpleCompiler.cook(SimpleCompiler.java:196) > at org.codehaus.commons.compiler.Cookable.cook(Cookable.java:91) > at > org.apache.spark.sql.catalyst.expressions.codegen.CodeGenerator$.org$apache$spark$sql$catalyst$expressions$codegen$CodeGenerator$$doCompile(CodeGenerator.scala:935) > ... 34 more > 17/07/12 21:59:35 INFO SparkContext: Invoking stop() from shutdown hook > 17/07/12 21:59:35 INFO SparkUI: Stopped Spark web UI at > http://192.168.0.101:4040 > 17/07/12 21:59:35 INFO MapOutputTrackerMasterEndpoint: > MapOutputTrackerMasterEndpoint stopped! > 17/07/12 21:59:35 INFO MemoryStore: MemoryStore cleared > 17/07/12 21:59:35 INFO BlockManager: BlockManager stopped > 17/07/12 21:59:35 INFO BlockManagerMaster: BlockManagerMaster stopped > 17/07/12 21:59:35 INFO > OutputCommitCoordinator$OutputCommitCoordinatorEndpoint: > OutputCommitCoordinator stopped! > 17/07/12 21:59:35 INFO SparkContext: Successfully stopped SparkContext > 17/07/12 21:59:35 INFO ShutdownHookManager: Shutdown hook called > 17/07/12 21:59:35 INFO ShutdownHookManager: Deleting directory > /private/var/folders/fw/lg6m2xps12bfwd7x0hj3rpzm0000gn/T/spark-5587bc9b-fc45-4b9a-a7b4-6d5cfec48eaf > Process finished with exit code 1 -- This message was sent by Atlassian JIRA (v6.4.14#64029) --------------------------------------------------------------------- To unsubscribe, e-mail: issues-unsubscr...@spark.apache.org For additional commands, e-mail: issues-h...@spark.apache.org