[ https://issues.apache.org/jira/browse/SPARK-18125?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel&focusedCommentId=15613497#comment-15613497 ]
Herman van Hovell commented on SPARK-18125: ------------------------------------------- Could one of you provide a reproducible example. > Spark generated code causes CompileException when groupByKey, reduceGroups > and map(_._2) are used > ------------------------------------------------------------------------------------------------- > > Key: SPARK-18125 > URL: https://issues.apache.org/jira/browse/SPARK-18125 > Project: Spark > Issue Type: Bug > Affects Versions: 2.0.1 > Reporter: Ray Qiu > Priority: Critical > > Code logic looks like this: > {noformat} > .groupByKey > .reduceGroups > .map(_._2) > {noformat} > Works fine with 2.0.0. > 2.0.1 error Message: > {noformat} > Caused by: java.util.concurrent.ExecutionException: java.lang.Exception: > failed to compile: org.codehaus.commons.compiler.CompileException: File > 'generated.java', Line 206, Column 123: Unknown variable or type "value4" > /* 001 */ public java.lang.Object generate(Object[] references) { > /* 002 */ return new SpecificMutableProjection(references); > /* 003 */ } > /* 004 */ > /* 005 */ class SpecificMutableProjection extends > org.apache.spark.sql.catalyst.expressions.codegen.BaseMutableProjection { > /* 006 */ > /* 007 */ private Object[] references; > /* 008 */ private MutableRow mutableRow; > /* 009 */ private Object[] values; > /* 010 */ private java.lang.String errMsg; > /* 011 */ private java.lang.String errMsg1; > /* 012 */ private boolean MapObjects_loopIsNull1; > /* 013 */ private io.mistnet.analytics.lib.ConnLog MapObjects_loopValue0; > /* 014 */ private java.lang.String errMsg2; > /* 015 */ private Object[] values1; > /* 016 */ private boolean MapObjects_loopIsNull3; > /* 017 */ private java.lang.String MapObjects_loopValue2; > /* 018 */ private boolean isNull_0; > /* 019 */ private boolean value_0; > /* 020 */ private boolean isNull_1; > /* 021 */ private InternalRow value_1; > /* 022 */ > /* 023 */ private void apply_4(InternalRow i) { > /* 024 */ > /* 025 */ boolean isNull52 = MapObjects_loopIsNull1; > /* 026 */ final double value52 = isNull52 ? -1.0 : > MapObjects_loopValue0.ts(); > /* 027 */ if (isNull52) { > /* 028 */ values1[8] = null; > /* 029 */ } else { > /* 030 */ values1[8] = value52; > /* 031 */ } > /* 032 */ boolean isNull54 = MapObjects_loopIsNull1; > /* 033 */ final java.lang.String value54 = isNull54 ? null : > (java.lang.String) MapObjects_loopValue0.uid(); > /* 034 */ isNull54 = value54 == null; > /* 035 */ boolean isNull53 = isNull54; > /* 036 */ final UTF8String value53 = isNull53 ? null : > org.apache.spark.unsafe.types.UTF8String.fromString(value54); > /* 037 */ isNull53 = value53 == null; > /* 038 */ if (isNull53) { > /* 039 */ values1[9] = null; > /* 040 */ } else { > /* 041 */ values1[9] = value53; > /* 042 */ } > /* 043 */ boolean isNull56 = MapObjects_loopIsNull1; > /* 044 */ final java.lang.String value56 = isNull56 ? null : > (java.lang.String) MapObjects_loopValue0.src(); > /* 045 */ isNull56 = value56 == null; > /* 046 */ boolean isNull55 = isNull56; > /* 047 */ final UTF8String value55 = isNull55 ? null : > org.apache.spark.unsafe.types.UTF8String.fromString(value56); > /* 048 */ isNull55 = value55 == null; > /* 049 */ if (isNull55) { > /* 050 */ values1[10] = null; > /* 051 */ } else { > /* 052 */ values1[10] = value55; > /* 053 */ } > /* 054 */ } > /* 055 */ > /* 056 */ > /* 057 */ private void apply_7(InternalRow i) { > /* 058 */ > /* 059 */ boolean isNull69 = MapObjects_loopIsNull1; > /* 060 */ final scala.Option value69 = isNull69 ? null : (scala.Option) > MapObjects_loopValue0.orig_bytes(); > /* 061 */ isNull69 = value69 == null; > /* 062 */ > /* 063 */ final boolean isNull68 = isNull69 || value69.isEmpty(); > /* 064 */ long value68 = isNull68 ? > /* 065 */ -1L : (Long) value69.get(); > /* 066 */ if (isNull68) { > /* 067 */ values1[17] = null; > /* 068 */ } else { > /* 069 */ values1[17] = value68; > /* 070 */ } > /* 071 */ boolean isNull71 = MapObjects_loopIsNull1; > /* 072 */ final scala.Option value71 = isNull71 ? null : (scala.Option) > MapObjects_loopValue0.resp_bytes(); > /* 073 */ isNull71 = value71 == null; > /* 074 */ > /* 075 */ final boolean isNull70 = isNull71 || value71.isEmpty(); > /* 076 */ long value70 = isNull70 ? > /* 077 */ -1L : (Long) value71.get(); > /* 078 */ if (isNull70) { > /* 079 */ values1[18] = null; > /* 080 */ } else { > /* 081 */ values1[18] = value70; > /* 082 */ } > /* 083 */ boolean isNull74 = MapObjects_loopIsNull1; > /* 084 */ final scala.Option value74 = isNull74 ? null : (scala.Option) > MapObjects_loopValue0.conn_state(); > /* 085 */ isNull74 = value74 == null; > /* 086 */ > /* 087 */ final boolean isNull73 = isNull74 || value74.isEmpty(); > /* 088 */ java.lang.String value73 = isNull73 ? > /* 089 */ null : (java.lang.String) value74.get(); > /* 090 */ boolean isNull72 = isNull73; > /* 091 */ final UTF8String value72 = isNull72 ? null : > org.apache.spark.unsafe.types.UTF8String.fromString(value73); > /* 092 */ isNull72 = value72 == null; > /* 093 */ if (isNull72) { > /* 094 */ values1[19] = null; > /* 095 */ } else { > /* 096 */ values1[19] = value72; > /* 097 */ } > /* 098 */ } > /* 099 */ > /* 100 */ > /* 101 */ private void apply_1(InternalRow i) { > /* 102 */ > /* 103 */ boolean isNull37 = MapObjects_loopIsNull1; > /* 104 */ final scala.Option value37 = isNull37 ? null : (scala.Option) > MapObjects_loopValue0.sensor_name(); > /* 105 */ isNull37 = value37 == null; > /* 106 */ > /* 107 */ final boolean isNull36 = isNull37 || value37.isEmpty(); > /* 108 */ java.lang.String value36 = isNull36 ? > /* 109 */ null : (java.lang.String) value37.get(); > /* 110 */ boolean isNull35 = isNull36; > /* 111 */ final UTF8String value35 = isNull35 ? null : > org.apache.spark.unsafe.types.UTF8String.fromString(value36); > /* 112 */ isNull35 = value35 == null; > /* 113 */ if (isNull35) { > /* 114 */ values1[2] = null; > /* 115 */ } else { > /* 116 */ values1[2] = value35; > /* 117 */ } > /* 118 */ boolean isNull40 = MapObjects_loopIsNull1; > /* 119 */ final scala.Option value40 = isNull40 ? null : (scala.Option) > MapObjects_loopValue0.ioa_uuid(); > /* 120 */ isNull40 = value40 == null; > /* 121 */ > /* 122 */ final boolean isNull39 = isNull40 || value40.isEmpty(); > /* 123 */ java.lang.String value39 = isNull39 ? > /* 124 */ null : (java.lang.String) value40.get(); > /* 125 */ boolean isNull38 = isNull39; > /* 126 */ final UTF8String value38 = isNull38 ? null : > org.apache.spark.unsafe.types.UTF8String.fromString(value39); > /* 127 */ isNull38 = value38 == null; > /* 128 */ if (isNull38) { > /* 129 */ values1[3] = null; > /* 130 */ } else { > /* 131 */ values1[3] = value38; > /* 132 */ } > /* 133 */ } > /* 134 */ > /* 135 */ > /* 136 */ private void apply_12(InternalRow i) { > /* 137 */ > /* 138 */ boolean isNull98 = MapObjects_loopIsNull1; > /* 139 */ final scala.Option value98 = isNull98 ? null : (scala.Option) > MapObjects_loopValue0.cc(); > /* 140 */ isNull98 = value98 == null; > /* 141 */ > /* 142 */ final boolean isNull97 = isNull98 || value98.isEmpty(); > /* 143 */ java.lang.String value97 = isNull97 ? > /* 144 */ null : (java.lang.String) value98.get(); > /* 145 */ boolean isNull96 = isNull97; > /* 146 */ final UTF8String value96 = isNull96 ? null : > org.apache.spark.unsafe.types.UTF8String.fromString(value97); > /* 147 */ isNull96 = value96 == null; > /* 148 */ if (isNull96) { > /* 149 */ values1[29] = null; > /* 150 */ } else { > /* 151 */ values1[29] = value96; > /* 152 */ } > /* 153 */ boolean isNull101 = MapObjects_loopIsNull1; > /* 154 */ final scala.Option value101 = isNull101 ? null : (scala.Option) > MapObjects_loopValue0.location(); > /* 155 */ isNull101 = value101 == null; > /* 156 */ > /* 157 */ final boolean isNull100 = isNull101 || value101.isEmpty(); > /* 158 */ java.lang.String value100 = isNull100 ? > /* 159 */ null : (java.lang.String) value101.get(); > /* 160 */ boolean isNull99 = isNull100; > /* 161 */ final UTF8String value99 = isNull99 ? null : > org.apache.spark.unsafe.types.UTF8String.fromString(value100); > /* 162 */ isNull99 = value99 == null; > /* 163 */ if (isNull99) { > /* 164 */ values1[30] = null; > /* 165 */ } else { > /* 166 */ values1[30] = value99; > /* 167 */ } > /* 168 */ } > /* 169 */ > /* 170 */ > /* 171 */ private void apply_9(InternalRow i) { > /* 172 */ > /* 173 */ boolean isNull83 = MapObjects_loopIsNull1; > /* 174 */ final scala.Option value83 = isNull83 ? null : (scala.Option) > MapObjects_loopValue0.history(); > /* 175 */ isNull83 = value83 == null; > /* 176 */ > /* 177 */ final boolean isNull82 = isNull83 || value83.isEmpty(); > /* 178 */ java.lang.String value82 = isNull82 ? > /* 179 */ null : (java.lang.String) value83.get(); > /* 180 */ boolean isNull81 = isNull82; > /* 181 */ final UTF8String value81 = isNull81 ? null : > org.apache.spark.unsafe.types.UTF8String.fromString(value82); > /* 182 */ isNull81 = value81 == null; > /* 183 */ if (isNull81) { > /* 184 */ values1[23] = null; > /* 185 */ } else { > /* 186 */ values1[23] = value81; > /* 187 */ } > /* 188 */ boolean isNull85 = MapObjects_loopIsNull1; > /* 189 */ final scala.Option value85 = isNull85 ? null : (scala.Option) > MapObjects_loopValue0.orig_pkts(); > /* 190 */ isNull85 = value85 == null; > /* 191 */ > /* 192 */ final boolean isNull84 = isNull85 || value85.isEmpty(); > /* 193 */ long value84 = isNull84 ? > /* 194 */ -1L : (Long) value85.get(); > /* 195 */ if (isNull84) { > /* 196 */ values1[24] = null; > /* 197 */ } else { > /* 198 */ values1[24] = value84; > /* 199 */ } > /* 200 */ } > /* 201 */ > /* 202 */ > /* 203 */ private void apply1_1(InternalRow i) { > /* 204 */ > /* 205 */ boolean isNull25 = false; > /* 206 */ final io.mistnet.analytics.scan.SrcDstGrouped value25 = > isNull25 ? null : (io.mistnet.analytics.scan.SrcDstGrouped) value4._2(); > /* 207 */ isNull25 = value25 == null; > /* 208 */ > /* 209 */ if (isNull25) { > /* 210 */ throw new RuntimeException(errMsg2); > /* 211 */ } > /* 212 */ > /* 213 */ boolean isNull23 = false; > /* 214 */ final scala.collection.Seq value23 = isNull23 ? null : > (scala.collection.Seq) value25.cs(); > /* 215 */ isNull23 = value23 == null; > /* 216 */ ArrayData value22 = null; > /* 217 */ > /* 218 */ if (!isNull23) { > /* 219 */ > /* 220 */ InternalRow[] convertedArray1 = null; > /* 221 */ int dataLength1 = value23.size(); > /* 222 */ convertedArray1 = new InternalRow[dataLength1]; > /* 223 */ > /* 224 */ int loopIndex1 = 0; > /* 225 */ while (loopIndex1 < dataLength1) { > /* 226 */ MapObjects_loopValue0 = (io.mistnet.analytics.lib.ConnLog) > (value23.apply(loopIndex1)); > /* 227 */ MapObjects_loopIsNull1 = MapObjects_loopValue0 == null; > /* 228 */ > /* 229 */ > /* 230 */ boolean isNull26 = false; > /* 231 */ InternalRow value26 = null; > /* 232 */ if (!false && MapObjects_loopIsNull1) { > /* 233 */ > /* 234 */ final InternalRow value28 = null; > /* 235 */ isNull26 = true; > /* 236 */ value26 = value28; > /* 237 */ } else { > /* 238 */ > /* 239 */ boolean isNull29 = false; > /* 240 */ values1 = new Object[31];apply_0(i); > /* 241 */ apply_1(i); > /* 242 */ apply_2(i); > /* 243 */ apply_3(i); > /* 244 */ apply_4(i); > /* 245 */ apply_5(i); > /* 246 */ apply_6(i); > /* 247 */ apply_7(i); > /* 248 */ apply_8(i); > /* 249 */ apply_9(i); > /* 250 */ apply_10(i); > /* 251 */ apply_11(i); > /* 252 */ apply_12(i); > /* 253 */ final InternalRow value29 = new > org.apache.spark.sql.catalyst.expressions.GenericInternalRow(values1); > /* 254 */ this.values1 = null; > /* 255 */ isNull26 = isNull29; > /* 256 */ value26 = value29; > /* 257 */ } > /* 258 */ if (isNull26) { > /* 259 */ convertedArray1[loopIndex1] = null; > /* 260 */ } else { > /* 261 */ convertedArray1[loopIndex1] = value26 instanceof > UnsafeRow? value26.copy() : value26; > /* 262 */ } > /* 263 */ > /* 264 */ loopIndex1 += 1; > /* 265 */ } > /* 266 */ > /* 267 */ value22 = new > org.apache.spark.sql.catalyst.util.GenericArrayData(convertedArray1); > /* 268 */ } > /* 269 */ if (isNull23) { > /* 270 */ values[2] = null; > /* 271 */ } else { > /* 272 */ values[2] = value22; > /* 273 */ } > /* 274 */ } > /* 275 */ > /* 276 */ > /* 277 */ private void apply_3(InternalRow i) { > /* 278 */ > /* 279 */ boolean isNull49 = MapObjects_loopIsNull1; > /* 280 */ final scala.Option value49 = isNull49 ? null : (scala.Option) > MapObjects_loopValue0.date(); > /* 281 */ isNull49 = value49 == null; > /* 282 */ > /* 283 */ final boolean isNull48 = isNull49 || value49.isEmpty(); > /* 284 */ java.lang.String value48 = isNull48 ? > /* 285 */ null : (java.lang.String) value49.get(); > /* 286 */ boolean isNull47 = isNull48; > /* 287 */ final UTF8String value47 = isNull47 ? null : > org.apache.spark.unsafe.types.UTF8String.fromString(value48); > /* 288 */ isNull47 = value47 == null; > /* 289 */ if (isNull47) { > /* 290 */ values1[6] = null; > /* 291 */ } else { > /* 292 */ values1[6] = value47; > /* 293 */ } > /* 294 */ boolean isNull51 = MapObjects_loopIsNull1; > /* 295 */ final scala.Option value51 = isNull51 ? null : (scala.Option) > MapObjects_loopValue0.hour(); > /* 296 */ isNull51 = value51 == null; > /* 297 */ > /* 298 */ final boolean isNull50 = isNull51 || value51.isEmpty(); > /* 299 */ int value50 = isNull50 ? > /* 300 */ -1 : (Integer) value51.get(); > /* 301 */ if (isNull50) { > /* 302 */ values1[7] = null; > /* 303 */ } else { > /* 304 */ values1[7] = value50; > /* 305 */ } > /* 306 */ } > /* 307 */ > /* 308 */ > /* 309 */ private void apply_6(InternalRow i) { > /* 310 */ > /* 311 */ boolean isNull65 = MapObjects_loopIsNull1; > /* 312 */ final scala.Option value65 = isNull65 ? null : (scala.Option) > MapObjects_loopValue0.service(); > /* 313 */ isNull65 = value65 == null; > /* 314 */ > /* 315 */ final boolean isNull64 = isNull65 || value65.isEmpty(); > /* 316 */ java.lang.String value64 = isNull64 ? > /* 317 */ null : (java.lang.String) value65.get(); > /* 318 */ boolean isNull63 = isNull64; > /* 319 */ final UTF8String value63 = isNull63 ? null : > org.apache.spark.unsafe.types.UTF8String.fromString(value64); > /* 320 */ isNull63 = value63 == null; > /* 321 */ if (isNull63) { > /* 322 */ values1[15] = null; > /* 323 */ } else { > /* 324 */ values1[15] = value63; > /* 325 */ } > /* 326 */ boolean isNull67 = MapObjects_loopIsNull1; > /* 327 */ final scala.Option value67 = isNull67 ? null : (scala.Option) > MapObjects_loopValue0.duration(); > /* 328 */ isNull67 = value67 == null; > /* 329 */ > /* 330 */ final boolean isNull66 = isNull67 || value67.isEmpty(); > /* 331 */ double value66 = isNull66 ? > /* 332 */ -1.0 : (Double) value67.get(); > /* 333 */ if (isNull66) { > /* 334 */ values1[16] = null; > /* 335 */ } else { > /* 336 */ values1[16] = value66; > /* 337 */ } > /* 338 */ } > /* 339 */ > /* 340 */ > /* 341 */ private void apply_0(InternalRow i) { > /* 342 */ > /* 343 */ boolean isNull32 = MapObjects_loopIsNull1; > /* 344 */ final scala.Option value32 = isNull32 ? null : (scala.Option) > MapObjects_loopValue0.log_type(); > /* 345 */ isNull32 = value32 == null; > /* 346 */ > /* 347 */ final boolean isNull31 = isNull32 || value32.isEmpty(); > /* 348 */ java.lang.String value31 = isNull31 ? > /* 349 */ null : (java.lang.String) value32.get(); > /* 350 */ boolean isNull30 = isNull31; > /* 351 */ final UTF8String value30 = isNull30 ? null : > org.apache.spark.unsafe.types.UTF8String.fromString(value31); > /* 352 */ isNull30 = value30 == null; > /* 353 */ if (isNull30) { > /* 354 */ values1[0] = null; > /* 355 */ } else { > /* 356 */ values1[0] = value30; > /* 357 */ } > /* 358 */ boolean isNull34 = MapObjects_loopIsNull1; > /* 359 */ final scala.Option value34 = isNull34 ? null : (scala.Option) > MapObjects_loopValue0.timestamp(); > /* 360 */ isNull34 = value34 == null; > /* 361 */ > /* 362 */ final boolean isNull33 = isNull34 || value34.isEmpty(); > /* 363 */ long value33 = isNull33 ? > /* 364 */ -1L : (Long) value34.get(); > /* 365 */ if (isNull33) { > /* 366 */ values1[1] = null; > /* 367 */ } else { > /* 368 */ values1[1] = value33; > /* 369 */ } > /* 370 */ } > /* 371 */ > /* 372 */ > /* 373 */ private void apply_11(InternalRow i) { > /* 374 */ > /* 375 */ boolean isNull94 = MapObjects_loopIsNull1; > /* 376 */ final scala.Option value94 = isNull94 ? null : (scala.Option) > MapObjects_loopValue0.tunnel_parents(); > /* 377 */ isNull94 = value94 == null; > /* 378 */ > /* 379 */ final boolean isNull93 = isNull94 || value94.isEmpty(); > /* 380 */ scala.collection.Seq value93 = isNull93 ? > /* 381 */ null : (scala.collection.Seq) value94.get(); > /* 382 */ ArrayData value92 = null; > /* 383 */ > /* 384 */ if (!isNull93) { > /* 385 */ > /* 386 */ UTF8String[] convertedArray = null; > /* 387 */ int dataLength = value93.size(); > /* 388 */ convertedArray = new UTF8String[dataLength]; > /* 389 */ > /* 390 */ int loopIndex = 0; > /* 391 */ while (loopIndex < dataLength) { > /* 392 */ MapObjects_loopValue2 = (java.lang.String) > (value93.apply(loopIndex)); > /* 393 */ MapObjects_loopIsNull3 = MapObjects_loopValue2 == null; > /* 394 */ > /* 395 */ > /* 396 */ boolean isNull95 = MapObjects_loopIsNull3; > /* 397 */ final UTF8String value95 = isNull95 ? null : > org.apache.spark.unsafe.types.UTF8String.fromString(MapObjects_loopValue2); > /* 398 */ isNull95 = value95 == null; > /* 399 */ if (isNull95) { > /* 400 */ convertedArray[loopIndex] = null; > /* 401 */ } else { > /* 402 */ convertedArray[loopIndex] = value95; > /* 403 */ } > /* 404 */ > /* 405 */ loopIndex += 1; > /* 406 */ } > /* 407 */ > /* 408 */ value92 = new > org.apache.spark.sql.catalyst.util.GenericArrayData(convertedArray); > /* 409 */ } > /* 410 */ if (isNull93) { > /* 411 */ values1[28] = null; > /* 412 */ } else { > /* 413 */ values1[28] = value92; > /* 414 */ } > /* 415 */ } > /* 416 */ > /* 417 */ > /* 418 */ private void apply_8(InternalRow i) { > /* 419 */ > /* 420 */ boolean isNull76 = MapObjects_loopIsNull1; > /* 421 */ final scala.Option value76 = isNull76 ? null : (scala.Option) > MapObjects_loopValue0.local_orig(); > /* 422 */ isNull76 = value76 == null; > /* 423 */ > /* 424 */ final boolean isNull75 = isNull76 || value76.isEmpty(); > /* 425 */ boolean value75 = isNull75 ? > /* 426 */ false : (Boolean) value76.get(); > /* 427 */ if (isNull75) { > /* 428 */ values1[20] = null; > /* 429 */ } else { > /* 430 */ values1[20] = value75; > /* 431 */ } > /* 432 */ boolean isNull78 = MapObjects_loopIsNull1; > /* 433 */ final scala.Option value78 = isNull78 ? null : (scala.Option) > MapObjects_loopValue0.local_resp(); > /* 434 */ isNull78 = value78 == null; > /* 435 */ > /* 436 */ final boolean isNull77 = isNull78 || value78.isEmpty(); > /* 437 */ boolean value77 = isNull77 ? > /* 438 */ false : (Boolean) value78.get(); > /* 439 */ if (isNull77) { > /* 440 */ values1[21] = null; > /* 441 */ } else { > /* 442 */ values1[21] = value77; > /* 443 */ } > /* 444 */ boolean isNull80 = MapObjects_loopIsNull1; > /* 445 */ final scala.Option value80 = isNull80 ? null : (scala.Option) > MapObjects_loopValue0.missed_bytes(); > /* 446 */ isNull80 = value80 == null; > /* 447 */ > /* 448 */ final boolean isNull79 = isNull80 || value80.isEmpty(); > /* 449 */ long value79 = isNull79 ? > /* 450 */ -1L : (Long) value80.get(); > /* 451 */ if (isNull79) { > /* 452 */ values1[22] = null; > /* 453 */ } else { > /* 454 */ values1[22] = value79; > /* 455 */ } > /* 456 */ } > /* 457 */ > /* 458 */ > /* 459 */ private void apply1_0(InternalRow i) { > /* 460 */ > /* 461 */ boolean isNull17 = false; > /* 462 */ final io.mistnet.analytics.scan.SrcDstGrouped value17 = > isNull17 ? null : (io.mistnet.analytics.scan.SrcDstGrouped) value4._2(); > /* 463 */ isNull17 = value17 == null; > /* 464 */ > /* 465 */ if (isNull17) { > /* 466 */ throw new RuntimeException(errMsg); > /* 467 */ } > /* 468 */ > /* 469 */ boolean isNull15 = false; > /* 470 */ final java.lang.String value15 = isNull15 ? null : > (java.lang.String) value17.src(); > /* 471 */ isNull15 = value15 == null; > /* 472 */ boolean isNull14 = isNull15; > /* 473 */ final UTF8String value14 = isNull14 ? null : > org.apache.spark.unsafe.types.UTF8String.fromString(value15); > /* 474 */ isNull14 = value14 == null; > /* 475 */ if (isNull14) { > /* 476 */ values[0] = null; > /* 477 */ } else { > /* 478 */ values[0] = value14; > /* 479 */ } > /* 480 */ boolean isNull21 = false; > /* 481 */ final io.mistnet.analytics.scan.SrcDstGrouped value21 = > isNull21 ? null : (io.mistnet.analytics.scan.SrcDstGrouped) value4._2(); > /* 482 */ isNull21 = value21 == null; > /* 483 */ > /* 484 */ if (isNull21) { > /* 485 */ throw new RuntimeException(errMsg1); > /* 486 */ } > /* 487 */ > /* 488 */ boolean isNull19 = false; > /* 489 */ final java.lang.String value19 = isNull19 ? null : > (java.lang.String) value21.dest(); > /* 490 */ isNull19 = value19 == null; > /* 491 */ boolean isNull18 = isNull19; > /* 492 */ final UTF8String value18 = isNull18 ? null : > org.apache.spark.unsafe.types.UTF8String.fromString(value19); > /* 493 */ isNull18 = value18 == null; > /* 494 */ if (isNull18) { > /* 495 */ values[1] = null; > /* 496 */ } else { > /* 497 */ values[1] = value18; > /* 498 */ } > /* 499 */ } > /* 500 */ > /* 501 */ > /* 502 */ private void apply_2(InternalRow i) { > /* 503 */ > /* 504 */ boolean isNull43 = MapObjects_loopIsNull1; > /* 505 */ final scala.Option value43 = isNull43 ? null : (scala.Option) > MapObjects_loopValue0.user_uuid(); > /* 506 */ isNull43 = value43 == null; > /* 507 */ > /* 508 */ final boolean isNull42 = isNull43 || value43.isEmpty(); > /* 509 */ java.lang.String value42 = isNull42 ? > /* 510 */ null : (java.lang.String) value43.get(); > /* 511 */ boolean isNull41 = isNull42; > /* 512 */ final UTF8String value41 = isNull41 ? null : > org.apache.spark.unsafe.types.UTF8String.fromString(value42); > /* 513 */ isNull41 = value41 == null; > /* 514 */ if (isNull41) { > /* 515 */ values1[4] = null; > /* 516 */ } else { > /* 517 */ values1[4] = value41; > /* 518 */ } > /* 519 */ boolean isNull46 = MapObjects_loopIsNull1; > /* 520 */ final scala.Option value46 = isNull46 ? null : (scala.Option) > MapObjects_loopValue0.host_uuid(); > /* 521 */ isNull46 = value46 == null; > /* 522 */ > /* 523 */ final boolean isNull45 = isNull46 || value46.isEmpty(); > /* 524 */ java.lang.String value45 = isNull45 ? > /* 525 */ null : (java.lang.String) value46.get(); > /* 526 */ boolean isNull44 = isNull45; > /* 527 */ final UTF8String value44 = isNull44 ? null : > org.apache.spark.unsafe.types.UTF8String.fromString(value45); > /* 528 */ isNull44 = value44 == null; > /* 529 */ if (isNull44) { > /* 530 */ values1[5] = null; > /* 531 */ } else { > /* 532 */ values1[5] = value44; > /* 533 */ } > /* 534 */ } > /* 535 */ > /* 536 */ > /* 537 */ private void apply_5(InternalRow i) { > /* 538 */ > /* 539 */ boolean isNull57 = MapObjects_loopIsNull1; > /* 540 */ final int value57 = isNull57 ? -1 : > MapObjects_loopValue0.src_port(); > /* 541 */ if (isNull57) { > /* 542 */ values1[11] = null; > /* 543 */ } else { > /* 544 */ values1[11] = value57; > /* 545 */ } > /* 546 */ boolean isNull59 = MapObjects_loopIsNull1; > /* 547 */ final java.lang.String value59 = isNull59 ? null : > (java.lang.String) MapObjects_loopValue0.dest(); > /* 548 */ isNull59 = value59 == null; > /* 549 */ boolean isNull58 = isNull59; > /* 550 */ final UTF8String value58 = isNull58 ? null : > org.apache.spark.unsafe.types.UTF8String.fromString(value59); > /* 551 */ isNull58 = value58 == null; > /* 552 */ if (isNull58) { > /* 553 */ values1[12] = null; > /* 554 */ } else { > /* 555 */ values1[12] = value58; > /* 556 */ } > /* 557 */ boolean isNull60 = MapObjects_loopIsNull1; > /* 558 */ final int value60 = isNull60 ? -1 : > MapObjects_loopValue0.dest_port(); > /* 559 */ if (isNull60) { > /* 560 */ values1[13] = null; > /* 561 */ } else { > /* 562 */ values1[13] = value60; > /* 563 */ } > /* 564 */ boolean isNull62 = MapObjects_loopIsNull1; > /* 565 */ final java.lang.String value62 = isNull62 ? null : > (java.lang.String) MapObjects_loopValue0.proto(); > /* 566 */ isNull62 = value62 == null; > /* 567 */ boolean isNull61 = isNull62; > /* 568 */ final UTF8String value61 = isNull61 ? null : > org.apache.spark.unsafe.types.UTF8String.fromString(value62); > /* 569 */ isNull61 = value61 == null; > /* 570 */ if (isNull61) { > /* 571 */ values1[14] = null; > /* 572 */ } else { > /* 573 */ values1[14] = value61; > /* 574 */ } > /* 575 */ } > /* 576 */ > /* 577 */ > /* 578 */ private void apply_10(InternalRow i) { > /* 579 */ > /* 580 */ boolean isNull87 = MapObjects_loopIsNull1; > /* 581 */ final scala.Option value87 = isNull87 ? null : (scala.Option) > MapObjects_loopValue0.orig_ip_bytes(); > /* 582 */ isNull87 = value87 == null; > /* 583 */ > /* 584 */ final boolean isNull86 = isNull87 || value87.isEmpty(); > /* 585 */ long value86 = isNull86 ? > /* 586 */ -1L : (Long) value87.get(); > /* 587 */ if (isNull86) { > /* 588 */ values1[25] = null; > /* 589 */ } else { > /* 590 */ values1[25] = value86; > /* 591 */ } > /* 592 */ boolean isNull89 = MapObjects_loopIsNull1; > /* 593 */ final scala.Option value89 = isNull89 ? null : (scala.Option) > MapObjects_loopValue0.resp_pkts(); > /* 594 */ isNull89 = value89 == null; > /* 595 */ > /* 596 */ final boolean isNull88 = isNull89 || value89.isEmpty(); > /* 597 */ long value88 = isNull88 ? > /* 598 */ -1L : (Long) value89.get(); > /* 599 */ if (isNull88) { > /* 600 */ values1[26] = null; > /* 601 */ } else { > /* 602 */ values1[26] = value88; > /* 603 */ } > /* 604 */ boolean isNull91 = MapObjects_loopIsNull1; > /* 605 */ final scala.Option value91 = isNull91 ? null : (scala.Option) > MapObjects_loopValue0.resp_ip_bytes(); > /* 606 */ isNull91 = value91 == null; > /* 607 */ > /* 608 */ final boolean isNull90 = isNull91 || value91.isEmpty(); > /* 609 */ long value90 = isNull90 ? > /* 610 */ -1L : (Long) value91.get(); > /* 611 */ if (isNull90) { > /* 612 */ values1[27] = null; > /* 613 */ } else { > /* 614 */ values1[27] = value90; > /* 615 */ } > /* 616 */ } > /* 617 */ > /* 618 */ > /* 619 */ public SpecificMutableProjection(Object[] references) { > /* 620 */ this.references = references; > /* 621 */ mutableRow = new > org.apache.spark.sql.catalyst.expressions.GenericMutableRow(2); > /* 622 */ this.values = null; > /* 623 */ this.errMsg = (java.lang.String) references[3]; > /* 624 */ this.errMsg1 = (java.lang.String) references[4]; > /* 625 */ > /* 626 */ > /* 627 */ this.errMsg2 = (java.lang.String) references[5]; > /* 628 */ this.values1 = null; > /* 629 */ > /* 630 */ > /* 631 */ this.isNull_0 = true; > /* 632 */ this.value_0 = false; > /* 633 */ this.isNull_1 = true; > /* 634 */ this.value_1 = null; > /* 635 */ } > /* 636 */ > /* 637 */ public > org.apache.spark.sql.catalyst.expressions.codegen.BaseMutableProjection > target(MutableRow row) { > /* 638 */ mutableRow = row; > /* 639 */ return this; > /* 640 */ } > /* 641 */ > /* 642 */ /* Provide immutable access to the last projected row. */ > /* 643 */ public InternalRow currentValue() { > /* 644 */ return (InternalRow) mutableRow; > /* 645 */ } > /* 646 */ > /* 647 */ public java.lang.Object apply(java.lang.Object _i) { > /* 648 */ InternalRow i = (InternalRow) _i; > /* 649 */ > /* 650 */ > /* 651 */ > /* 652 */ Object obj = ((Expression) references[0]).eval(null); > /* 653 */ scala.Tuple2 value1 = (scala.Tuple2) obj; > /* 654 */ > /* 655 */ boolean isNull2 = false; > /* 656 */ final boolean value2 = isNull2 ? false : (Boolean) value1._1(); > /* 657 */ this.isNull_0 = isNull2; > /* 658 */ this.value_0 = value2; > /* 659 */ > /* 660 */ > /* 661 */ Object obj1 = ((Expression) references[1]).eval(null); > /* 662 */ scala.Tuple2 value4 = (scala.Tuple2) obj1; > /* 663 */ > /* 664 */ boolean isNull8 = false; > /* 665 */ final io.mistnet.analytics.scan.SrcDstGrouped value8 = isNull8 > ? null : (io.mistnet.analytics.scan.SrcDstGrouped) value4._2(); > /* 666 */ isNull8 = value8 == null; > /* 667 */ boolean isNull6 = false; > /* 668 */ boolean value6 = true; > /* 669 */ > /* 670 */ if (!false && isNull8) { > /* 671 */ } else { > /* 672 */ > /* 673 */ Object obj2 = ((Expression) references[2]).eval(null); > /* 674 */ scala.None$ value10 = (scala.None$) obj2; > /* 675 */ > /* 676 */ boolean isNull11 = false; > /* 677 */ final io.mistnet.analytics.scan.SrcDstGrouped value11 = > isNull11 ? null : (io.mistnet.analytics.scan.SrcDstGrouped) value4._2(); > /* 678 */ isNull11 = value11 == null; > /* 679 */ boolean isNull9 = false || isNull11; > /* 680 */ final boolean value9 = isNull9 ? false : > value10.equals(value11); > /* 681 */ if (!isNull9 && value9) { > /* 682 */ } else if (!false && !isNull9) { > /* 683 */ value6 = false; > /* 684 */ } else { > /* 685 */ isNull6 = true; > /* 686 */ } > /* 687 */ } > /* 688 */ boolean isNull5 = false; > /* 689 */ InternalRow value5 = null; > /* 690 */ if (!isNull6 && value6) { > /* 691 */ > /* 692 */ final InternalRow value12 = null; > /* 693 */ isNull5 = true; > /* 694 */ value5 = value12; > /* 695 */ } else { > /* 696 */ > /* 697 */ boolean isNull13 = false; > /* 698 */ this.values = new Object[3];apply1_0(i); > /* 699 */ apply1_1(i); > /* 700 */ final InternalRow value13 = new > org.apache.spark.sql.catalyst.expressions.GenericInternalRow(values); > /* 701 */ this.values = null; > /* 702 */ isNull5 = isNull13; > /* 703 */ value5 = value13; > /* 704 */ } > /* 705 */ this.isNull_1 = isNull5; > /* 706 */ this.value_1 = value5; > /* 707 */ > /* 708 */ // copy all the results into MutableRow > /* 709 */ > /* 710 */ if (!this.isNull_0) { > /* 711 */ mutableRow.setBoolean(0, this.value_0); > /* 712 */ } else { > /* 713 */ mutableRow.setNullAt(0); > /* 714 */ } > /* 715 */ > /* 716 */ if (!this.isNull_1) { > /* 717 */ mutableRow.update(1, this.value_1); > /* 718 */ } else { > /* 719 */ mutableRow.setNullAt(1); > /* 720 */ } > /* 721 */ > /* 722 */ return mutableRow; > /* 723 */ } > /* 724 */ } > at > org.spark_project.guava.util.concurrent.AbstractFuture$Sync.getValue(AbstractFuture.java:306) > at > org.spark_project.guava.util.concurrent.AbstractFuture$Sync.get(AbstractFuture.java:293) > at > org.spark_project.guava.util.concurrent.AbstractFuture.get(AbstractFuture.java:116) > at > org.spark_project.guava.util.concurrent.Uninterruptibles.getUninterruptibly(Uninterruptibles.java:135) > at > org.spark_project.guava.cache.LocalCache$Segment.getAndRecordStats(LocalCache.java:2410) > at > org.spark_project.guava.cache.LocalCache$Segment.loadSync(LocalCache.java:2380) > at > org.spark_project.guava.cache.LocalCache$Segment.lockedGetOrLoad(LocalCache.java:2342) > at > org.spark_project.guava.cache.LocalCache$Segment.get(LocalCache.java:2257) > at org.spark_project.guava.cache.LocalCache.get(LocalCache.java:4000) > at > org.spark_project.guava.cache.LocalCache.getOrLoad(LocalCache.java:4004) > at > org.spark_project.guava.cache.LocalCache$LocalLoadingCache.get(LocalCache.java:4874) > at > org.apache.spark.sql.catalyst.expressions.codegen.CodeGenerator$.compile(CodeGenerator.scala:841) > at > org.apache.spark.sql.catalyst.expressions.codegen.GenerateMutableProjection$.create(GenerateMutableProjection.scala:140) > at > org.apache.spark.sql.catalyst.expressions.codegen.GenerateMutableProjection$.generate(GenerateMutableProjection.scala:44) > at > org.apache.spark.sql.execution.SparkPlan.newMutableProjection(SparkPlan.scala:369) > at > org.apache.spark.sql.execution.aggregate.SortAggregateExec$$anonfun$doExecute$1$$anonfun$3$$anonfun$4.apply(SortAggregateExec.scala:93) > at > org.apache.spark.sql.execution.aggregate.SortAggregateExec$$anonfun$doExecute$1$$anonfun$3$$anonfun$4.apply(SortAggregateExec.scala:92) > at > org.apache.spark.sql.execution.aggregate.AggregationIterator.<init>(AggregationIterator.scala:143) > at > org.apache.spark.sql.execution.aggregate.SortBasedAggregationIterator.<init>(SortBasedAggregationIterator.scala:39) > at > org.apache.spark.sql.execution.aggregate.SortAggregateExec$$anonfun$doExecute$1$$anonfun$3.apply(SortAggregateExec.scala:84) > at > org.apache.spark.sql.execution.aggregate.SortAggregateExec$$anonfun$doExecute$1$$anonfun$3.apply(SortAggregateExec.scala:75) > at > org.apache.spark.rdd.RDD$$anonfun$mapPartitionsInternal$1$$anonfun$apply$24.apply(RDD.scala:803) > at > org.apache.spark.rdd.RDD$$anonfun$mapPartitionsInternal$1$$anonfun$apply$24.apply(RDD.scala:803) > at > org.apache.spark.rdd.MapPartitionsRDD.compute(MapPartitionsRDD.scala:38) > at org.apache.spark.rdd.RDD.computeOrReadCheckpoint(RDD.scala:319) > at org.apache.spark.rdd.RDD.iterator(RDD.scala:283) > at > org.apache.spark.rdd.MapPartitionsRDD.compute(MapPartitionsRDD.scala:38) > at org.apache.spark.rdd.RDD.computeOrReadCheckpoint(RDD.scala:319) > at org.apache.spark.rdd.RDD.iterator(RDD.scala:283) > at > org.apache.spark.scheduler.ShuffleMapTask.runTask(ShuffleMapTask.scala:79) > at > org.apache.spark.scheduler.ShuffleMapTask.runTask(ShuffleMapTask.scala:47) > at org.apache.spark.scheduler.Task.run(Task.scala:86) > at org.apache.spark.executor.Executor$TaskRunner.run(Executor.scala:274) > at > java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) > at > java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) > at java.lang.Thread.run(Thread.java:745) > {noformat} -- This message was sent by Atlassian JIRA (v6.3.4#6332) --------------------------------------------------------------------- To unsubscribe, e-mail: issues-unsubscr...@spark.apache.org For additional commands, e-mail: issues-h...@spark.apache.org