[
https://issues.apache.org/jira/browse/SPARK-36862?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel&focusedCommentId=17599013#comment-17599013
]
Lukas Waldmann edited comment on SPARK-36862 at 9/1/22 2:55 PM:
----------------------------------------------------------------
I managed to reproduce the issue in my environment. Problem is on line 192 -
variable name in function header having array index
Here is the generated code
{code:java}
/* 001 */ public Object generate(Object[] references) {
/* 002 */ return new GeneratedIteratorForCodegenStage636(references);
/* 003 */ }
/* 004 */
/* 005 */ // codegenStageId=636
/* 006 */ final class GeneratedIteratorForCodegenStage636 extends
org.apache.spark.sql.execution.BufferedRowIterator {
/* 007 */ private Object[] references;
/* 008 */ private scala.collection.Iterator[] inputs;
/* 009 */ private scala.collection.Iterator smj_leftInput_0;
/* 010 */ private scala.collection.Iterator smj_rightInput_0;
/* 011 */ private InternalRow smj_leftRow_0;
/* 012 */ private InternalRow smj_rightRow_0;
/* 013 */ private boolean smj_globalIsNull_0;
/* 014 */ private boolean smj_globalIsNull_1;
/* 015 */ private double smj_value_27;
/* 016 */ private
org.apache.spark.sql.execution.ExternalAppendOnlyUnsafeRowArray smj_matches_0;
/* 017 */ private double smj_value_28;
/* 018 */ private boolean smj_isNull_25;
/* 019 */ private boolean smj_isNull_26;
/* 020 */ private boolean smj_isNull_27;
/* 021 */ private boolean smj_isNull_28;
/* 022 */ private boolean smj_isNull_29;
/* 023 */ private boolean smj_isNull_30;
/* 024 */ private boolean project_subExprIsNull_0;
/* 025 */ private
org.apache.spark.sql.catalyst.expressions.codegen.UnsafeRowWriter[]
smj_mutableStateArray_2 = new
org.apache.spark.sql.catalyst.expressions.codegen.UnsafeRowWriter[2];
/* 026 */ private java.util.regex.Pattern[] project_mutableStateArray_0 = new
java.util.regex.Pattern[1];
/* 027 */ private Decimal[] smj_mutableStateArray_1 = new Decimal[1];
/* 028 */ private String[] project_mutableStateArray_1 = new String[1];
/* 029 */ private UTF8String[] smj_mutableStateArray_0 = new UTF8String[7];
/* 030 */
/* 031 */ public GeneratedIteratorForCodegenStage636(Object[] references) {
/* 032 */ this.references = references;
/* 033 */ }
/* 034 */
/* 035 */ public void init(int index, scala.collection.Iterator[] inputs) {
/* 036 */ partitionIndex = index;
/* 037 */ this.inputs = inputs;
/* 038 */ smj_leftInput_0 = inputs[0];
/* 039 */ smj_rightInput_0 = inputs[1];
/* 040 */
/* 041 */ smj_matches_0 = new
org.apache.spark.sql.execution.ExternalAppendOnlyUnsafeRowArray(2147483632,
2147483647);
/* 042 */ smj_mutableStateArray_2[0] = new
org.apache.spark.sql.catalyst.expressions.codegen.UnsafeRowWriter(6, 192);
/* 043 */ smj_mutableStateArray_2[1] = new
org.apache.spark.sql.catalyst.expressions.codegen.UnsafeRowWriter(6, 192);
/* 044 */
/* 045 */ }
/* 046 */
/* 047 */ private boolean smj_findNextOuterJoinRows_0(
/* 048 */ scala.collection.Iterator leftIter,
/* 049 */ scala.collection.Iterator rightIter) {
/* 050 */ smj_leftRow_0 = null;
/* 051 */ int comp = 0;
/* 052 */ while (smj_leftRow_0 == null) {
/* 053 */ if (!leftIter.hasNext()) return false;
/* 054 */ smj_leftRow_0 = (InternalRow) leftIter.next();
/* 055 */ UTF8String smj_value_22 = smj_If_0(smj_leftRow_0);
/* 056 */ boolean smj_isNull_2 = smj_globalIsNull_1;
/* 057 */ double smj_value_2 = -1.0;
/* 058 */ if (!smj_globalIsNull_1) {
/* 059 */ final String smj_doubleStr_0 = smj_value_22.toString();
/* 060 */ try {
/* 061 */ smj_value_2 = Double.valueOf(smj_doubleStr_0);
/* 062 */ } catch (java.lang.NumberFormatException e) {
/* 063 */ final Double d = (Double)
Cast.processFloatingPointSpecialLiterals(smj_doubleStr_0, false);
/* 064 */ if (d == null) {
/* 065 */ smj_isNull_2 = true;
/* 066 */ } else {
/* 067 */ smj_value_2 = d.doubleValue();
/* 068 */ }
/* 069 */ }
/* 070 */ }
/* 071 */ boolean smj_isNull_1 = smj_isNull_2;
/* 072 */ double smj_value_1 = -1.0;
/* 073 */
/* 074 */ if (!smj_isNull_2) {
/* 075 */ if (Double.isNaN(smj_value_2)) {
/* 076 */ smj_value_1 = Double.NaN;
/* 077 */ } else if (smj_value_2 == -0.0d) {
/* 078 */ smj_value_1 = 0.0d;
/* 079 */ } else {
/* 080 */ smj_value_1 = smj_value_2;
/* 081 */ }
/* 082 */
/* 083 */ }
/* 084 */ if (smj_isNull_1) {
/* 085 */ if (!smj_matches_0.isEmpty()) {
/* 086 */ smj_matches_0.clear();
/* 087 */ }
/* 088 */ return true;
/* 089 */ }
/* 090 */ if (!smj_matches_0.isEmpty()) {
/* 091 */ comp = 0;
/* 092 */ if (comp == 0) {
/* 093 */ comp =
org.apache.spark.sql.catalyst.util.SQLOrderingUtil.compareDoubles(smj_value_1,
smj_value_28);
/* 094 */ }
/* 095 */
/* 096 */ if (comp == 0) {
/* 097 */ return true;
/* 098 */ }
/* 099 */ smj_matches_0.clear();
/* 100 */ }
/* 101 */
/* 102 */ do {
/* 103 */ if (smj_rightRow_0 == null) {
/* 104 */ if (!rightIter.hasNext()) {
/* 105 */ if (!smj_matches_0.isEmpty()) {
/* 106 */ smj_value_28 = smj_value_1;
/* 107 */ }
/* 108 */ return true;
/* 109 */ }
/* 110 */ smj_rightRow_0 = (InternalRow) rightIter.next();
/* 111 */ Decimal smj_value_26 = smj_rightRow_0.getDecimal(1, 38, 0);
/* 112 */ boolean smj_isNull_23 = false;
/* 113 */ double smj_value_25 = -1.0;
/* 114 */ if (!false) {
/* 115 */ smj_value_25 = smj_value_26.toDouble();
/* 116 */ }
/* 117 */ double smj_value_24 = -1.0;
/* 118 */
/* 119 */ if (Double.isNaN(smj_value_25)) {
/* 120 */ smj_value_24 = Double.NaN;
/* 121 */ } else if (smj_value_25 == -0.0d) {
/* 122 */ smj_value_24 = 0.0d;
/* 123 */ } else {
/* 124 */ smj_value_24 = smj_value_25;
/* 125 */ }
/* 126 */ if (false) {
/* 127 */ smj_rightRow_0 = null;
/* 128 */ continue;
/* 129 */ }
/* 130 */ smj_value_27 = smj_value_24;
/* 131 */ }
/* 132 */
/* 133 */ comp = 0;
/* 134 */ if (comp == 0) {
/* 135 */ comp =
org.apache.spark.sql.catalyst.util.SQLOrderingUtil.compareDoubles(smj_value_1,
smj_value_27);
/* 136 */ }
/* 137 */
/* 138 */ if (comp > 0) {
/* 139 */ smj_rightRow_0 = null;
/* 140 */ } else if (comp < 0) {
/* 141 */ if (!smj_matches_0.isEmpty()) {
/* 142 */ smj_value_28 = smj_value_1;
/* 143 */ }
/* 144 */ return true;
/* 145 */ } else {
/* 146 */ smj_matches_0.add((UnsafeRow) smj_rightRow_0);
/* 147 */ smj_rightRow_0 = null;
/* 148 */ }
/* 149 */ } while (true);
/* 150 */ }
/* 151 */ return false; // unreachable
/* 152 */ }
/* 153 */
/* 154 */ private UTF8String smj_If_0(InternalRow smj_leftRow_0) {
/* 155 */ boolean smj_isNull_5 = false;
/* 156 */ boolean smj_isNull_6 = true;
/* 157 */ ArrayData smj_value_6 = null;
/* 158 */ boolean smj_isNull_7 = smj_leftRow_0.isNullAt(2);
/* 159 */ UTF8String smj_value_7 = smj_isNull_7 ?
/* 160 */ null : (smj_leftRow_0.getUTF8String(2));
/* 161 */ if (!smj_isNull_7) {
/* 162 */ smj_isNull_6 = false; // resultCode could change nullability.
/* 163 */ smj_value_6 = new
org.apache.spark.sql.catalyst.util.GenericArrayData(smj_value_7.split(((UTF8String)
references[0] /* literal */),-1));
/* 164 */
/* 165 */ }
/* 166 */ int smj_value_5 = smj_isNull_6 ? -1 :
/* 167 */ (smj_value_6).numElements();
/* 168 */
/* 169 */ boolean smj_value_4 = false;
/* 170 */ smj_value_4 = smj_value_5 > 2;
/* 171 */ boolean smj_isNull_3 = false;
/* 172 */ UTF8String smj_value_3 = null;
/* 173 */ if (!false && smj_value_4) {
/* 174 */ boolean smj_isNull_11 = true;
/* 175 */ UTF8String smj_value_11 = null;
/* 176 */ UTF8String smj_value_18 = smj_GetArrayItem_0(smj_leftRow_0);
/* 177 */ if (!smj_globalIsNull_0) {
/* 178 */ smj_isNull_11 = false; // resultCode could change nullability.
/* 179 */ smj_value_11 = smj_value_18.substringSQL(1, 10);
/* 180 */
/* 181 */ }
/* 182 */ smj_isNull_3 = smj_isNull_11;
/* 183 */ smj_value_3 = smj_value_11;
/* 184 */ } else {
/* 185 */ smj_isNull_3 = true;
/* 186 */ smj_value_3 = ((UTF8String)null);
/* 187 */ }
/* 188 */ smj_globalIsNull_1 = smj_isNull_3;
/* 189 */ return smj_value_3;
/* 190 */ }
/* 191 */
/* 192 */ private UTF8String
project_subExpr_0(org.apache.spark.unsafe.types.UTF8String
smj_mutableStateArray_0[3], boolean smj_isNull_28) {
/* 193 */ boolean project_isNull_0 = true;
/* 194 */ UTF8String project_value_0 = null;
/* 195 */
/* 196 */ if (!smj_isNull_28) {
/* 197 */ project_isNull_0 = false; // resultCode could change nullability.
/* 198 */
/* 199 */ if (!((UTF8String) references[3] /* literal
*/).equals(smj_mutableStateArray_0[5])) {
/* 200 */ // regex value changed
/* 201 */ smj_mutableStateArray_0[5] = ((UTF8String) references[3] /* literal
*/).clone();
/* 202 */ project_mutableStateArray_0[0] =
java.util.regex.Pattern.compile(smj_mutableStateArr",
"ay_0[5].toString());
/* 203 */ }
/* 204 */ if (!((UTF8String) references[4] /* literal
*/).equals(smj_mutableStateArray_0[6])) {
/* 205 */ // replacement string changed
/* 206 */ smj_mutableStateArray_0[6] = ((UTF8String) references[4] /* literal
*/).clone();
/* 207 */ project_mutableStateArray_1[0] =
smj_mutableStateArray_0[6].toString();
/* 208 */ }
/* 209 */ String project_source_0 = smj_mutableStateArray_0[3].toString();
/* 210 */ int project_position_0 = 1 - 1;
/* 211 */ if (project_position_0 < project_source_0.length()) {
/* 212 */ java.lang.StringBuffer project_termResult_0 = new
java.lang.StringBuffer();
/* 213 */ java.util.regex.Matcher project_matcher_0 =
project_mutableStateArray_0[0].matcher(project_source_0);
/* 214 */ project_matcher_0.region(project_position_0,
project_source_0.length());
/* 215 */
/* 216 */ while (project_matcher_0.find()) {
/* 217 */ project_matcher_0.appendReplacement(project_termResult_0,
project_mutableStateArray_1[0]);
/* 218 */ }
/* 219 */ project_matcher_0.appendTail(project_termResult_0);
/* 220 */ project_value_0 =
UTF8String.fromString(project_termResult_0.toString());
/* 221 */ project_termResult_0 = null;
/* 222 */ } else {
/* 223 */ project_value_0 = smj_mutableStateArray_0[3];
/* 224 */ }
/* 225 */ project_isNull_0 = false;
/* 226 */
/* 227 */ }
/* 228 */ project_subExprIsNull_0 = project_isNull_0;
/* 229 */ return project_value_0;
/* 230 */ }
/* 231 */
/* 232 */ private UTF8String smj_GetArrayItem_0(InternalRow smj_leftRow_0) {
/* 233 */ boolean smj_isNull_12 = true;
/* 234 */ UTF8String smj_value_12 = null;
/* 235 */ boolean smj_isNull_13 = true;
/* 236 */ ArrayData smj_value_13 = null;
/* 237 */ boolean smj_isNull_14 = smj_leftRow_0.isNullAt(2);
/* 238 */ UTF8String smj_value_14 = smj_isNull_14 ?
/* 239 */ null : (smj_leftRow_0.getUTF8String(2));
/* 240 */ if (!smj_isNull_14) {
/* 241 */ smj_isNull_13 = false; // resultCode could change nullability.
/* 242 */ smj_value_13 = new
org.apache.spark.sql.catalyst.util.GenericArrayData(smj_value_14.split(((UTF8String)
references[1] /* literal */),-1));
/* 243 */
/* 244 */ }
/* 245 */ if (!smj_isNull_13) {
/* 246 */ smj_isNull_12 = false; // resultCode could change nullability.
/* 247 */
/* 248 */ final int smj_index_0 = (int) 2;
/* 249 */ if (smj_index_0 >= smj_value_13.numElements() || smj_index_0 < 0) {
/* 250 */ smj_isNull_12 = true;
/* 251 */ } else if (smj_value_13.isNullAt(smj_index_0)) {
/* 252 */ smj_isNull_12 = true;
/* 253 */ }
/* 254 */ else {
/* 255 */ smj_value_12 = smj_value_13.getUTF8String(smj_index_0);
/* 256 */ }
/* 257 */
/* 258 */ }
/* 259 */ smj_globalIsNull_0 = smj_isNull_12;
/* 260 */ return smj_value_12;
/* 261 */ }
/* 262 */
/* 263 */ protected void processNext() throws java.io.IOException {
/* 264 */ while (smj_findNextOuterJoinRows_0(smj_leftInput_0,
smj_rightInput_0)) {
/* 265 */ boolean smj_loaded_0 = false;
/* 266 */ smj_isNull_25 = smj_leftRow_0.isNullAt(0);
/* 267 */ smj_mutableStateArray_0[0] = smj_isNull_25 ? null :
(smj_leftRow_0.getUTF8String(0));
/* 268 */ smj_isNull_26 = smj_leftRow_0.isNullAt(1);
/* 269 */ smj_mutableStateArray_0[1] = smj_isNull_26 ? null :
(smj_leftRow_0.getUTF8String(1));
/* 270 */ smj_isNull_27 = smj_leftRow_0.isNullAt(2);
/* 271 */ smj_mutableStateArray_0[2] = smj_isNull_27 ? null :
(smj_leftRow_0.getUTF8String(2));
/* 272 */ smj_isNull_28 = smj_leftRow_0.isNullAt(3);
/* 273 */ smj_mutableStateArray_0[3] = smj_isNull_28 ? null :
(smj_leftRow_0.getUTF8String(3));
/* 274 */ scala.collection.Iterator<UnsafeRow> smj_iterator_0 =
smj_matches_0.generateIterator();
/* 275 */ while (smj_iterator_0.hasNext()) {
/* 276 */ InternalRow smj_rightRow_1 = (InternalRow) smj_iterator_0.next();
/* 277 */ smj_isNull_29 = smj_rightRow_1.isNullAt(0);
/* 278 */ smj_mutableStateArray_0[4] = smj_isNull_29 ? null :
(smj_rightRow_1.getUTF8String(0));
/* 279 */ smj_isNull_30 = false;
/* 280 */ smj_mutableStateArray_1[0] = smj_rightRow_1.getDecimal(1, 38, 0);
/* 281 */
/* 282 */ smj_loaded_0 = true;
/* 283 */ smj_writeJoinRows_0();
/* 284 */ }
/* 285 */ if (!smj_loaded_0) {
/* 286 */ smj_isNull_29 = true;
/* 287 */ smj_isNull_30 = true;
/* 288 */ smj_writeJoinRows_0();
/* 289 */ }
/* 290 */ if (shouldStop()) return;
/* 291 */ }
/* 292 */ ((org.apache.spark.sql.execution.joins.SortMergeJoinExec)
references[5] /* plan */).cleanupResources();
/* 293 */ }
/* 294 */
/* 295 */ private void smj_writeJoinRows_0() throws java.io.IOException {
/* 296 */ ((org.apache.spark.sql.execution.metric.SQLMetric) references[2] /*
numOutputRows */).add(1);
/* 297 */
/* 298 */ // common sub-expressions
/* 299 */
/* 300 */ UTF8String project_subExprValue_0 =
project_subExpr_0(smj_mutableStateArray_0[3], smj_isNull_28);
/* 301 */
/* 302 */ smj_mutableStateArray_2[1].reset();
/* 303 */
/* 304 */ smj_mutableStateArray_2[1].zeroOutNullBytes();
/* 305 */
/* 306 */ if (smj_isNull_25) {
/* 307 */ smj_mutableStateArray_2[1].setNullAt(0);
/* 308 */ } else {
/* 309 */ smj_mutableStateArray_2[1].write(0, smj_mutableStateArray_0[0]);
/* 310 */ }
/* 311 */
/* 312 */ if (smj_isNull_26) {
/* 313 */ smj_mutableStateArray_2[1].setNullAt(1);
/* 314 */ } else {
/* 315 */ smj_mutableStateArray_2[1].write(1, smj_mutableStateArray_0[1]);
/* 316 */ }
/* 317 */
/* 318 */ if (project_subExprIsNull_0) {
/* 319 */ smj_mutableStateArray_2[1].setNullAt(2);
/* 320 */ } else {
/* 321 */ smj_mutableStateArray_2[1].write(2, project_subExprValue_0);
/* 322 */ }
/* 323 */
/* 324 */ if (project_subExprIsNull_0) {
/* 325 */ smj_mutableStateArray_2[1].setNullAt(3);
/* 326 */ } else {
/* 327 */ smj_mutableStateArray_2[1].write(3, project_subExprValue_0);
/* 328 */ }
/* 329 */
/* 330 */ if (smj_isNull_29) {
/* 331 */ smj_mutableStateArray_2[1].setNullAt(4);
/* 332 */ } else {
/* 333 */ smj_mutableStateArray_2[1].write(4, smj_mutableStateArray_0[4]);
/* 334 */ }
/* 335 */
/* 336 */ if (smj_isNull_25) {
/* 337 */ smj_mutableStateArray_2[1].setNullAt(5);
/* 338 */ } else {
/* 339 */ smj_mutableStateArray_2[1].write(5, smj_mutableStateArray_0[0]);
/* 340 */ }
/* 341 */ append((smj_mutableStateArray_2[1].getRow()).copy());
/* 342 */
/* 343 */ }
/* 344 */
/* 345 */ } {code}
was (Author: luky):
I manage reproduce the issue in my environment. Problem is on line 192 -
variable name in function header having array index
Here is the generated code
{code:java}
/* 001 */ public Object generate(Object[] references) {
/* 002 */ return new GeneratedIteratorForCodegenStage636(references);
/* 003 */ }
/* 004 */
/* 005 */ // codegenStageId=636
/* 006 */ final class GeneratedIteratorForCodegenStage636 extends
org.apache.spark.sql.execution.BufferedRowIterator {
/* 007 */ private Object[] references;
/* 008 */ private scala.collection.Iterator[] inputs;
/* 009 */ private scala.collection.Iterator smj_leftInput_0;
/* 010 */ private scala.collection.Iterator smj_rightInput_0;
/* 011 */ private InternalRow smj_leftRow_0;
/* 012 */ private InternalRow smj_rightRow_0;
/* 013 */ private boolean smj_globalIsNull_0;
/* 014 */ private boolean smj_globalIsNull_1;
/* 015 */ private double smj_value_27;
/* 016 */ private
org.apache.spark.sql.execution.ExternalAppendOnlyUnsafeRowArray smj_matches_0;
/* 017 */ private double smj_value_28;
/* 018 */ private boolean smj_isNull_25;
/* 019 */ private boolean smj_isNull_26;
/* 020 */ private boolean smj_isNull_27;
/* 021 */ private boolean smj_isNull_28;
/* 022 */ private boolean smj_isNull_29;
/* 023 */ private boolean smj_isNull_30;
/* 024 */ private boolean project_subExprIsNull_0;
/* 025 */ private
org.apache.spark.sql.catalyst.expressions.codegen.UnsafeRowWriter[]
smj_mutableStateArray_2 = new
org.apache.spark.sql.catalyst.expressions.codegen.UnsafeRowWriter[2];
/* 026 */ private java.util.regex.Pattern[] project_mutableStateArray_0 = new
java.util.regex.Pattern[1];
/* 027 */ private Decimal[] smj_mutableStateArray_1 = new Decimal[1];
/* 028 */ private String[] project_mutableStateArray_1 = new String[1];
/* 029 */ private UTF8String[] smj_mutableStateArray_0 = new UTF8String[7];
/* 030 */
/* 031 */ public GeneratedIteratorForCodegenStage636(Object[] references) {
/* 032 */ this.references = references;
/* 033 */ }
/* 034 */
/* 035 */ public void init(int index, scala.collection.Iterator[] inputs) {
/* 036 */ partitionIndex = index;
/* 037 */ this.inputs = inputs;
/* 038 */ smj_leftInput_0 = inputs[0];
/* 039 */ smj_rightInput_0 = inputs[1];
/* 040 */
/* 041 */ smj_matches_0 = new
org.apache.spark.sql.execution.ExternalAppendOnlyUnsafeRowArray(2147483632,
2147483647);
/* 042 */ smj_mutableStateArray_2[0] = new
org.apache.spark.sql.catalyst.expressions.codegen.UnsafeRowWriter(6, 192);
/* 043 */ smj_mutableStateArray_2[1] = new
org.apache.spark.sql.catalyst.expressions.codegen.UnsafeRowWriter(6, 192);
/* 044 */
/* 045 */ }
/* 046 */
/* 047 */ private boolean smj_findNextOuterJoinRows_0(
/* 048 */ scala.collection.Iterator leftIter,
/* 049 */ scala.collection.Iterator rightIter) {
/* 050 */ smj_leftRow_0 = null;
/* 051 */ int comp = 0;
/* 052 */ while (smj_leftRow_0 == null) {
/* 053 */ if (!leftIter.hasNext()) return false;
/* 054 */ smj_leftRow_0 = (InternalRow) leftIter.next();
/* 055 */ UTF8String smj_value_22 = smj_If_0(smj_leftRow_0);
/* 056 */ boolean smj_isNull_2 = smj_globalIsNull_1;
/* 057 */ double smj_value_2 = -1.0;
/* 058 */ if (!smj_globalIsNull_1) {
/* 059 */ final String smj_doubleStr_0 = smj_value_22.toString();
/* 060 */ try {
/* 061 */ smj_value_2 = Double.valueOf(smj_doubleStr_0);
/* 062 */ } catch (java.lang.NumberFormatException e) {
/* 063 */ final Double d = (Double)
Cast.processFloatingPointSpecialLiterals(smj_doubleStr_0, false);
/* 064 */ if (d == null) {
/* 065 */ smj_isNull_2 = true;
/* 066 */ } else {
/* 067 */ smj_value_2 = d.doubleValue();
/* 068 */ }
/* 069 */ }
/* 070 */ }
/* 071 */ boolean smj_isNull_1 = smj_isNull_2;
/* 072 */ double smj_value_1 = -1.0;
/* 073 */
/* 074 */ if (!smj_isNull_2) {
/* 075 */ if (Double.isNaN(smj_value_2)) {
/* 076 */ smj_value_1 = Double.NaN;
/* 077 */ } else if (smj_value_2 == -0.0d) {
/* 078 */ smj_value_1 = 0.0d;
/* 079 */ } else {
/* 080 */ smj_value_1 = smj_value_2;
/* 081 */ }
/* 082 */
/* 083 */ }
/* 084 */ if (smj_isNull_1) {
/* 085 */ if (!smj_matches_0.isEmpty()) {
/* 086 */ smj_matches_0.clear();
/* 087 */ }
/* 088 */ return true;
/* 089 */ }
/* 090 */ if (!smj_matches_0.isEmpty()) {
/* 091 */ comp = 0;
/* 092 */ if (comp == 0) {
/* 093 */ comp =
org.apache.spark.sql.catalyst.util.SQLOrderingUtil.compareDoubles(smj_value_1,
smj_value_28);
/* 094 */ }
/* 095 */
/* 096 */ if (comp == 0) {
/* 097 */ return true;
/* 098 */ }
/* 099 */ smj_matches_0.clear();
/* 100 */ }
/* 101 */
/* 102 */ do {
/* 103 */ if (smj_rightRow_0 == null) {
/* 104 */ if (!rightIter.hasNext()) {
/* 105 */ if (!smj_matches_0.isEmpty()) {
/* 106 */ smj_value_28 = smj_value_1;
/* 107 */ }
/* 108 */ return true;
/* 109 */ }
/* 110 */ smj_rightRow_0 = (InternalRow) rightIter.next();
/* 111 */ Decimal smj_value_26 = smj_rightRow_0.getDecimal(1, 38, 0);
/* 112 */ boolean smj_isNull_23 = false;
/* 113 */ double smj_value_25 = -1.0;
/* 114 */ if (!false) {
/* 115 */ smj_value_25 = smj_value_26.toDouble();
/* 116 */ }
/* 117 */ double smj_value_24 = -1.0;
/* 118 */
/* 119 */ if (Double.isNaN(smj_value_25)) {
/* 120 */ smj_value_24 = Double.NaN;
/* 121 */ } else if (smj_value_25 == -0.0d) {
/* 122 */ smj_value_24 = 0.0d;
/* 123 */ } else {
/* 124 */ smj_value_24 = smj_value_25;
/* 125 */ }
/* 126 */ if (false) {
/* 127 */ smj_rightRow_0 = null;
/* 128 */ continue;
/* 129 */ }
/* 130 */ smj_value_27 = smj_value_24;
/* 131 */ }
/* 132 */
/* 133 */ comp = 0;
/* 134 */ if (comp == 0) {
/* 135 */ comp =
org.apache.spark.sql.catalyst.util.SQLOrderingUtil.compareDoubles(smj_value_1,
smj_value_27);
/* 136 */ }
/* 137 */
/* 138 */ if (comp > 0) {
/* 139 */ smj_rightRow_0 = null;
/* 140 */ } else if (comp < 0) {
/* 141 */ if (!smj_matches_0.isEmpty()) {
/* 142 */ smj_value_28 = smj_value_1;
/* 143 */ }
/* 144 */ return true;
/* 145 */ } else {
/* 146 */ smj_matches_0.add((UnsafeRow) smj_rightRow_0);
/* 147 */ smj_rightRow_0 = null;
/* 148 */ }
/* 149 */ } while (true);
/* 150 */ }
/* 151 */ return false; // unreachable
/* 152 */ }
/* 153 */
/* 154 */ private UTF8String smj_If_0(InternalRow smj_leftRow_0) {
/* 155 */ boolean smj_isNull_5 = false;
/* 156 */ boolean smj_isNull_6 = true;
/* 157 */ ArrayData smj_value_6 = null;
/* 158 */ boolean smj_isNull_7 = smj_leftRow_0.isNullAt(2);
/* 159 */ UTF8String smj_value_7 = smj_isNull_7 ?
/* 160 */ null : (smj_leftRow_0.getUTF8String(2));
/* 161 */ if (!smj_isNull_7) {
/* 162 */ smj_isNull_6 = false; // resultCode could change nullability.
/* 163 */ smj_value_6 = new
org.apache.spark.sql.catalyst.util.GenericArrayData(smj_value_7.split(((UTF8String)
references[0] /* literal */),-1));
/* 164 */
/* 165 */ }
/* 166 */ int smj_value_5 = smj_isNull_6 ? -1 :
/* 167 */ (smj_value_6).numElements();
/* 168 */
/* 169 */ boolean smj_value_4 = false;
/* 170 */ smj_value_4 = smj_value_5 > 2;
/* 171 */ boolean smj_isNull_3 = false;
/* 172 */ UTF8String smj_value_3 = null;
/* 173 */ if (!false && smj_value_4) {
/* 174 */ boolean smj_isNull_11 = true;
/* 175 */ UTF8String smj_value_11 = null;
/* 176 */ UTF8String smj_value_18 = smj_GetArrayItem_0(smj_leftRow_0);
/* 177 */ if (!smj_globalIsNull_0) {
/* 178 */ smj_isNull_11 = false; // resultCode could change nullability.
/* 179 */ smj_value_11 = smj_value_18.substringSQL(1, 10);
/* 180 */
/* 181 */ }
/* 182 */ smj_isNull_3 = smj_isNull_11;
/* 183 */ smj_value_3 = smj_value_11;
/* 184 */ } else {
/* 185 */ smj_isNull_3 = true;
/* 186 */ smj_value_3 = ((UTF8String)null);
/* 187 */ }
/* 188 */ smj_globalIsNull_1 = smj_isNull_3;
/* 189 */ return smj_value_3;
/* 190 */ }
/* 191 */
/* 192 */ private UTF8String
project_subExpr_0(org.apache.spark.unsafe.types.UTF8String
smj_mutableStateArray_0[3], boolean smj_isNull_28) {
/* 193 */ boolean project_isNull_0 = true;
/* 194 */ UTF8String project_value_0 = null;
/* 195 */
/* 196 */ if (!smj_isNull_28) {
/* 197 */ project_isNull_0 = false; // resultCode could change nullability.
/* 198 */
/* 199 */ if (!((UTF8String) references[3] /* literal
*/).equals(smj_mutableStateArray_0[5])) {
/* 200 */ // regex value changed
/* 201 */ smj_mutableStateArray_0[5] = ((UTF8String) references[3] /* literal
*/).clone();
/* 202 */ project_mutableStateArray_0[0] =
java.util.regex.Pattern.compile(smj_mutableStateArr",
"ay_0[5].toString());
/* 203 */ }
/* 204 */ if (!((UTF8String) references[4] /* literal
*/).equals(smj_mutableStateArray_0[6])) {
/* 205 */ // replacement string changed
/* 206 */ smj_mutableStateArray_0[6] = ((UTF8String) references[4] /* literal
*/).clone();
/* 207 */ project_mutableStateArray_1[0] =
smj_mutableStateArray_0[6].toString();
/* 208 */ }
/* 209 */ String project_source_0 = smj_mutableStateArray_0[3].toString();
/* 210 */ int project_position_0 = 1 - 1;
/* 211 */ if (project_position_0 < project_source_0.length()) {
/* 212 */ java.lang.StringBuffer project_termResult_0 = new
java.lang.StringBuffer();
/* 213 */ java.util.regex.Matcher project_matcher_0 =
project_mutableStateArray_0[0].matcher(project_source_0);
/* 214 */ project_matcher_0.region(project_position_0,
project_source_0.length());
/* 215 */
/* 216 */ while (project_matcher_0.find()) {
/* 217 */ project_matcher_0.appendReplacement(project_termResult_0,
project_mutableStateArray_1[0]);
/* 218 */ }
/* 219 */ project_matcher_0.appendTail(project_termResult_0);
/* 220 */ project_value_0 =
UTF8String.fromString(project_termResult_0.toString());
/* 221 */ project_termResult_0 = null;
/* 222 */ } else {
/* 223 */ project_value_0 = smj_mutableStateArray_0[3];
/* 224 */ }
/* 225 */ project_isNull_0 = false;
/* 226 */
/* 227 */ }
/* 228 */ project_subExprIsNull_0 = project_isNull_0;
/* 229 */ return project_value_0;
/* 230 */ }
/* 231 */
/* 232 */ private UTF8String smj_GetArrayItem_0(InternalRow smj_leftRow_0) {
/* 233 */ boolean smj_isNull_12 = true;
/* 234 */ UTF8String smj_value_12 = null;
/* 235 */ boolean smj_isNull_13 = true;
/* 236 */ ArrayData smj_value_13 = null;
/* 237 */ boolean smj_isNull_14 = smj_leftRow_0.isNullAt(2);
/* 238 */ UTF8String smj_value_14 = smj_isNull_14 ?
/* 239 */ null : (smj_leftRow_0.getUTF8String(2));
/* 240 */ if (!smj_isNull_14) {
/* 241 */ smj_isNull_13 = false; // resultCode could change nullability.
/* 242 */ smj_value_13 = new
org.apache.spark.sql.catalyst.util.GenericArrayData(smj_value_14.split(((UTF8String)
references[1] /* literal */),-1));
/* 243 */
/* 244 */ }
/* 245 */ if (!smj_isNull_13) {
/* 246 */ smj_isNull_12 = false; // resultCode could change nullability.
/* 247 */
/* 248 */ final int smj_index_0 = (int) 2;
/* 249 */ if (smj_index_0 >= smj_value_13.numElements() || smj_index_0 < 0) {
/* 250 */ smj_isNull_12 = true;
/* 251 */ } else if (smj_value_13.isNullAt(smj_index_0)) {
/* 252 */ smj_isNull_12 = true;
/* 253 */ }
/* 254 */ else {
/* 255 */ smj_value_12 = smj_value_13.getUTF8String(smj_index_0);
/* 256 */ }
/* 257 */
/* 258 */ }
/* 259 */ smj_globalIsNull_0 = smj_isNull_12;
/* 260 */ return smj_value_12;
/* 261 */ }
/* 262 */
/* 263 */ protected void processNext() throws java.io.IOException {
/* 264 */ while (smj_findNextOuterJoinRows_0(smj_leftInput_0,
smj_rightInput_0)) {
/* 265 */ boolean smj_loaded_0 = false;
/* 266 */ smj_isNull_25 = smj_leftRow_0.isNullAt(0);
/* 267 */ smj_mutableStateArray_0[0] = smj_isNull_25 ? null :
(smj_leftRow_0.getUTF8String(0));
/* 268 */ smj_isNull_26 = smj_leftRow_0.isNullAt(1);
/* 269 */ smj_mutableStateArray_0[1] = smj_isNull_26 ? null :
(smj_leftRow_0.getUTF8String(1));
/* 270 */ smj_isNull_27 = smj_leftRow_0.isNullAt(2);
/* 271 */ smj_mutableStateArray_0[2] = smj_isNull_27 ? null :
(smj_leftRow_0.getUTF8String(2));
/* 272 */ smj_isNull_28 = smj_leftRow_0.isNullAt(3);
/* 273 */ smj_mutableStateArray_0[3] = smj_isNull_28 ? null :
(smj_leftRow_0.getUTF8String(3));
/* 274 */ scala.collection.Iterator<UnsafeRow> smj_iterator_0 =
smj_matches_0.generateIterator();
/* 275 */ while (smj_iterator_0.hasNext()) {
/* 276 */ InternalRow smj_rightRow_1 = (InternalRow) smj_iterator_0.next();
/* 277 */ smj_isNull_29 = smj_rightRow_1.isNullAt(0);
/* 278 */ smj_mutableStateArray_0[4] = smj_isNull_29 ? null :
(smj_rightRow_1.getUTF8String(0));
/* 279 */ smj_isNull_30 = false;
/* 280 */ smj_mutableStateArray_1[0] = smj_rightRow_1.getDecimal(1, 38, 0);
/* 281 */
/* 282 */ smj_loaded_0 = true;
/* 283 */ smj_writeJoinRows_0();
/* 284 */ }
/* 285 */ if (!smj_loaded_0) {
/* 286 */ smj_isNull_29 = true;
/* 287 */ smj_isNull_30 = true;
/* 288 */ smj_writeJoinRows_0();
/* 289 */ }
/* 290 */ if (shouldStop()) return;
/* 291 */ }
/* 292 */ ((org.apache.spark.sql.execution.joins.SortMergeJoinExec)
references[5] /* plan */).cleanupResources();
/* 293 */ }
/* 294 */
/* 295 */ private void smj_writeJoinRows_0() throws java.io.IOException {
/* 296 */ ((org.apache.spark.sql.execution.metric.SQLMetric) references[2] /*
numOutputRows */).add(1);
/* 297 */
/* 298 */ // common sub-expressions
/* 299 */
/* 300 */ UTF8String project_subExprValue_0 =
project_subExpr_0(smj_mutableStateArray_0[3], smj_isNull_28);
/* 301 */
/* 302 */ smj_mutableStateArray_2[1].reset();
/* 303 */
/* 304 */ smj_mutableStateArray_2[1].zeroOutNullBytes();
/* 305 */
/* 306 */ if (smj_isNull_25) {
/* 307 */ smj_mutableStateArray_2[1].setNullAt(0);
/* 308 */ } else {
/* 309 */ smj_mutableStateArray_2[1].write(0, smj_mutableStateArray_0[0]);
/* 310 */ }
/* 311 */
/* 312 */ if (smj_isNull_26) {
/* 313 */ smj_mutableStateArray_2[1].setNullAt(1);
/* 314 */ } else {
/* 315 */ smj_mutableStateArray_2[1].write(1, smj_mutableStateArray_0[1]);
/* 316 */ }
/* 317 */
/* 318 */ if (project_subExprIsNull_0) {
/* 319 */ smj_mutableStateArray_2[1].setNullAt(2);
/* 320 */ } else {
/* 321 */ smj_mutableStateArray_2[1].write(2, project_subExprValue_0);
/* 322 */ }
/* 323 */
/* 324 */ if (project_subExprIsNull_0) {
/* 325 */ smj_mutableStateArray_2[1].setNullAt(3);
/* 326 */ } else {
/* 327 */ smj_mutableStateArray_2[1].write(3, project_subExprValue_0);
/* 328 */ }
/* 329 */
/* 330 */ if (smj_isNull_29) {
/* 331 */ smj_mutableStateArray_2[1].setNullAt(4);
/* 332 */ } else {
/* 333 */ smj_mutableStateArray_2[1].write(4, smj_mutableStateArray_0[4]);
/* 334 */ }
/* 335 */
/* 336 */ if (smj_isNull_25) {
/* 337 */ smj_mutableStateArray_2[1].setNullAt(5);
/* 338 */ } else {
/* 339 */ smj_mutableStateArray_2[1].write(5, smj_mutableStateArray_0[0]);
/* 340 */ }
/* 341 */ append((smj_mutableStateArray_2[1].getRow()).copy());
/* 342 */
/* 343 */ }
/* 344 */
/* 345 */ } {code}
> ERROR CodeGenerator: failed to compile:
> org.codehaus.commons.compiler.CompileException: File 'generated.java'
> -------------------------------------------------------------------------------------------------------------
>
> Key: SPARK-36862
> URL: https://issues.apache.org/jira/browse/SPARK-36862
> Project: Spark
> Issue Type: Bug
> Components: Spark Submit, SQL
> Affects Versions: 3.1.1
> Environment: Spark 3.1.1 and Spark 3.1.2
> hadoop 3.2.1
> Reporter: Magdalena Pilawska
> Priority: Major
>
> Hi,
> I am getting the following error running spark-submit command:
> ERROR CodeGenerator: failed to compile:
> org.codehaus.commons.compiler.CompileException: File 'generated.java', Line
> 321, Column 103: ')' expected instead of '['
>
> It fails running the spark sql command on delta lake:
> spark.sql(sqlTransformation)
> The template of sqlTransformation is as follows:
> MERGE INTO target_table AS d
> USING source_table AS s
> on s.id = d.id
> WHEN MATCHED AND d.hash_value <> s.hash_value
> THEN UPDATE SET d.name =s.name, d.address = s.address
>
> It is permanent error both for *spark 3.1.1* version.
>
> The same works fine with spark 3.0.0.
>
> Here is the full log:
> 2021-09-22 16:43:22,110 ERROR CodeGenerator: failed to compile:
> org.codehaus.commons.compiler.CompileException: File 'generated.java', Line
> 55, Column 103: ')' expected instead of '['2021-09-22 16:43:22,110 ERROR
> CodeGenerator: failed to compile:
> org.codehaus.commons.compiler.CompileException: File 'generated.java', Line
> 55, Column 103: ')' expected instead of
> '['org.codehaus.commons.compiler.CompileException: File 'generated.java',
> Line 55, Column 103: ')' expected instead of '[' at
> org.codehaus.janino.TokenStreamImpl.compileException(TokenStreamImpl.java:362)
> at org.codehaus.janino.TokenStreamImpl.read(TokenStreamImpl.java:150) at
> org.codehaus.janino.Parser.read(Parser.java:3703) at
> org.codehaus.janino.Parser.parseFormalParameters(Parser.java:1622) at
> org.codehaus.janino.Parser.parseMethodDeclarationRest(Parser.java:1518) at
> org.codehaus.janino.Parser.parseClassBodyDeclaration(Parser.java:1028) at
> org.codehaus.janino.Parser.parseClassBody(Parser.java:841) at
> org.codehaus.janino.Parser.parseClassDeclarationRest(Parser.java:736) at
> org.codehaus.janino.Parser.parseClassBodyDeclaration(Parser.java:941) at
> org.codehaus.janino.ClassBodyEvaluator.cook(ClassBodyEvaluator.java:234) at
> org.codehaus.janino.SimpleCompiler.cook(SimpleCompiler.java:205) at
> org.codehaus.commons.compiler.Cookable.cook(Cookable.java:80) at
> org.apache.spark.sql.catalyst.expressions.codegen.CodeGenerator$.org$apache$spark$sql$catalyst$expressions$codegen$CodeGenerator$$doCompile(CodeGenerator.scala:1427)
> at
> org.apache.spark.sql.catalyst.expressions.codegen.CodeGenerator$$anon$1.load(CodeGenerator.scala:1524)
> at
> org.apache.spark.sql.catalyst.expressions.codegen.CodeGenerator$$anon$1.load(CodeGenerator.scala:1521)
> at
> org.sparkproject.guava.cache.LocalCache$LoadingValueReference.loadFuture(LocalCache.java:3599)
> at
> org.sparkproject.guava.cache.LocalCache$Segment.loadSync(LocalCache.java:2379)
> at
> org.sparkproject.guava.cache.LocalCache$Segment.lockedGetOrLoad(LocalCache.java:2342)
> at org.sparkproject.guava.cache.LocalCache$Segment.get(LocalCache.java:2257)
> at org.sparkproject.guava.cache.LocalCache.get(LocalCache.java:4000) at
> org.sparkproject.guava.cache.LocalCache.getOrLoad(LocalCache.java:4004) at
> org.sparkproject.guava.cache.LocalCache$LocalLoadingCache.get(LocalCache.java:4874)
> at
> org.apache.spark.sql.catalyst.expressions.codegen.CodeGenerator$.compile(CodeGenerator.scala:1375)
> at
> org.apache.spark.sql.execution.WholeStageCodegenExec.liftedTree1$1(WholeStageCodegenExec.scala:721)
> at
> org.apache.spark.sql.execution.WholeStageCodegenExec.doExecute(WholeStageCodegenExec.scala:720)
> at
> org.apache.spark.sql.execution.SparkPlan.$anonfun$execute$1(SparkPlan.scala:185)
> at
> org.apache.spark.sql.execution.SparkPlan.$anonfun$executeQuery$1(SparkPlan.scala:223)
> at
> org.apache.spark.rdd.RDDOperationScope$.withScope(RDDOperationScope.scala:151)
> at
> org.apache.spark.sql.execution.SparkPlan.executeQuery(SparkPlan.scala:220) at
> org.apache.spark.sql.execution.SparkPlan.execute(SparkPlan.scala:181) at
> org.apache.spark.sql.execution.exchange.ShuffleExchangeExec.inputRDD$lzycompute(ShuffleExchangeExec.scala:160)
> at
> org.apache.spark.sql.execution.exchange.ShuffleExchangeExec.inputRDD(ShuffleExchangeExec.scala:160)
> at
> org.apache.spark.sql.execution.exchange.ShuffleExchangeExec.mapOutputStatisticsFuture$lzycompute(ShuffleExchangeExec.scala:164)
> at
> org.apache.spark.sql.execution.exchange.ShuffleExchangeExec.mapOutputStatisticsFuture(ShuffleExchangeExec.scala:163)
> at
> org.apache.spark.sql.execution.exchange.ShuffleExchangeLike.$anonfun$materializeFuture$2(ShuffleExchangeExec.scala:100)
> at
> org.apache.spark.sql.catalyst.errors.package$.attachTree(package.scala:52) at
> org.apache.spark.sql.execution.exchange.ShuffleExchangeLike.$anonfun$materializeFuture$1(ShuffleExchangeExec.scala:100)
> at org.apache.spark.sql.util.LazyValue.getOrInit(LazyValue.scala:41) at
> org.apache.spark.sql.execution.exchange.Exchange.getOrInitMaterializeFuture(Exchange.scala:68)
> at
> org.apache.spark.sql.execution.exchange.ShuffleExchangeLike.materializeFuture(ShuffleExchangeExec.scala:96)
> at
> org.apache.spark.sql.execution.exchange.ShuffleExchangeLike.materialize(ShuffleExchangeExec.scala:84)
> at
> org.apache.spark.sql.execution.exchange.ShuffleExchangeLike.materialize$(ShuffleExchangeExec.scala:83)
> at
> org.apache.spark.sql.execution.exchange.ShuffleExchangeExec.materialize(ShuffleExchangeExec.scala:128)
> at
> org.apache.spark.sql.execution.adaptive.ShuffleQueryStageExec.doMaterialize(QueryStageExec.scala:161)
> at
> org.apache.spark.sql.execution.adaptive.QueryStageExec.$anonfun$materialize$1(QueryStageExec.scala:74)
> at
> org.apache.spark.sql.execution.SparkPlan.$anonfun$executeQuery$1(SparkPlan.scala:223)
> at
> org.apache.spark.rdd.RDDOperationScope$.withScope(RDDOperationScope.scala:151)
> at
> org.apache.spark.sql.execution.SparkPlan.executeQuery(SparkPlan.scala:220) at
> org.apache.spark.sql.execution.adaptive.QueryStageExec.materialize(QueryStageExec.scala:74)
> at
> org.apache.spark.sql.execution.adaptive.MaterializeExecutable.tryStart(AdaptiveExecutable.scala:396)
> at
> org.apache.spark.sql.execution.adaptive.AdaptiveExecutorRuntime.startChild(AdaptiveExecutor.scala:225)
> at
> org.apache.spark.sql.execution.adaptive.ExecutionHelper.start(ExecutionHelper.scala:47)
> at
> org.apache.spark.sql.execution.adaptive.QueryStageExecutable$$anon$2.$anonfun$new$1(AdaptiveExecutable.scala:251)
> at
> org.apache.spark.sql.execution.adaptive.ExecutionHelper$Listener.$anonfun$onChildSuccess$2(ExecutionHelper.scala:55)
> at
> org.apache.spark.sql.execution.adaptive.ExecutionHelper$Listener.$anonfun$onChildSuccess$2$adapted(ExecutionHelper.scala:54)
> at scala.Option.foreach(Option.scala:407) at
> org.apache.spark.sql.execution.adaptive.ExecutionHelper$Listener.$anonfun$onChildSuccess$1(ExecutionHelper.scala:54)
> at
> org.apache.spark.sql.execution.adaptive.ExecutionHelper$Listener.$anonfun$onChildSuccess$1$adapted(ExecutionHelper.scala:53)
> at scala.collection.mutable.ResizableArray.foreach(ResizableArray.scala:62)
> at scala.collection.mutable.ResizableArray.foreach$(ResizableArray.scala:55)
> at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:49) at
> org.apache.spark.sql.execution.adaptive.ExecutionHelper$Listener.onChildSuccess(ExecutionHelper.scala:53)
> at
> org.apache.spark.sql.execution.adaptive.AdaptiveExecutorRuntime.$anonfun$onActiveChildSuccess$2(AdaptiveExecutor.scala:314)
> at
> org.apache.spark.sql.execution.adaptive.AdaptiveExecutorRuntime.$anonfun$onActiveChildSuccess$2$adapted(AdaptiveExecutor.scala:314)
> at scala.collection.mutable.ResizableArray.foreach(ResizableArray.scala:62)
> at scala.collection.mutable.ResizableArray.foreach$(ResizableArray.scala:55)
> at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:49) at
> org.apache.spark.sql.execution.adaptive.AdaptiveExecutorRuntime.onActiveChildSuccess(AdaptiveExecutor.scala:314)
> at
> org.apache.spark.sql.execution.adaptive.AdaptiveExecutorRuntime.onChildSuccess(AdaptiveExecutor.scala:284)
> at
> org.apache.spark.sql.execution.adaptive.AdaptiveExecutor.$anonfun$doRun$1(AdaptiveExecutor.scala:92)
> at
> org.apache.spark.sql.execution.adaptive.AdaptiveExecutor.$anonfun$doRun$1$adapted(AdaptiveExecutor.scala:91)
> at scala.collection.mutable.HashMap.$anonfun$foreach$1(HashMap.scala:149) at
> scala.collection.mutable.HashTable.foreachEntry(HashTable.scala:237) at
> scala.collection.mutable.HashTable.foreachEntry$(HashTable.scala:230) at
> scala.collection.mutable.HashMap.foreachEntry(HashMap.scala:44) at
> scala.collection.mutable.HashMap.foreach(HashMap.scala:149) at
> org.apache.spark.sql.execution.adaptive.AdaptiveExecutor.doRun(AdaptiveExecutor.scala:91)
> at
> org.apache.spark.sql.execution.adaptive.AdaptiveExecutor.tryRunningAndGetFuture(AdaptiveExecutor.scala:66)
> at
> org.apache.spark.sql.execution.adaptive.AdaptiveExecutor.execute(AdaptiveExecutor.scala:57)
> at
> org.apache.spark.sql.execution.adaptive.AdaptiveSparkPlanExec.$anonfun$getFinalPhysicalPlan$1(AdaptiveSparkPlanExec.scala:184)
> at org.apache.spark.sql.SparkSession.withActive(SparkSession.scala:772) at
> org.apache.spark.sql.execution.adaptive.AdaptiveSparkPlanExec.getFinalPhysicalPlan(AdaptiveSparkPlanExec.scala:183)
> at
> org.apache.spark.sql.execution.adaptive.AdaptiveSparkPlanExec.doExecute(AdaptiveSparkPlanExec.scala:434)
> at
> org.apache.spark.sql.execution.SparkPlan.$anonfun$execute$1(SparkPlan.scala:185)
> at
> org.apache.spark.sql.execution.SparkPlan.$anonfun$executeQuery$1(SparkPlan.scala:223)
> at
> org.apache.spark.rdd.RDDOperationScope$.withScope(RDDOperationScope.scala:151)
> at
> org.apache.spark.sql.execution.SparkPlan.executeQuery(SparkPlan.scala:220) at
> org.apache.spark.sql.execution.SparkPlan.execute(SparkPlan.scala:181) at
> org.apache.spark.sql.delta.constraints.DeltaInvariantCheckerExec.doExecute(DeltaInvariantCheckerExec.scala:78)
> at
> org.apache.spark.sql.execution.SparkPlan.$anonfun$execute$1(SparkPlan.scala:185)
> at
> org.apache.spark.sql.execution.SparkPlan.$anonfun$executeQuery$1(SparkPlan.scala:223)
> at
> org.apache.spark.rdd.RDDOperationScope$.withScope(RDDOperationScope.scala:151)
> at
> org.apache.spark.sql.execution.SparkPlan.executeQuery(SparkPlan.scala:220) at
> org.apache.spark.sql.execution.SparkPlan.execute(SparkPlan.scala:181) at
> org.apache.spark.sql.execution.datasources.FileFormatWriter$.write(FileFormatWriter.scala:177)
> at
> org.apache.spark.sql.delta.files.TransactionalWrite.$anonfun$writeFiles$1(TransactionalWrite.scala:192)
> at
> org.apache.spark.sql.catalyst.QueryPlanningTracker$.withTracker(QueryPlanningTracker.scala:107)
> at
> org.apache.spark.sql.execution.SQLExecution$.withTracker(SQLExecution.scala:232)
> at
> org.apache.spark.sql.execution.SQLExecution$.executeQuery$1(SQLExecution.scala:110)
> at
> org.apache.spark.sql.execution.SQLExecution$.$anonfun$withNewExecutionId$6(SQLExecution.scala:135)
> at
> org.apache.spark.sql.catalyst.QueryPlanningTracker$.withTracker(QueryPlanningTracker.scala:107)
> at
> org.apache.spark.sql.execution.SQLExecution$.withTracker(SQLExecution.scala:232)
> at
> org.apache.spark.sql.execution.SQLExecution$.$anonfun$withNewExecutionId$5(SQLExecution.scala:135)
> at
> org.apache.spark.sql.execution.SQLExecution$.withSQLConfPropagated(SQLExecution.scala:253)
> at
> org.apache.spark.sql.execution.SQLExecution$.$anonfun$withNewExecutionId$1(SQLExecution.scala:134)
> at org.apache.spark.sql.SparkSession.withActive(SparkSession.scala:772) at
> org.apache.spark.sql.execution.SQLExecution$.withNewExecutionId(SQLExecution.scala:68)
> at
> org.apache.spark.sql.delta.files.TransactionalWrite.writeFiles(TransactionalWrite.scala:163)
> at
> org.apache.spark.sql.delta.files.TransactionalWrite.writeFiles$(TransactionalWrite.scala:142)
> at
> org.apache.spark.sql.delta.OptimisticTransaction.writeFiles(OptimisticTransaction.scala:84)
> at
> org.apache.spark.sql.delta.commands.MergeIntoCommand.$anonfun$writeAllChanges$1(MergeIntoCommand.scala:552)
> at
> org.apache.spark.sql.delta.commands.MergeIntoCommand.recordMergeOperation(MergeIntoCommand.scala:654)
> at
> org.apache.spark.sql.delta.commands.MergeIntoCommand.writeAllChanges(MergeIntoCommand.scala:460)
> at
> org.apache.spark.sql.delta.commands.MergeIntoCommand.$anonfun$run$4(MergeIntoCommand.scala:274)
> at
> org.apache.spark.sql.delta.util.DeltaProgressReporter.withJobDescription(DeltaProgressReporter.scala:53)
> at
> org.apache.spark.sql.delta.util.DeltaProgressReporter.withStatusCode(DeltaProgressReporter.scala:32)
> at
> org.apache.spark.sql.delta.util.DeltaProgressReporter.withStatusCode$(DeltaProgressReporter.scala:27)
> at
> org.apache.spark.sql.delta.commands.MergeIntoCommand.withStatusCode(MergeIntoCommand.scala:201)
> at
> org.apache.spark.sql.delta.commands.MergeIntoCommand.$anonfun$run$2(MergeIntoCommand.scala:274)
> at
> org.apache.spark.sql.delta.commands.MergeIntoCommand.$anonfun$run$2$adapted(MergeIntoCommand.scala:255)
> at
> org.apache.spark.sql.delta.DeltaLog.withNewTransaction(DeltaLog.scala:187) at
> org.apache.spark.sql.delta.commands.MergeIntoCommand.$anonfun$run$1(MergeIntoCommand.scala:255)
> at scala.runtime.java8.JFunction0$mcV$sp.apply(JFunction0$mcV$sp.java:23) at
> com.databricks.spark.util.DatabricksLogging.recordOperation(DatabricksLogging.scala:77)
> at
> com.databricks.spark.util.DatabricksLogging.recordOperation$(DatabricksLogging.scala:67)
> at
> org.apache.spark.sql.delta.commands.MergeIntoCommand.recordOperation(MergeIntoCommand.scala:201)
> at
> org.apache.spark.sql.delta.metering.DeltaLogging.recordDeltaOperation(DeltaLogging.scala:106)
> at
> org.apache.spark.sql.delta.metering.DeltaLogging.recordDeltaOperation$(DeltaLogging.scala:91)
> at
> org.apache.spark.sql.delta.commands.MergeIntoCommand.recordDeltaOperation(MergeIntoCommand.scala:201)
> at
> org.apache.spark.sql.delta.commands.MergeIntoCommand.run(MergeIntoCommand.scala:253)
> at
> org.apache.spark.sql.execution.command.ExecutedCommandExec.sideEffectResult$lzycompute(commands.scala:70)
> at
> org.apache.spark.sql.execution.command.ExecutedCommandExec.sideEffectResult(commands.scala:68)
> at
> org.apache.spark.sql.execution.command.ExecutedCommandExec.executeCollect(commands.scala:79)
> at org.apache.spark.sql.Dataset.$anonfun$logicalPlan$1(Dataset.scala:229) at
> org.apache.spark.sql.Dataset.$anonfun$withAction$1(Dataset.scala:3724) at
> org.apache.spark.sql.catalyst.QueryPlanningTracker$.withTracker(QueryPlanningTracker.scala:107)
> at
> org.apache.spark.sql.execution.SQLExecution$.withTracker(SQLExecution.scala:232)
> at
> org.apache.spark.sql.execution.SQLExecution$.executeQuery$1(SQLExecution.scala:110)
> at
> org.apache.spark.sql.execution.SQLExecution$.$anonfun$withNewExecutionId$6(SQLExecution.scala:135)
> at
> org.apache.spark.sql.catalyst.QueryPlanningTracker$.withTracker(QueryPlanningTracker.scala:107)
> at
> org.apache.spark.sql.execution.SQLExecution$.withTracker(SQLExecution.scala:232)
> at
> org.apache.spark.sql.execution.SQLExecution$.$anonfun$withNewExecutionId$5(SQLExecution.scala:135)
> at
> org.apache.spark.sql.execution.SQLExecution$.withSQLConfPropagated(SQLExecution.scala:253)
> at
> org.apache.spark.sql.execution.SQLExecution$.$anonfun$withNewExecutionId$1(SQLExecution.scala:134)
> at org.apache.spark.sql.SparkSession.withActive(SparkSession.scala:772) at
> org.apache.spark.sql.execution.SQLExecution$.withNewExecutionId(SQLExecution.scala:68)
> at org.apache.spark.sql.Dataset.withAction(Dataset.scala:3722) at
> org.apache.spark.sql.Dataset.<init>(Dataset.scala:229) at
> org.apache.spark.sql.Dataset$.$anonfun$ofRows$2(Dataset.scala:100) at
> org.apache.spark.sql.SparkSession.withActive(SparkSession.scala:772) at
> org.apache.spark.sql.Dataset$.ofRows(Dataset.scala:97) at
> org.apache.spark.sql.SparkSession.$anonfun$sql$1(SparkSession.scala:615) at
> org.apache.spark.sql.SparkSession.withActive(SparkSession.scala:772) at
> org.apache.spark.sql.SparkSession.sql(SparkSession.scala:610) at
> com.aup.daab.ds.cf.external.DeltaIntegration.executeDeltaQuery(DeltaIntegration.scala:214)
> at com.aup.daab.ds.cf.TraServs.Step$.ExecuteTransformation(Step.scala:77) at
> com.aup.daab.ds.cf.TraServs.TraServ.$anonfun$transformData$5(TraServ.scala:268)
> at
> com.aup.daab.ds.cf.TraServs.TraServ.$anonfun$transformData$5$adapted(TraServ.scala:240)
> at
> scala.collection.IndexedSeqOptimized.foreach(IndexedSeqOptimized.scala:36) at
> scala.collection.IndexedSeqOptimized.foreach$(IndexedSeqOptimized.scala:33)
> at scala.collection.mutable.ArrayOps$ofRef.foreach(ArrayOps.scala:198) at
> com.aup.daab.ds.cf.TraServs.TraServ.transformData(TraServ.scala:240) at
> com.aup.daab.ds.cf.driver.Driver$.main(Driver.scala:147) at
> com.aup.daab.ds.cf.driver.Driver.main(Driver.scala) at
> sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at
> sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
> at
> sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
> at java.lang.reflect.Method.invoke(Method.java:498) at
> org.apache.spark.deploy.JavaMainApplication.start(SparkApplication.scala:52)
> at
> org.apache.spark.deploy.SparkSubmit.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:959)
> at org.apache.spark.deploy.SparkSubmit.doRunMain$1(SparkSubmit.scala:180) at
> org.apache.spark.deploy.SparkSubmit.submit(SparkSubmit.scala:203) at
> org.apache.spark.deploy.SparkSubmit.doSubmit(SparkSubmit.scala:90) at
> org.apache.spark.deploy.SparkSubmit$$anon$2.doSubmit(SparkSubmit.scala:1038)
> at org.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:1047) at
> org.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala)2021-09-22
> 16:43:22,112 WARN WholeStageCodegenExec: Whole-stage codegen disabled for
> plan (id=9):
--
This message was sent by Atlassian Jira
(v8.20.10#820010)
---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscr...@spark.apache.org
For additional commands, e-mail: issues-h...@spark.apache.org
