c21 commented on a change in pull request #32476: URL: https://github.com/apache/spark/pull/32476#discussion_r628977186
########## File path: sql/core/src/main/scala/org/apache/spark/sql/execution/joins/SortMergeJoinExec.scala ########## @@ -418,115 +443,140 @@ case class SortMergeJoinExec( // Inline mutable state since not many join operations in a task val matches = ctx.addMutableState(clsName, "matches", v => s"$v = new $clsName($inMemoryThreshold, $spillThreshold);", forceInline = true) - // Copy the left keys as class members so they could be used in next function call. - val matchedKeyVars = copyKeys(ctx, leftKeyVars) + // Copy the streamed keys as class members so they could be used in next function call. + val matchedKeyVars = copyKeys(ctx, streamedKeyVars) + + // Handle the case when streamed rows has any NULL keys. + val handleStreamedAnyNull = joinType match { + case _: InnerLike => + // Skip streamed row. + s""" + |$streamedRow = null; + |continue; + """.stripMargin + case LeftOuter | RightOuter => + // Eagerly return streamed row. + s""" + |if (!$matches.isEmpty()) { + | $matches.clear(); + |} + |return false; + """.stripMargin + case x => + throw new IllegalArgumentException( + s"SortMergeJoin.genScanner should not take $x as the JoinType") + } - ctx.addNewFunction("findNextInnerJoinRows", + // Handle the case when streamed keys less than buffered keys. + val handleStreamedLessThanBuffered = joinType match { + case _: InnerLike => + // Skip streamed row. + s"$streamedRow = null;" + case LeftOuter | RightOuter => + // Eagerly return with streamed row. + "return false;" + case x => + throw new IllegalArgumentException( + s"SortMergeJoin.genScanner should not take $x as the JoinType") + } + + ctx.addNewFunction("findNextJoinRows", s""" - |private boolean findNextInnerJoinRows( - | scala.collection.Iterator leftIter, - | scala.collection.Iterator rightIter) { - | $leftRow = null; + |private boolean findNextJoinRows( + | scala.collection.Iterator streamedIter, + | scala.collection.Iterator bufferedIter) { + | $streamedRow = null; | int comp = 0; - | while ($leftRow == null) { - | if (!leftIter.hasNext()) return false; - | $leftRow = (InternalRow) leftIter.next(); - | ${leftKeyVars.map(_.code).mkString("\n")} - | if ($leftAnyNull) { - | $leftRow = null; - | continue; + | while ($streamedRow == null) { + | if (!streamedIter.hasNext()) return false; + | $streamedRow = (InternalRow) streamedIter.next(); + | ${streamedKeyVars.map(_.code).mkString("\n")} + | if ($streamedAnyNull) { + | $handleStreamedAnyNull | } | if (!$matches.isEmpty()) { - | ${genComparison(ctx, leftKeyVars, matchedKeyVars)} + | ${genComparison(ctx, streamedKeyVars, matchedKeyVars)} | if (comp == 0) { | return true; | } | $matches.clear(); | } | | do { - | if ($rightRow == null) { - | if (!rightIter.hasNext()) { + | if ($bufferedRow == null) { + | if (!bufferedIter.hasNext()) { | ${matchedKeyVars.map(_.code).mkString("\n")} | return !$matches.isEmpty(); | } - | $rightRow = (InternalRow) rightIter.next(); - | ${rightKeyTmpVars.map(_.code).mkString("\n")} - | if ($rightAnyNull) { - | $rightRow = null; + | $bufferedRow = (InternalRow) bufferedIter.next(); + | ${bufferedKeyTmpVars.map(_.code).mkString("\n")} + | if ($bufferedAnyNull) { + | $bufferedRow = null; | continue; | } - | ${rightKeyVars.map(_.code).mkString("\n")} + | ${bufferedKeyVars.map(_.code).mkString("\n")} | } - | ${genComparison(ctx, leftKeyVars, rightKeyVars)} + | ${genComparison(ctx, streamedKeyVars, bufferedKeyVars)} | if (comp > 0) { - | $rightRow = null; + | $bufferedRow = null; | } else if (comp < 0) { | if (!$matches.isEmpty()) { | ${matchedKeyVars.map(_.code).mkString("\n")} | return true; + | } else { + | $handleStreamedLessThanBuffered | } - | $leftRow = null; | } else { - | $matches.add((UnsafeRow) $rightRow); - | $rightRow = null; + | $matches.add((UnsafeRow) $bufferedRow); + | $bufferedRow = null; | } - | } while ($leftRow != null); + | } while ($streamedRow != null); | } | return false; // unreachable Review comment: @maropu - sounds good to me. Will update. -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. For queries about this service, please contact Infrastructure at: us...@infra.apache.org --------------------------------------------------------------------- To unsubscribe, e-mail: reviews-unsubscr...@spark.apache.org For additional commands, e-mail: reviews-h...@spark.apache.org