Kontinuation commented on code in PR #706:
URL: https://github.com/apache/incubator-sedona/pull/706#discussion_r1010342845
##########
sql/src/main/scala/org/apache/spark/sql/sedona_sql/strategy/join/TraitJoinQueryExec.scala:
##########
@@ -135,27 +129,25 @@ trait TraitJoinQueryExec extends TraitJoinQueryBase {
logDebug(s"Join result has ${matchesRDD.count()} rows")
matchesRDD.mapPartitions { iter =>
- val filtered =
- if (extraCondition.isDefined) {
- val boundCondition = Predicate.create(extraCondition.get,
left.output ++ right.output) // SPARK3 anchor
-// val boundCondition = newPredicate(extraCondition.get, left.output
++ right.output) // SPARK2 anchor
- iter.filter {
- case (l, r) =>
- val leftRow = l.getUserData.asInstanceOf[UnsafeRow]
- val rightRow = r.getUserData.asInstanceOf[UnsafeRow]
- var joiner = GenerateUnsafeRowJoiner.create(left.schema,
right.schema)
- boundCondition.eval(joiner.join(leftRow, rightRow))
- }
- } else {
- iter
- }
+ val joinRow = if (!swappedLeftAndRight) {
+ val joiner = GenerateUnsafeRowJoiner.create(left.schema, right.schema)
+ (l: UnsafeRow, r: UnsafeRow) => joiner.join(l, r)
+ } else {
+ val joiner = GenerateUnsafeRowJoiner.create(right.schema, left.schema)
+ (l: UnsafeRow, r: UnsafeRow) => joiner.join(r, l)
+ }
+
+ val joined = iter.map { case (l, r) =>
+ val leftRow = l.getUserData.asInstanceOf[UnsafeRow]
+ val rightRow = r.getUserData.asInstanceOf[UnsafeRow]
+ joinRow(leftRow, rightRow)
+ }
- filtered.map {
- case (l, r) =>
- val leftRow = l.getUserData.asInstanceOf[UnsafeRow]
- val rightRow = r.getUserData.asInstanceOf[UnsafeRow]
- var joiner = GenerateUnsafeRowJoiner.create(left.schema,
right.schema)
- joiner.join(leftRow, rightRow)
+ extraCondition match {
+ case Some(condition) =>
+ val boundCondition = Predicate.create(condition, output)
Review Comment:
I assume that we don't need to add a SPARK2 anchor here since we'll drop
Spark 2.x support in the next release.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
