uros-b commented on code in PR #58201:
URL: https://github.com/apache/spark/pull/58201#discussion_r3831091669
##########
sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/optimizer/FilterPushdownSuite.scala:
##########
@@ -202,6 +202,68 @@ class FilterPushdownSuite extends PlanTest {
comparePlans(optimized, correctAnswer)
}
+ test("SPARK-39481: avoid evaluating scalar Python UDFs twice") {
Review Comment:
Should this be SPARK-39481 or SPARK-47672 (as per the PR title)?
##########
sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/optimizer/FilterPushdownSuite.scala:
##########
@@ -202,6 +202,68 @@ class FilterPushdownSuite extends PlanTest {
comparePlans(optimized, correctAnswer)
}
+ test("SPARK-39481: avoid evaluating scalar Python UDFs twice") {
+ val evalTypes = Seq(
+ "regular" -> PythonEvalType.SQL_BATCHED_UDF,
+ "Arrow-optimized" -> PythonEvalType.SQL_ARROW_BATCHED_UDF,
+ "Arrow element-wise" -> PythonEvalType.SQL_ARROW_ELEMENTWISE_UDF,
+ "scalar Pandas element-wise" ->
PythonEvalType.SQL_SCALAR_PANDAS_ELEMENTWISE_UDF,
+ "scalar Pandas iterator element-wise" ->
+ PythonEvalType.SQL_SCALAR_PANDAS_ITER_ELEMENTWISE_UDF,
+ "scalar Arrow element-wise" ->
PythonEvalType.SQL_SCALAR_ARROW_ELEMENTWISE_UDF,
+ "scalar Arrow iterator element-wise" ->
+ PythonEvalType.SQL_SCALAR_ARROW_ITER_ELEMENTWISE_UDF,
+ "scalar Pandas" -> PythonEvalType.SQL_SCALAR_PANDAS_UDF,
+ "scalar Pandas iterator" -> PythonEvalType.SQL_SCALAR_PANDAS_ITER_UDF,
+ "scalar Arrow" -> PythonEvalType.SQL_SCALAR_ARROW_UDF,
+ "scalar Arrow iterator" -> PythonEvalType.SQL_SCALAR_ARROW_ITER_UDF)
+
+ evalTypes.foreach { case (name, evalType) =>
+ withClue(s"$name Python UDF: ") {
+ val pythonUDF = PythonUDF(
+ "pythonUDF",
+ null,
+ BooleanType,
+ Seq(attrA),
+ evalType,
+ udfDeterministic = true)
+ val originalQuery = testRelation
+ .select(pythonUDF.as("result"))
+ .where($"result")
+ .analyze
+
+ comparePlans(Optimize.execute(originalQuery), originalQuery)
+ }
+ }
+ }
+
+ test("SPARK-39481: all Python function expressions are expensive") {
Review Comment:
Should this be SPARK-39481 or SPARK-47672 (as per the PR title)?
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
