This is an automated email from the ASF dual-hosted git repository. gurwls223 pushed a commit to branch branch-3.0 in repository https://gitbox.apache.org/repos/asf/spark.git
The following commit(s) were added to refs/heads/branch-3.0 by this push: new 29ee5cd [SPARK-35687][SQL][TEST] PythonUDFSuite move assume into its methods 29ee5cd is described below commit 29ee5cd6a8a40c25081afe363368cab7c0b74331 Author: ulysses-you <ulyssesyo...@gmail.com> AuthorDate: Wed Jun 9 15:57:56 2021 +0900 [SPARK-35687][SQL][TEST] PythonUDFSuite move assume into its methods ### What changes were proposed in this pull request? Move `assume` into methods at `PythonUDFSuite`. ### Why are the changes needed? When we run Spark test with such command: `./build/mvn -Phadoop-2.7 -Phive -Phive-thriftserver -Pyarn -Pkubernetes clean test` get this exception: ``` PythonUDFSuite: org.apache.spark.sql.execution.python.PythonUDFSuite *** ABORTED *** java.lang.RuntimeException: Unable to load a Suite class that was discovered in the runpath: org.apache.spark.sql.execution.python.PythonUDFSuite at org.scalatest.tools.DiscoverySuite$.getSuiteInstance(DiscoverySuite.scala:81) at org.scalatest.tools.DiscoverySuite.$anonfun$nestedSuites$1(DiscoverySuite.scala:38) at scala.collection.TraversableLike.$anonfun$map$1(TraversableLike.scala:238) at scala.collection.Iterator.foreach(Iterator.scala:941) at scala.collection.Iterator.foreach$(Iterator.scala:941) at scala.collection.AbstractIterator.foreach(Iterator.scala:1429) at scala.collection.IterableLike.foreach(IterableLike.scala:74) at scala.collection.IterableLike.foreach$(IterableLike.scala:73) at scala.collection.AbstractIterable.foreach(Iterable.scala:56) at scala.collection.TraversableLike.map(TraversableLike.scala:238) ``` The test env has no PYSpark module so it failed. ### Does this PR introduce _any_ user-facing change? No ### How was this patch tested? manual Closes #32833 from ulysses-you/SPARK-35687. Authored-by: ulysses-you <ulyssesyo...@gmail.com> Signed-off-by: Hyukjin Kwon <gurwls...@apache.org> (cherry picked from commit 825b62086254ee5edeaf16fccf632674711b1bd8) Signed-off-by: Hyukjin Kwon <gurwls...@apache.org> --- .../scala/org/apache/spark/sql/execution/python/PythonUDFSuite.scala | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/python/PythonUDFSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/python/PythonUDFSuite.scala index 8cf1b7f..45b5720 100644 --- a/sql/core/src/test/scala/org/apache/spark/sql/execution/python/PythonUDFSuite.scala +++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/python/PythonUDFSuite.scala @@ -28,7 +28,6 @@ class PythonUDFSuite extends QueryTest with SharedSparkSession { val scalaTestUDF = TestScalaUDF(name = "scalaUDF") val pythonTestUDF = TestPythonUDF(name = "pyUDF") - assume(shouldTestPythonUDFs) lazy val base = Seq( (Some(1), Some(1)), (Some(1), Some(2)), (Some(2), Some(1)), @@ -36,6 +35,7 @@ class PythonUDFSuite extends QueryTest with SharedSparkSession { (None, Some(1)), (Some(3), None), (None, None)).toDF("a", "b") test("SPARK-28445: PythonUDF as grouping key and aggregate expressions") { + assume(shouldTestPythonUDFs) val df1 = base.groupBy(scalaTestUDF(base("a") + 1)) .agg(scalaTestUDF(base("a") + 1), scalaTestUDF(count(base("b")))) val df2 = base.groupBy(pythonTestUDF(base("a") + 1)) @@ -44,6 +44,7 @@ class PythonUDFSuite extends QueryTest with SharedSparkSession { } test("SPARK-28445: PythonUDF as grouping key and used in aggregate expressions") { + assume(shouldTestPythonUDFs) val df1 = base.groupBy(scalaTestUDF(base("a") + 1)) .agg(scalaTestUDF(base("a") + 1) + 1, scalaTestUDF(count(base("b")))) val df2 = base.groupBy(pythonTestUDF(base("a") + 1)) @@ -52,6 +53,7 @@ class PythonUDFSuite extends QueryTest with SharedSparkSession { } test("SPARK-28445: PythonUDF in aggregate expression has grouping key in its arguments") { + assume(shouldTestPythonUDFs) val df1 = base.groupBy(scalaTestUDF(base("a") + 1)) .agg(scalaTestUDF(scalaTestUDF(base("a") + 1)), scalaTestUDF(count(base("b")))) val df2 = base.groupBy(pythonTestUDF(base("a") + 1)) @@ -60,6 +62,7 @@ class PythonUDFSuite extends QueryTest with SharedSparkSession { } test("SPARK-28445: PythonUDF over grouping key is argument to aggregate function") { + assume(shouldTestPythonUDFs) val df1 = base.groupBy(scalaTestUDF(base("a") + 1)) .agg(scalaTestUDF(scalaTestUDF(base("a") + 1)), scalaTestUDF(count(scalaTestUDF(base("a") + 1)))) --------------------------------------------------------------------- To unsubscribe, e-mail: commits-unsubscr...@spark.apache.org For additional commands, e-mail: commits-h...@spark.apache.org