This is an automated email from the ASF dual-hosted git repository. dongjoon pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/spark.git
The following commit(s) were added to refs/heads/master by this push: new a5db9b976d8 [SPARK-42410][CONNECT][TESTS] Support Scala 2.12/2.13 tests in `connect` module a5db9b976d8 is described below commit a5db9b976d8f37d22f8a91f461c05cbb20601d8a Author: Dongjoon Hyun <dongj...@apache.org> AuthorDate: Sun Feb 12 17:31:26 2023 -0800 [SPARK-42410][CONNECT][TESTS] Support Scala 2.12/2.13 tests in `connect` module ### What changes were proposed in this pull request? This PR aims to support both Scala 2.12/13 tests in `connect` module by splitting the golden files. ### Why are the changes needed? After #39933, Scala 2.13 CIs are broken . - **master**: https://github.com/apache/spark/actions/runs/4157806226/jobs/7192575602 - **branch-3.4**: https://github.com/apache/spark/actions/runs/4155578848/jobs/7188777977 ``` [info] - function_udf *** FAILED *** (29 milliseconds) [info] java.io.InvalidClassException: org.apache.spark.sql.TestUDFs$$anon$1; local class incompatible: stream classdesc serialVersionUID = 505010451380771093, local class serialVersionUID = 643575318841761245 ``` ### Does this PR introduce _any_ user-facing change? No. ### How was this patch tested? Pass the CIs and do Scala 2.13 tests manually. ``` $ dev/change-scala-version.sh 2.13 $ build/sbt -Dscala.version=2.13.8 -Pscala-2.13 -Phadoop-3 assembly/package "connect/test" ... [info] - function_udf_2.13 (19 milliseconds) [info] Run completed in 8 seconds, 964 milliseconds. [info] Total number of tests run: 119 [info] Suites: completed 8, aborted 0 [info] Tests: succeeded 119, failed 0, canceled 0, ignored 0, pending 0 [info] All tests passed. [success] Total time: 18 s, completed Feb 12, 2023, 3:33:40 PM ``` Closes #39982 from dongjoon-hyun/SPARK-42410. Authored-by: Dongjoon Hyun <dongj...@apache.org> Signed-off-by: Dongjoon Hyun <dongj...@apache.org> --- ...ction_udf.explain => function_udf_2.12.explain} | 0 ...ction_udf.explain => function_udf_2.13.explain} | 0 .../{function_udf.json => function_udf_2.12.json} | 0 ...n_udf.proto.bin => function_udf_2.12.proto.bin} | Bin .../query-tests/queries/function_udf_2.13.json | 96 +++++++++++++++++++++ .../queries/function_udf_2.13.proto.bin | Bin 0 -> 12092 bytes .../sql/connect/ProtoToParsedPlanTestSuite.scala | 5 ++ 7 files changed, 101 insertions(+) diff --git a/connector/connect/common/src/test/resources/query-tests/explain-results/function_udf.explain b/connector/connect/common/src/test/resources/query-tests/explain-results/function_udf_2.12.explain similarity index 100% copy from connector/connect/common/src/test/resources/query-tests/explain-results/function_udf.explain copy to connector/connect/common/src/test/resources/query-tests/explain-results/function_udf_2.12.explain diff --git a/connector/connect/common/src/test/resources/query-tests/explain-results/function_udf.explain b/connector/connect/common/src/test/resources/query-tests/explain-results/function_udf_2.13.explain similarity index 100% rename from connector/connect/common/src/test/resources/query-tests/explain-results/function_udf.explain rename to connector/connect/common/src/test/resources/query-tests/explain-results/function_udf_2.13.explain diff --git a/connector/connect/common/src/test/resources/query-tests/queries/function_udf.json b/connector/connect/common/src/test/resources/query-tests/queries/function_udf_2.12.json similarity index 100% rename from connector/connect/common/src/test/resources/query-tests/queries/function_udf.json rename to connector/connect/common/src/test/resources/query-tests/queries/function_udf_2.12.json diff --git a/connector/connect/common/src/test/resources/query-tests/queries/function_udf.proto.bin b/connector/connect/common/src/test/resources/query-tests/queries/function_udf_2.12.proto.bin similarity index 100% rename from connector/connect/common/src/test/resources/query-tests/queries/function_udf.proto.bin rename to connector/connect/common/src/test/resources/query-tests/queries/function_udf_2.12.proto.bin diff --git a/connector/connect/common/src/test/resources/query-tests/queries/function_udf_2.13.json b/connector/connect/common/src/test/resources/query-tests/queries/function_udf_2.13.json new file mode 100644 index 00000000000..23aeb078543 --- /dev/null +++ b/connector/connect/common/src/test/resources/query-tests/queries/function_udf_2.13.json @@ -0,0 +1,96 @@ +{ + "project": { + "input": { + "localRelation": { + "schema": "struct\u003cid:bigint,a:int,b:double\u003e" + } + }, + "expressions": [{ + "commonInlineUserDefinedFunction": { + "scalarScalaUdf": { + "payload": "rO0ABXNyAC1vcmcuYXBhY2hlLnNwYXJrLnNxbC5jb25uZWN0LmNvbW1vbi5VZGZQYWNrZXR7DRDpFmMz1QIAA0wACGZ1bmN0aW9udAASTGphdmEvbGFuZy9PYmplY3Q7TAANaW5wdXRFbmNvZGVyc3QAIExzY2FsYS9jb2xsZWN0aW9uL2ltbXV0YWJsZS9TZXE7TAANb3V0cHV0RW5jb2RlcnQAOExvcmcvYXBhY2hlL3NwYXJrL3NxbC9jYXRhbHlzdC9lbmNvZGVycy9BZ25vc3RpY0VuY29kZXI7eHBzcgAlb3JnLmFwYWNoZS5zcGFyay5zcWwuVGVzdFVERnMkJGFub24kMQjucFe5xZ3dAgAAeHBzcgAyc2NhbGEuY29sbGVjdGlvbi5nZW5lcmljLkRlZmF1bHRTZXJpYWxpemF0aW9uUHJveHkAAAAAAAAAAwMAAUwAB2ZhY3Rvcn [...] + } + } + }, { + "commonInlineUserDefinedFunction": { + "functionName": "foo", + "deterministic": true, + "arguments": [{ + "unresolvedAttribute": { + "unparsedIdentifier": "id" + } + }], + "scalarScalaUdf": { + "payload": "rO0ABXNyAC1vcmcuYXBhY2hlLnNwYXJrLnNxbC5jb25uZWN0LmNvbW1vbi5VZGZQYWNrZXR7DRDpFmMz1QIAA0wACGZ1bmN0aW9udAASTGphdmEvbGFuZy9PYmplY3Q7TAANaW5wdXRFbmNvZGVyc3QAIExzY2FsYS9jb2xsZWN0aW9uL2ltbXV0YWJsZS9TZXE7TAANb3V0cHV0RW5jb2RlcnQAOExvcmcvYXBhY2hlL3NwYXJrL3NxbC9jYXRhbHlzdC9lbmNvZGVycy9BZ25vc3RpY0VuY29kZXI7eHBzcgAlb3JnLmFwYWNoZS5zcGFyay5zcWwuVGVzdFVERnMkJGFub24kMmifcExfceVGAgAAeHBzcgApc2NhbGEuY29sbGVjdGlvbi5pbW11dGFibGUuQXJyYXlTZXEkb2ZSZWYAAAAAAAAAAwIAAVsAC3Vuc2FmZUFycmF5dAATW0 [...] + "nullable": true + } + } + }, { + "commonInlineUserDefinedFunction": { + "functionName": "f3", + "deterministic": true, + "arguments": [{ + "unresolvedAttribute": { + "unparsedIdentifier": "id" + } + }, { + "unresolvedAttribute": { + "unparsedIdentifier": "id" + } + }], + "scalarScalaUdf": { + "payload": "rO0ABXNyAC1vcmcuYXBhY2hlLnNwYXJrLnNxbC5jb25uZWN0LmNvbW1vbi5VZGZQYWNrZXR7DRDpFmMz1QIAA0wACGZ1bmN0aW9udAASTGphdmEvbGFuZy9PYmplY3Q7TAANaW5wdXRFbmNvZGVyc3QAIExzY2FsYS9jb2xsZWN0aW9uL2ltbXV0YWJsZS9TZXE7TAANb3V0cHV0RW5jb2RlcnQAOExvcmcvYXBhY2hlL3NwYXJrL3NxbC9jYXRhbHlzdC9lbmNvZGVycy9BZ25vc3RpY0VuY29kZXI7eHBzcgAlb3JnLmFwYWNoZS5zcGFyay5zcWwuVGVzdFVERnMkJGFub24kM6Rv/pH6bPiLAgAAeHBzcgApc2NhbGEuY29sbGVjdGlvbi5pbW11dGFibGUuQXJyYXlTZXEkb2ZSZWYAAAAAAAAAAwIAAVsAC3Vuc2FmZUFycmF5dAATW0 [...] + "nullable": true + } + } + }, { + "commonInlineUserDefinedFunction": { + "functionName": "bar", + "deterministic": true, + "arguments": [{ + "unresolvedAttribute": { + "unparsedIdentifier": "id" + } + }, { + "unresolvedAttribute": { + "unparsedIdentifier": "id" + } + }, { + "unresolvedAttribute": { + "unparsedIdentifier": "id" + } + }], + "scalarScalaUdf": { + "payload": "rO0ABXNyAC1vcmcuYXBhY2hlLnNwYXJrLnNxbC5jb25uZWN0LmNvbW1vbi5VZGZQYWNrZXR7DRDpFmMz1QIAA0wACGZ1bmN0aW9udAASTGphdmEvbGFuZy9PYmplY3Q7TAANaW5wdXRFbmNvZGVyc3QAIExzY2FsYS9jb2xsZWN0aW9uL2ltbXV0YWJsZS9TZXE7TAANb3V0cHV0RW5jb2RlcnQAOExvcmcvYXBhY2hlL3NwYXJrL3NxbC9jYXRhbHlzdC9lbmNvZGVycy9BZ25vc3RpY0VuY29kZXI7eHBzcgAlb3JnLmFwYWNoZS5zcGFyay5zcWwuVGVzdFVERnMkJGFub24kNAv76myB5zZ4AgAAeHBzcgApc2NhbGEuY29sbGVjdGlvbi5pbW11dGFibGUuQXJyYXlTZXEkb2ZSZWYAAAAAAAAAAwIAAVsAC3Vuc2FmZUFycmF5dAATW0 [...] + "nullable": true + } + } + }, { + "commonInlineUserDefinedFunction": { + "functionName": "f_four", + "deterministic": true, + "arguments": [{ + "unresolvedAttribute": { + "unparsedIdentifier": "id" + } + }, { + "unresolvedAttribute": { + "unparsedIdentifier": "id" + } + }, { + "unresolvedAttribute": { + "unparsedIdentifier": "id" + } + }, { + "unresolvedAttribute": { + "unparsedIdentifier": "id" + } + }], + "scalarScalaUdf": { + "payload": "rO0ABXNyAC1vcmcuYXBhY2hlLnNwYXJrLnNxbC5jb25uZWN0LmNvbW1vbi5VZGZQYWNrZXR7DRDpFmMz1QIAA0wACGZ1bmN0aW9udAASTGphdmEvbGFuZy9PYmplY3Q7TAANaW5wdXRFbmNvZGVyc3QAIExzY2FsYS9jb2xsZWN0aW9uL2ltbXV0YWJsZS9TZXE7TAANb3V0cHV0RW5jb2RlcnQAOExvcmcvYXBhY2hlL3NwYXJrL3NxbC9jYXRhbHlzdC9lbmNvZGVycy9BZ25vc3RpY0VuY29kZXI7eHBzcgAlb3JnLmFwYWNoZS5zcGFyay5zcWwuVGVzdFVERnMkJGFub24kNbf+MdXT5ejEAgAAeHBzcgApc2NhbGEuY29sbGVjdGlvbi5pbW11dGFibGUuQXJyYXlTZXEkb2ZSZWYAAAAAAAAAAwIAAVsAC3Vuc2FmZUFycmF5dAATW0 [...] + "nullable": true + } + } + }] + } +} \ No newline at end of file diff --git a/connector/connect/common/src/test/resources/query-tests/queries/function_udf_2.13.proto.bin b/connector/connect/common/src/test/resources/query-tests/queries/function_udf_2.13.proto.bin new file mode 100644 index 00000000000..d7b3c84579c Binary files /dev/null and b/connector/connect/common/src/test/resources/query-tests/queries/function_udf_2.13.proto.bin differ diff --git a/connector/connect/server/src/test/scala/org/apache/spark/sql/connect/ProtoToParsedPlanTestSuite.scala b/connector/connect/server/src/test/scala/org/apache/spark/sql/connect/ProtoToParsedPlanTestSuite.scala index 8f47b7b7843..1626d9278ff 100644 --- a/connector/connect/server/src/test/scala/org/apache/spark/sql/connect/ProtoToParsedPlanTestSuite.scala +++ b/connector/connect/server/src/test/scala/org/apache/spark/sql/connect/ProtoToParsedPlanTestSuite.scala @@ -21,6 +21,7 @@ import java.nio.file.{Files, FileVisitResult, Path, SimpleFileVisitor} import java.nio.file.attribute.BasicFileAttributes import scala.util.{Failure, Success, Try} +import scala.util.Properties.versionNumberString import org.apache.spark.SparkFunSuite import org.apache.spark.connect.proto @@ -74,6 +75,10 @@ class ProtoToParsedPlanTestSuite extends SparkFunSuite with SharedSparkSession { private def createTest(file: Path): Unit = { val relativePath = inputFilePath.relativize(file) val fileName = relativePath.getFileName.toString + if ((versionNumberString.startsWith("2.12") && fileName.endsWith("2.13.proto.bin")) || + (versionNumberString.startsWith("2.13") && fileName.endsWith("2.12.proto.bin"))) { + return + } if (!fileName.endsWith(".proto.bin")) { logError(s"Skipping $fileName") return --------------------------------------------------------------------- To unsubscribe, e-mail: commits-unsubscr...@spark.apache.org For additional commands, e-mail: commits-h...@spark.apache.org