This is an automated email from the ASF dual-hosted git repository. yangjie01 pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/spark.git
The following commit(s) were added to refs/heads/master by this push: new 10f31904636 [SPARK-45029][CONNECT][TESTS] Ignore `from_protobuf messageClassName/from_protobuf messageClassName options` in `PlanGenerationTestSuite` 10f31904636 is described below commit 10f31904636da727e439d5f1792c3ab7e8e1d24b Author: yangjie01 <yangji...@baidu.com> AuthorDate: Fri Sep 1 10:52:44 2023 +0800 [SPARK-45029][CONNECT][TESTS] Ignore `from_protobuf messageClassName/from_protobuf messageClassName options` in `PlanGenerationTestSuite` ### What changes were proposed in this pull request? This pr aims ignore `from_protobuf messageClassName` and `from_protobuf messageClassName options` in `PlanGenerationTestSuite` and remove the related golden files, after this change `from_protobuf_messageClassName` and `from_protobuf_messageClassName_options` in `ProtoToParsedPlanTestSuite` be ignored too. ### Why are the changes needed? SPARK-43646 | (https://github.com/apache/spark/pull/42236) makes both Maven and SBT use the shaded `spark-protobuf` module when testing the connect module, this allows `mvn clean install` and `mvn package test` to successfully pass tests. But if `mvn clean test` is executed directly, an error `package org.sparkproject.spark_protobuf.protobuf does not exist` will occur. This is because `mvn clean test` directly uses the classes file of the `spark-protobuf` module for testing, without t [...] On the other hand, the change of SPARK-43646 breaks the usability of importing Spark as a Maven project into IDEA(https://github.com/apache/spark/pull/42236#issuecomment-1700493815). So https://github.com/apache/spark/pull/42746 revert the change of [SPARK-43646](https://issues.apache.org/jira/browse/SPARK-43646). It's difficult to find a perfect solution to solve this maven test issues now, as in certain scenarios tests would use the `shaded spark-protobuf jar`, like `mvn package test`, while in some other scenarios it will use the `unshaded classes directory`, such as `mvn clean test`. so this pr ignores the relevant tests first and leaves a TODO(SPARK-45030), to re-enable these tests when we find a better solution. ### Does this PR introduce _any_ user-facing change? No ### How was this patch tested? Pass Github Actions ### Was this patch authored or co-authored using generative AI tooling? No Closes #42751 from LuciferYang/SPARK-45029. Authored-by: yangjie01 <yangji...@baidu.com> Signed-off-by: yangjie01 <yangji...@baidu.com> --- .../apache/spark/sql/PlanGenerationTestSuite.scala | 8 +++- .../from_protobuf_messageClassName.explain | 2 - .../from_protobuf_messageClassName_options.explain | 2 - .../queries/from_protobuf_messageClassName.json | 29 -------------- .../from_protobuf_messageClassName.proto.bin | Bin 125 -> 0 bytes .../from_protobuf_messageClassName_options.json | 42 --------------------- ...rom_protobuf_messageClassName_options.proto.bin | Bin 174 -> 0 bytes 7 files changed, 6 insertions(+), 77 deletions(-) diff --git a/connector/connect/client/jvm/src/test/scala/org/apache/spark/sql/PlanGenerationTestSuite.scala b/connector/connect/client/jvm/src/test/scala/org/apache/spark/sql/PlanGenerationTestSuite.scala index ccd68f75bda..c457f269213 100644 --- a/connector/connect/client/jvm/src/test/scala/org/apache/spark/sql/PlanGenerationTestSuite.scala +++ b/connector/connect/client/jvm/src/test/scala/org/apache/spark/sql/PlanGenerationTestSuite.scala @@ -3235,11 +3235,15 @@ class PlanGenerationTestSuite private val testDescFilePath: String = s"${IntegrationTestUtils.sparkHome}/connector/" + "connect/common/src/test/resources/protobuf-tests/common.desc" - test("from_protobuf messageClassName") { + // TODO(SPARK-45030): Re-enable this test when all Maven test scenarios succeed and there + // are no other negative impacts. For the problem description, please refer to SPARK-45029 + ignore("from_protobuf messageClassName") { binary.select(pbFn.from_protobuf(fn.col("bytes"), classOf[StorageLevel].getName)) } - test("from_protobuf messageClassName options") { + // TODO(SPARK-45030): Re-enable this test when all Maven test scenarios succeed and there + // are no other negative impacts. For the problem description, please refer to SPARK-45029 + ignore("from_protobuf messageClassName options") { binary.select( pbFn.from_protobuf( fn.col("bytes"), diff --git a/connector/connect/common/src/test/resources/query-tests/explain-results/from_protobuf_messageClassName.explain b/connector/connect/common/src/test/resources/query-tests/explain-results/from_protobuf_messageClassName.explain deleted file mode 100644 index e7a1867fe90..00000000000 --- a/connector/connect/common/src/test/resources/query-tests/explain-results/from_protobuf_messageClassName.explain +++ /dev/null @@ -1,2 +0,0 @@ -Project [from_protobuf(bytes#0, org.apache.spark.connect.proto.StorageLevel, None) AS from_protobuf(bytes)#0] -+- LocalRelation <empty>, [id#0L, bytes#0] diff --git a/connector/connect/common/src/test/resources/query-tests/explain-results/from_protobuf_messageClassName_options.explain b/connector/connect/common/src/test/resources/query-tests/explain-results/from_protobuf_messageClassName_options.explain deleted file mode 100644 index c02d829fcac..00000000000 --- a/connector/connect/common/src/test/resources/query-tests/explain-results/from_protobuf_messageClassName_options.explain +++ /dev/null @@ -1,2 +0,0 @@ -Project [from_protobuf(bytes#0, org.apache.spark.connect.proto.StorageLevel, None, (recursive.fields.max.depth,2)) AS from_protobuf(bytes)#0] -+- LocalRelation <empty>, [id#0L, bytes#0] diff --git a/connector/connect/common/src/test/resources/query-tests/queries/from_protobuf_messageClassName.json b/connector/connect/common/src/test/resources/query-tests/queries/from_protobuf_messageClassName.json deleted file mode 100644 index dc23ac2a117..00000000000 --- a/connector/connect/common/src/test/resources/query-tests/queries/from_protobuf_messageClassName.json +++ /dev/null @@ -1,29 +0,0 @@ -{ - "common": { - "planId": "1" - }, - "project": { - "input": { - "common": { - "planId": "0" - }, - "localRelation": { - "schema": "struct\u003cid:bigint,bytes:binary\u003e" - } - }, - "expressions": [{ - "unresolvedFunction": { - "functionName": "from_protobuf", - "arguments": [{ - "unresolvedAttribute": { - "unparsedIdentifier": "bytes" - } - }, { - "literal": { - "string": "org.apache.spark.connect.proto.StorageLevel" - } - }] - } - }] - } -} \ No newline at end of file diff --git a/connector/connect/common/src/test/resources/query-tests/queries/from_protobuf_messageClassName.proto.bin b/connector/connect/common/src/test/resources/query-tests/queries/from_protobuf_messageClassName.proto.bin deleted file mode 100644 index cc46234b747..00000000000 Binary files a/connector/connect/common/src/test/resources/query-tests/queries/from_protobuf_messageClassName.proto.bin and /dev/null differ diff --git a/connector/connect/common/src/test/resources/query-tests/queries/from_protobuf_messageClassName_options.json b/connector/connect/common/src/test/resources/query-tests/queries/from_protobuf_messageClassName_options.json deleted file mode 100644 index 36f69646ef8..00000000000 --- a/connector/connect/common/src/test/resources/query-tests/queries/from_protobuf_messageClassName_options.json +++ /dev/null @@ -1,42 +0,0 @@ -{ - "common": { - "planId": "1" - }, - "project": { - "input": { - "common": { - "planId": "0" - }, - "localRelation": { - "schema": "struct\u003cid:bigint,bytes:binary\u003e" - } - }, - "expressions": [{ - "unresolvedFunction": { - "functionName": "from_protobuf", - "arguments": [{ - "unresolvedAttribute": { - "unparsedIdentifier": "bytes" - } - }, { - "literal": { - "string": "org.apache.spark.connect.proto.StorageLevel" - } - }, { - "unresolvedFunction": { - "functionName": "map", - "arguments": [{ - "literal": { - "string": "recursive.fields.max.depth" - } - }, { - "literal": { - "string": "2" - } - }] - } - }] - } - }] - } -} \ No newline at end of file diff --git a/connector/connect/common/src/test/resources/query-tests/queries/from_protobuf_messageClassName_options.proto.bin b/connector/connect/common/src/test/resources/query-tests/queries/from_protobuf_messageClassName_options.proto.bin deleted file mode 100644 index 72a1c6b8207..00000000000 Binary files a/connector/connect/common/src/test/resources/query-tests/queries/from_protobuf_messageClassName_options.proto.bin and /dev/null differ --------------------------------------------------------------------- To unsubscribe, e-mail: commits-unsubscr...@spark.apache.org For additional commands, e-mail: commits-h...@spark.apache.org