This is an automated email from the ASF dual-hosted git repository. gurwls223 pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/spark.git
The following commit(s) were added to refs/heads/master by this push: new 8cf82700a4f [SPARK-41177][PROTOBUF][TESTS] Fix maven test failed of `protobuf` module 8cf82700a4f is described below commit 8cf82700a4f09e54b3d872bf4902a41df5a151cb Author: yangjie01 <yangji...@baidu.com> AuthorDate: Fri Nov 18 11:37:48 2022 +0900 [SPARK-41177][PROTOBUF][TESTS] Fix maven test failed of `protobuf` module ### What changes were proposed in this pull request? Maven and sbt use different protobuf plugin, this causes the contents of `connector/protobuf/src/test/resources/protobuf` to be compiled into different directories in `scala-2.12/test-classes` by different tools: - Maven : compiled to `connector/protobuf/target/scala-2.12/test-classes/protobuf ` - sbt: compiled to `connector/protobuf/target/scala-2.12/test-classes/` The files in the `resources/protobuf` directory are read in the test case of the `protobuf` module, which can pass with sbt test, but will fail in the maven test because the `protobuf/` prefix should be added when using maven. So this pr add a helper method to passing in two alternative directories to make maven test also successful. ### Why are the changes needed? Fix maven test failed of `protobuf` module. ### Does this PR introduce _any_ user-facing change? No, just for test ### How was this patch tested? - Pass GA - Manual test ``` mvn clean install -pl connector/protobuf -am -DskipTests mvn test -pl connector/protobuf ``` **Before** ``` ProtobufSerdeSuite: org.apache.spark.sql.protobuf.ProtobufSerdeSuite *** ABORTED *** java.lang.RuntimeException: Unable to load a Suite class org.apache.spark.sql.protobuf.ProtobufSerdeSuite that was discovered in the runpath: null at org.scalatest.tools.DiscoverySuite$.getSuiteInstance(DiscoverySuite.scala:80) at org.scalatest.tools.DiscoverySuite.$anonfun$nestedSuites$1(DiscoverySuite.scala:38) at scala.collection.TraversableLike.$anonfun$map$1(TraversableLike.scala:286) at scala.collection.Iterator.foreach(Iterator.scala:943) at scala.collection.Iterator.foreach$(Iterator.scala:943) at scala.collection.AbstractIterator.foreach(Iterator.scala:1431) at scala.collection.IterableLike.foreach(IterableLike.scala:74) at scala.collection.IterableLike.foreach$(IterableLike.scala:73) at scala.collection.AbstractIterable.foreach(Iterable.scala:56) at scala.collection.TraversableLike.map(TraversableLike.scala:286) ... Cause: java.lang.NullPointerException: at org.apache.spark.sql.test.SQLTestUtilsBase.testFile(SQLTestUtils.scala:466) at org.apache.spark.sql.test.SQLTestUtilsBase.testFile$(SQLTestUtils.scala:465) at org.apache.spark.sql.protobuf.ProtobufSerdeSuite.testFile(ProtobufSerdeSuite.scala:34) at org.apache.spark.sql.protobuf.ProtobufSerdeSuite.<init>(ProtobufSerdeSuite.scala:39) at sun.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) at sun.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:62) at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) at java.lang.reflect.Constructor.newInstance(Constructor.java:423) at java.lang.Class.newInstance(Class.java:442) at org.scalatest.tools.DiscoverySuite$.getSuiteInstance(DiscoverySuite.scala:66) ... ProtobufFunctionsSuite: org.apache.spark.sql.protobuf.ProtobufFunctionsSuite *** ABORTED *** java.lang.RuntimeException: Unable to load a Suite class org.apache.spark.sql.protobuf.ProtobufFunctionsSuite that was discovered in the runpath: null at org.scalatest.tools.DiscoverySuite$.getSuiteInstance(DiscoverySuite.scala:80) at org.scalatest.tools.DiscoverySuite.$anonfun$nestedSuites$1(DiscoverySuite.scala:38) at scala.collection.TraversableLike.$anonfun$map$1(TraversableLike.scala:286) at scala.collection.Iterator.foreach(Iterator.scala:943) at scala.collection.Iterator.foreach$(Iterator.scala:943) at scala.collection.AbstractIterator.foreach(Iterator.scala:1431) at scala.collection.IterableLike.foreach(IterableLike.scala:74) at scala.collection.IterableLike.foreach$(IterableLike.scala:73) at scala.collection.AbstractIterable.foreach(Iterable.scala:56) at scala.collection.TraversableLike.map(TraversableLike.scala:286) ... Cause: java.lang.NullPointerException: at org.apache.spark.sql.test.SQLTestUtilsBase.testFile(SQLTestUtils.scala:466) at org.apache.spark.sql.test.SQLTestUtilsBase.testFile$(SQLTestUtils.scala:465) at org.apache.spark.sql.protobuf.ProtobufFunctionsSuite.testFile(ProtobufFunctionsSuite.scala:35) at org.apache.spark.sql.protobuf.ProtobufFunctionsSuite.<init>(ProtobufFunctionsSuite.scala:39) at sun.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) at sun.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:62) at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) at java.lang.reflect.Constructor.newInstance(Constructor.java:423) at java.lang.Class.newInstance(Class.java:442) at org.scalatest.tools.DiscoverySuite$.getSuiteInstance(DiscoverySuite.scala:66) ... ProtobufCatalystDataConversionSuite: org.apache.spark.sql.protobuf.ProtobufCatalystDataConversionSuite *** ABORTED *** java.lang.RuntimeException: Unable to load a Suite class org.apache.spark.sql.protobuf.ProtobufCatalystDataConversionSuite that was discovered in the runpath: null at org.scalatest.tools.DiscoverySuite$.getSuiteInstance(DiscoverySuite.scala:80) at org.scalatest.tools.DiscoverySuite.$anonfun$nestedSuites$1(DiscoverySuite.scala:38) at scala.collection.TraversableLike.$anonfun$map$1(TraversableLike.scala:286) at scala.collection.Iterator.foreach(Iterator.scala:943) at scala.collection.Iterator.foreach$(Iterator.scala:943) at scala.collection.AbstractIterator.foreach(Iterator.scala:1431) at scala.collection.IterableLike.foreach(IterableLike.scala:74) at scala.collection.IterableLike.foreach$(IterableLike.scala:73) at scala.collection.AbstractIterable.foreach(Iterable.scala:56) at scala.collection.TraversableLike.map(TraversableLike.scala:286) ... Cause: java.lang.NullPointerException: at org.apache.spark.sql.test.SQLTestUtilsBase.testFile(SQLTestUtils.scala:466) at org.apache.spark.sql.test.SQLTestUtilsBase.testFile$(SQLTestUtils.scala:465) at org.apache.spark.sql.protobuf.ProtobufCatalystDataConversionSuite.testFile(ProtobufCatalystDataConversionSuite.scala:34) at org.apache.spark.sql.protobuf.ProtobufCatalystDataConversionSuite.<init>(ProtobufCatalystDataConversionSuite.scala:39) at sun.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) at sun.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:62) at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) at java.lang.reflect.Constructor.newInstance(Constructor.java:423) at java.lang.Class.newInstance(Class.java:442) at org.scalatest.tools.DiscoverySuite$.getSuiteInstance(DiscoverySuite.scala:66) ... Run completed in 717 milliseconds. Total number of tests run: 0 Suites: completed 1, aborted 3 Tests: succeeded 0, failed 0, canceled 0, ignored 0, pending 0 *** 3 SUITES ABORTED *** ``` **After** ``` [info] ProtobufFunctionsSuite: [info] - roundtrip in to_protobuf and from_protobuf - struct (4 seconds, 168 milliseconds) [info] - roundtrip in from_protobuf and to_protobuf - Repeated (570 milliseconds) [info] - roundtrip in from_protobuf and to_protobuf - Repeated Message Once (340 milliseconds) [info] - roundtrip in from_protobuf and to_protobuf - Repeated Message Twice (215 milliseconds) [info] - roundtrip in from_protobuf and to_protobuf - Map (589 milliseconds) [info] - roundtrip in from_protobuf and to_protobuf - Enum (187 milliseconds) [info] - roundtrip in from_protobuf and to_protobuf - Multiple Message (346 milliseconds) [info] - Handle recursive fields in Protobuf schema, A->B->A (117 milliseconds) [info] - Handle recursive fields in Protobuf schema, C->D->Array(C) (12 milliseconds) [info] - Handle extra fields : oldProducer -> newConsumer (91 milliseconds) [info] - Handle extra fields : newProducer -> oldConsumer (7 milliseconds) [info] - roundtrip in to_protobuf and from_protobuf - with nulls (457 milliseconds) [info] - from_protobuf filter to_protobuf (356 milliseconds) [info] - Handle TimestampType between to_protobuf and from_protobuf (560 milliseconds) [info] - Handle DayTimeIntervalType between to_protobuf and from_protobuf (547 milliseconds) [info] - raise cannot construct protobuf descriptor error (13 milliseconds) [info] ProtobufCatalystDataConversionSuite: [info] - single StructType(StructField(int32_type,IntegerType,true)) with seed 989 (76 milliseconds) [info] - single StructType(StructField(double_type,DoubleType,true)) with seed 65 (51 milliseconds) [info] - single StructType(StructField(float_type,FloatType,true)) with seed 43 (32 milliseconds) [info] - single StructType(StructField(bytes_type,BinaryType,true)) with seed 657 (62 milliseconds) [info] - single StructType(StructField(string_type,StringType,true)) with seed 965 (43 milliseconds) [info] - Handle unsupported input of message type (31 milliseconds) [info] - filter push-down to Protobuf deserializer (23 milliseconds) [info] - ProtobufDeserializer with binary type (1 millisecond) [info] - Full names for message using descriptor file (1 millisecond) [info] ProtobufSerdeSuite: [info] - Test basic conversion (7 milliseconds) [info] - Fail to convert with field type mismatch (3 milliseconds) [info] - Fail to convert with missing nested Protobuf fields for serializer (2 milliseconds) [info] - Fail to convert with deeply nested field type mismatch (8 milliseconds) [info] - Fail to convert with missing Catalyst fields (3 milliseconds) [info] - raise cannot parse and construct protobuf descriptor error (2 milliseconds) [info] Run completed in 12 seconds, 167 milliseconds. [info] Total number of tests run: 31 [info] Suites: completed 3, aborted 0 [info] Tests: succeeded 31, failed 0, canceled 0, ignored 0, pending 0 [info] All tests passed. ``` Closes #38690 from LuciferYang/protobuf-maven-test. Authored-by: yangjie01 <yangji...@baidu.com> Signed-off-by: Hyukjin Kwon <gurwls...@apache.org> --- .../ProtobufCatalystDataConversionSuite.scala | 5 +-- .../sql/protobuf/ProtobufFunctionsSuite.scala | 12 ++++--- .../spark/sql/protobuf/ProtobufSerdeSuite.scala | 8 ++--- .../spark/sql/protobuf/ProtobufTestBase.scala | 37 ++++++++++++++++++++++ 4 files changed, 51 insertions(+), 11 deletions(-) diff --git a/connector/protobuf/src/test/scala/org/apache/spark/sql/protobuf/ProtobufCatalystDataConversionSuite.scala b/connector/protobuf/src/test/scala/org/apache/spark/sql/protobuf/ProtobufCatalystDataConversionSuite.scala index ec75ebb5507..3e9273835e3 100644 --- a/connector/protobuf/src/test/scala/org/apache/spark/sql/protobuf/ProtobufCatalystDataConversionSuite.scala +++ b/connector/protobuf/src/test/scala/org/apache/spark/sql/protobuf/ProtobufCatalystDataConversionSuite.scala @@ -34,9 +34,10 @@ import org.apache.spark.unsafe.types.UTF8String class ProtobufCatalystDataConversionSuite extends SparkFunSuite with SharedSparkSession - with ExpressionEvalHelper { + with ExpressionEvalHelper + with ProtobufTestBase { - private val testFileDesc = testFile("catalyst_types.desc").replace("file:/", "/") + private val testFileDesc = testFile("catalyst_types.desc", "protobuf/catalyst_types.desc") private val javaClassNamePrefix = "org.apache.spark.sql.protobuf.protos.CatalystTypes$" private def checkResultWithEval( diff --git a/connector/protobuf/src/test/scala/org/apache/spark/sql/protobuf/ProtobufFunctionsSuite.scala b/connector/protobuf/src/test/scala/org/apache/spark/sql/protobuf/ProtobufFunctionsSuite.scala index 00ec56f90a6..e493bc66ca7 100644 --- a/connector/protobuf/src/test/scala/org/apache/spark/sql/protobuf/ProtobufFunctionsSuite.scala +++ b/connector/protobuf/src/test/scala/org/apache/spark/sql/protobuf/ProtobufFunctionsSuite.scala @@ -32,11 +32,12 @@ import org.apache.spark.sql.protobuf.utils.ProtobufUtils import org.apache.spark.sql.test.SharedSparkSession import org.apache.spark.sql.types.{DayTimeIntervalType, IntegerType, StringType, StructField, StructType, TimestampType} -class ProtobufFunctionsSuite extends QueryTest with SharedSparkSession with Serializable { +class ProtobufFunctionsSuite extends QueryTest with SharedSparkSession with ProtobufTestBase + with Serializable { import testImplicits._ - val testFileDesc = testFile("functions_suite.desc").replace("file:/", "/") + val testFileDesc = testFile("functions_suite.desc", "protobuf/functions_suite.desc") private val javaClassNamePrefix = "org.apache.spark.sql.protobuf.protos.SimpleMessageProtos$" /** @@ -458,7 +459,7 @@ class ProtobufFunctionsSuite extends QueryTest with SharedSparkSession with Seri } test("Handle extra fields : oldProducer -> newConsumer") { - val testFileDesc = testFile("catalyst_types.desc").replace("file:/", "/") + val testFileDesc = testFile("catalyst_types.desc", "protobuf/catalyst_types.desc") val oldProducer = ProtobufUtils.buildDescriptor(testFileDesc, "oldProducer") val newConsumer = ProtobufUtils.buildDescriptor(testFileDesc, "newConsumer") @@ -498,7 +499,7 @@ class ProtobufFunctionsSuite extends QueryTest with SharedSparkSession with Seri } test("Handle extra fields : newProducer -> oldConsumer") { - val testFileDesc = testFile("catalyst_types.desc").replace("file:/", "/") + val testFileDesc = testFile("catalyst_types.desc", "protobuf/catalyst_types.desc") val newProducer = ProtobufUtils.buildDescriptor(testFileDesc, "newProducer") val oldConsumer = ProtobufUtils.buildDescriptor(testFileDesc, "oldConsumer") @@ -680,7 +681,8 @@ class ProtobufFunctionsSuite extends QueryTest with SharedSparkSession with Seri test("raise cannot construct protobuf descriptor error") { val df = Seq(ByteString.empty().toByteArray).toDF("value") - val testFileDescriptor = testFile("basicmessage_noimports.desc").replace("file:/", "/") + val testFileDescriptor = + testFile("basicmessage_noimports.desc", "protobuf/basicmessage_noimports.desc") val e = intercept[AnalysisException] { df.select(functions.from_protobuf($"value", "BasicMessage", testFileDescriptor) as 'sample) diff --git a/connector/protobuf/src/test/scala/org/apache/spark/sql/protobuf/ProtobufSerdeSuite.scala b/connector/protobuf/src/test/scala/org/apache/spark/sql/protobuf/ProtobufSerdeSuite.scala index 22b9d58bbd4..87ed5340943 100644 --- a/connector/protobuf/src/test/scala/org/apache/spark/sql/protobuf/ProtobufSerdeSuite.scala +++ b/connector/protobuf/src/test/scala/org/apache/spark/sql/protobuf/ProtobufSerdeSuite.scala @@ -31,12 +31,12 @@ import org.apache.spark.sql.types.{IntegerType, StructType} * Tests for [[ProtobufSerializer]] and [[ProtobufDeserializer]] with a more specific focus on * those classes. */ -class ProtobufSerdeSuite extends SharedSparkSession { +class ProtobufSerdeSuite extends SharedSparkSession with ProtobufTestBase { import ProtoSerdeSuite._ import ProtoSerdeSuite.MatchType._ - val testFileDesc = testFile("serde_suite.desc").replace("file:/", "/") + val testFileDesc = testFile("serde_suite.desc", "protobuf/serde_suite.desc") private val javaClassNamePrefix = "org.apache.spark.sql.protobuf.protos.SerdeSuiteProtos$" test("Test basic conversion") { @@ -179,7 +179,7 @@ class ProtobufSerdeSuite extends SharedSparkSession { test("raise cannot parse and construct protobuf descriptor error") { // passing serde_suite.proto instead serde_suite.desc - var testFileDesc = testFile("serde_suite.proto").replace("file:/", "/") + var testFileDesc = testFile("serde_suite.proto", "protobuf/serde_suite.proto") val e1 = intercept[AnalysisException] { ProtobufUtils.buildDescriptor(testFileDesc, "FieldMissingInSQLRoot") } @@ -189,7 +189,7 @@ class ProtobufSerdeSuite extends SharedSparkSession { errorClass = "CANNOT_PARSE_PROTOBUF_DESCRIPTOR", parameters = Map("descFilePath" -> testFileDesc)) - testFileDesc = testFile("basicmessage_noimports.desc").replace("file:/", "/") + testFileDesc = testFile("basicmessage_noimports.desc", "protobuf/basicmessage_noimports.desc") val e2 = intercept[AnalysisException] { ProtobufUtils.buildDescriptor(testFileDesc, "FieldMissingInSQLRoot") } diff --git a/connector/protobuf/src/test/scala/org/apache/spark/sql/protobuf/ProtobufTestBase.scala b/connector/protobuf/src/test/scala/org/apache/spark/sql/protobuf/ProtobufTestBase.scala new file mode 100644 index 00000000000..831b4a26c06 --- /dev/null +++ b/connector/protobuf/src/test/scala/org/apache/spark/sql/protobuf/ProtobufTestBase.scala @@ -0,0 +1,37 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.spark.sql.protobuf + +import org.apache.spark.sql.test.SQLTestUtils + +trait ProtobufTestBase extends SQLTestUtils { + + /** + * Returns full path to the given file in the resource folder, + * if the first choice throw NPE, try to return the full path of alternative. + * The result path doesn't contain the `file:/` protocol part. + */ + protected def testFile(fileName: String, alternateFileName: String): String = { + val ret = try { + testFile(fileName) + } catch { + case _: NullPointerException => testFile(alternateFileName) + } + ret.replace("file:/", "/") + } +} --------------------------------------------------------------------- To unsubscribe, e-mail: commits-unsubscr...@spark.apache.org For additional commands, e-mail: commits-h...@spark.apache.org