This is an automated email from the ASF dual-hosted git repository. hvanhovell pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/spark.git
The following commit(s) were added to refs/heads/master by this push: new 50d9a56f824 [SPARK-44832][CONNECT] Make transitive dependencies work properly for Scala Client 50d9a56f824 is described below commit 50d9a56f824ae51d10543f4573753ff60dc9053b Author: Herman van Hovell <her...@databricks.com> AuthorDate: Mon Aug 28 19:53:40 2023 +0200 [SPARK-44832][CONNECT] Make transitive dependencies work properly for Scala Client ### What changes were proposed in this pull request? This PR cleans up the Maven build for the Spark Connect Client and Spark Connect Common. The most important change is that we move `sql-api` from a `provided` to `compile` dependency. The net effect of this is that when a user takes a dependency on the client, all of its required (transitive) dependencies are automatically added. Please note that this does not address concerns around creating an überjar and shading. That is for a different day :) ### Why are the changes needed? When you take a dependency on the connect scala client you need to manually add the `sql-api` module as a dependency. This is rather poor UX. ### Does this PR introduce _any_ user-facing change? No. ### How was this patch tested? Manually running maven, checking dependency tree, ... Closes #42518 from hvanhovell/SPARK-44832. Authored-by: Herman van Hovell <her...@databricks.com> Signed-off-by: Herman van Hovell <her...@databricks.com> --- connector/connect/client/jvm/pom.xml | 48 ---------------------- .../CheckConnectJvmClientCompatibility.scala | 33 ++++++++++++--- connector/connect/common/pom.xml | 6 --- dev/connect-jvm-client-mima-check | 2 +- 4 files changed, 28 insertions(+), 61 deletions(-) diff --git a/connector/connect/client/jvm/pom.xml b/connector/connect/client/jvm/pom.xml index d4e9b147e02..8cb6758ec9f 100644 --- a/connector/connect/client/jvm/pom.xml +++ b/connector/connect/client/jvm/pom.xml @@ -39,55 +39,21 @@ <groupId>org.apache.spark</groupId> <artifactId>spark-connect-common_${scala.binary.version}</artifactId> <version>${project.version}</version> - <exclusions> - <exclusion> - <groupId>com.google.guava</groupId> - <artifactId>guava</artifactId> - </exclusion> - </exclusions> </dependency> <dependency> <groupId>org.apache.spark</groupId> <artifactId>spark-sql-api_${scala.binary.version}</artifactId> <version>${project.version}</version> - <scope>provided</scope> </dependency> <dependency> <groupId>org.apache.spark</groupId> <artifactId>spark-sketch_${scala.binary.version}</artifactId> <version>${project.version}</version> </dependency> - <dependency> - <groupId>com.google.protobuf</groupId> - <artifactId>protobuf-java</artifactId> - <scope>compile</scope> - </dependency> <dependency> <groupId>com.google.guava</groupId> <artifactId>guava</artifactId> <version>${connect.guava.version}</version> - <scope>compile</scope> - </dependency> - <dependency> - <groupId>com.google.guava</groupId> - <artifactId>failureaccess</artifactId> - <version>${guava.failureaccess.version}</version> - <scope>compile</scope> - </dependency> - <dependency> - <groupId>io.netty</groupId> - <artifactId>netty-codec-http2</artifactId> - <version>${netty.version}</version> - </dependency> - <dependency> - <groupId>io.netty</groupId> - <artifactId>netty-handler-proxy</artifactId> - <version>${netty.version}</version> - </dependency> - <dependency> - <groupId>io.netty</groupId> - <artifactId>netty-transport-native-unix-common</artifactId> - <version>${netty.version}</version> </dependency> <dependency> <groupId>com.lihaoyi</groupId> @@ -95,19 +61,6 @@ <version>${ammonite.version}</version> <scope>provided</scope> </dependency> - <dependency> - <groupId>org.apache.spark</groupId> - <artifactId>spark-connect-common_${scala.binary.version}</artifactId> - <version>${project.version}</version> - <type>test-jar</type> - <scope>test</scope> - <exclusions> - <exclusion> - <groupId>com.google.guava</groupId> - <artifactId>guava</artifactId> - </exclusion> - </exclusions> - </dependency> <dependency> <groupId>org.scalacheck</groupId> <artifactId>scalacheck_${scala.binary.version}</artifactId> @@ -148,7 +101,6 @@ <include>org.codehaus.mojo:*</include> <include>org.checkerframework:*</include> <include>org.apache.spark:spark-connect-common_${scala.binary.version}</include> - <include>org.apache.spark:spark-common-utils_${scala.binary.version}</include> </includes> </artifactSet> <relocations> diff --git a/connector/connect/client/jvm/src/test/scala/org/apache/spark/sql/connect/client/CheckConnectJvmClientCompatibility.scala b/connector/connect/client/jvm/src/test/scala/org/apache/spark/sql/connect/client/CheckConnectJvmClientCompatibility.scala index 1100babde79..1e536cd37fe 100644 --- a/connector/connect/client/jvm/src/test/scala/org/apache/spark/sql/connect/client/CheckConnectJvmClientCompatibility.scala +++ b/connector/connect/client/jvm/src/test/scala/org/apache/spark/sql/connect/client/CheckConnectJvmClientCompatibility.scala @@ -24,6 +24,7 @@ import java.util.regex.Pattern import com.typesafe.tools.mima.core._ import com.typesafe.tools.mima.lib.MiMaLib +import org.apache.spark.SparkBuildInfo.spark_version import org.apache.spark.sql.test.IntegrationTestUtils._ /** @@ -46,18 +47,38 @@ object CheckConnectJvmClientCompatibility { sys.env("SPARK_HOME") } + private val sqlJar = { + val path = Paths.get( + sparkHome, + "sql", + "core", + "target", + "scala-" + scalaVersion, + "spark-sql_" + scalaVersion + "-" + spark_version + ".jar") + assert(Files.exists(path), s"$path does not exist") + path.toFile + } + + private val clientJar = { + val path = Paths.get( + sparkHome, + "connector", + "connect", + "client", + "jvm", + "target", + "scala-" + scalaVersion, + "spark-connect-client-jvm_" + scalaVersion + "-" + spark_version + ".jar") + assert(Files.exists(path), s"$path does not exist") + path.toFile + } + def main(args: Array[String]): Unit = { var resultWriter: Writer = null try { resultWriter = Files.newBufferedWriter( Paths.get(s"$sparkHome/.connect-mima-check-result"), StandardCharsets.UTF_8) - val clientJar: File = - findJar( - "connector/connect/client/jvm", - "spark-connect-client-jvm-assembly", - "spark-connect-client-jvm") - val sqlJar: File = findJar("sql/core", "spark-sql", "spark-sql") val problemsWithSqlModule = checkMiMaCompatibilityWithSqlModule(clientJar, sqlJar) appendMimaCheckErrorMessageIfNeeded( resultWriter, diff --git a/connector/connect/common/pom.xml b/connector/connect/common/pom.xml index 8382c1f5db1..5fd6c885025 100644 --- a/connector/connect/common/pom.xml +++ b/connector/connect/common/pom.xml @@ -38,7 +38,6 @@ <groupId>org.apache.spark</groupId> <artifactId>spark-sql-api_${scala.binary.version}</artifactId> <version>${project.version}</version> - <scope>provided</scope> </dependency> <dependency> <groupId>org.scala-lang</groupId> @@ -47,7 +46,6 @@ <dependency> <groupId>com.google.protobuf</groupId> <artifactId>protobuf-java</artifactId> - <scope>compile</scope> </dependency> <dependency> <groupId>com.google.guava</groupId> @@ -85,25 +83,21 @@ <groupId>io.netty</groupId> <artifactId>netty-codec-http2</artifactId> <version>${netty.version}</version> - <scope>provided</scope> </dependency> <dependency> <groupId>io.netty</groupId> <artifactId>netty-handler-proxy</artifactId> <version>${netty.version}</version> - <scope>provided</scope> </dependency> <dependency> <groupId>io.netty</groupId> <artifactId>netty-transport-native-unix-common</artifactId> <version>${netty.version}</version> - <scope>provided</scope> </dependency> <dependency> <!-- necessary for Java 9+ --> <groupId>org.apache.tomcat</groupId> <artifactId>annotations-api</artifactId> <version>${tomcat.annotations.api.version}</version> - <scope>provided</scope> </dependency> <!-- This spark-tags test-dep is needed even though it isn't used in this module, diff --git a/dev/connect-jvm-client-mima-check b/dev/connect-jvm-client-mima-check index 6a29cbf08ce..7564a614be9 100755 --- a/dev/connect-jvm-client-mima-check +++ b/dev/connect-jvm-client-mima-check @@ -35,7 +35,7 @@ fi rm -f .connect-mima-check-result echo "Build required modules..." -build/sbt "sql/package;connect-client-jvm/assembly;connect-client-jvm/Test/package;avro/package;protobuf/assembly" +build/sbt "sql/package;connect-client-jvm/package;connect-client-jvm/Test/package;avro/package;protobuf/assembly" CONNECT_TEST_CLASSPATH="$(build/sbt -DcopyDependencies=false "export connect-client-jvm/Test/fullClasspath" | grep jar | tail -n1)" SQL_CLASSPATH="$(build/sbt -DcopyDependencies=false "export sql/fullClasspath" | grep jar | tail -n1)" --------------------------------------------------------------------- To unsubscribe, e-mail: commits-unsubscr...@spark.apache.org For additional commands, e-mail: commits-h...@spark.apache.org