This is an automated email from the ASF dual-hosted git repository. hvanhovell pushed a commit to branch branch-3.5 in repository https://gitbox.apache.org/repos/asf/spark.git
The following commit(s) were added to refs/heads/branch-3.5 by this push: new fd07239505c [SPARK-44832][CONNECT] Make transitive dependencies work properly for Scala Client fd07239505c is described below commit fd07239505cff8baafa4f1684034278d41234de7 Author: Herman van Hovell <her...@databricks.com> AuthorDate: Mon Aug 28 19:53:40 2023 +0200 [SPARK-44832][CONNECT] Make transitive dependencies work properly for Scala Client ### What changes were proposed in this pull request? This PR cleans up the Maven build for the Spark Connect Client and Spark Connect Common. The most important change is that we move `sql-api` from a `provided` to `compile` dependency. The net effect of this is that when a user takes a dependency on the client, all of its required (transitive) dependencies are automatically added. Please note that this does not address concerns around creating an überjar and shading. That is for a different day :) ### Why are the changes needed? When you take a dependency on the connect scala client you need to manually add the `sql-api` module as a dependency. This is rather poor UX. ### Does this PR introduce _any_ user-facing change? No. ### How was this patch tested? Manually running maven, checking dependency tree, ... Closes #42518 from hvanhovell/SPARK-44832. Authored-by: Herman van Hovell <her...@databricks.com> Signed-off-by: Herman van Hovell <her...@databricks.com> (cherry picked from commit 50d9a56f824ae51d10543f4573753ff60dc9053b) Signed-off-by: Herman van Hovell <her...@databricks.com> --- connector/connect/client/jvm/pom.xml | 48 ---------------------- .../CheckConnectJvmClientCompatibility.scala | 33 ++++++++++++--- connector/connect/common/pom.xml | 6 --- dev/connect-jvm-client-mima-check | 2 +- 4 files changed, 28 insertions(+), 61 deletions(-) diff --git a/connector/connect/client/jvm/pom.xml b/connector/connect/client/jvm/pom.xml index a7e5c5c2bab..67227ef38eb 100644 --- a/connector/connect/client/jvm/pom.xml +++ b/connector/connect/client/jvm/pom.xml @@ -39,55 +39,21 @@ <groupId>org.apache.spark</groupId> <artifactId>spark-connect-common_${scala.binary.version}</artifactId> <version>${project.version}</version> - <exclusions> - <exclusion> - <groupId>com.google.guava</groupId> - <artifactId>guava</artifactId> - </exclusion> - </exclusions> </dependency> <dependency> <groupId>org.apache.spark</groupId> <artifactId>spark-sql-api_${scala.binary.version}</artifactId> <version>${project.version}</version> - <scope>provided</scope> </dependency> <dependency> <groupId>org.apache.spark</groupId> <artifactId>spark-sketch_${scala.binary.version}</artifactId> <version>${project.version}</version> </dependency> - <dependency> - <groupId>com.google.protobuf</groupId> - <artifactId>protobuf-java</artifactId> - <scope>compile</scope> - </dependency> <dependency> <groupId>com.google.guava</groupId> <artifactId>guava</artifactId> <version>${connect.guava.version}</version> - <scope>compile</scope> - </dependency> - <dependency> - <groupId>com.google.guava</groupId> - <artifactId>failureaccess</artifactId> - <version>${guava.failureaccess.version}</version> - <scope>compile</scope> - </dependency> - <dependency> - <groupId>io.netty</groupId> - <artifactId>netty-codec-http2</artifactId> - <version>${netty.version}</version> - </dependency> - <dependency> - <groupId>io.netty</groupId> - <artifactId>netty-handler-proxy</artifactId> - <version>${netty.version}</version> - </dependency> - <dependency> - <groupId>io.netty</groupId> - <artifactId>netty-transport-native-unix-common</artifactId> - <version>${netty.version}</version> </dependency> <dependency> <groupId>com.lihaoyi</groupId> @@ -95,19 +61,6 @@ <version>${ammonite.version}</version> <scope>provided</scope> </dependency> - <dependency> - <groupId>org.apache.spark</groupId> - <artifactId>spark-connect-common_${scala.binary.version}</artifactId> - <version>${project.version}</version> - <type>test-jar</type> - <scope>test</scope> - <exclusions> - <exclusion> - <groupId>com.google.guava</groupId> - <artifactId>guava</artifactId> - </exclusion> - </exclusions> - </dependency> <dependency> <groupId>org.scalacheck</groupId> <artifactId>scalacheck_${scala.binary.version}</artifactId> @@ -148,7 +101,6 @@ <include>org.codehaus.mojo:*</include> <include>org.checkerframework:*</include> <include>org.apache.spark:spark-connect-common_${scala.binary.version}</include> - <include>org.apache.spark:spark-common-utils_${scala.binary.version}</include> </includes> </artifactSet> <relocations> diff --git a/connector/connect/client/jvm/src/test/scala/org/apache/spark/sql/connect/client/CheckConnectJvmClientCompatibility.scala b/connector/connect/client/jvm/src/test/scala/org/apache/spark/sql/connect/client/CheckConnectJvmClientCompatibility.scala index 1f599f2346e..72b0f02f378 100644 --- a/connector/connect/client/jvm/src/test/scala/org/apache/spark/sql/connect/client/CheckConnectJvmClientCompatibility.scala +++ b/connector/connect/client/jvm/src/test/scala/org/apache/spark/sql/connect/client/CheckConnectJvmClientCompatibility.scala @@ -24,6 +24,7 @@ import java.util.regex.Pattern import com.typesafe.tools.mima.core._ import com.typesafe.tools.mima.lib.MiMaLib +import org.apache.spark.SparkBuildInfo.spark_version import org.apache.spark.sql.test.IntegrationTestUtils._ /** @@ -46,18 +47,38 @@ object CheckConnectJvmClientCompatibility { sys.env("SPARK_HOME") } + private val sqlJar = { + val path = Paths.get( + sparkHome, + "sql", + "core", + "target", + "scala-" + scalaVersion, + "spark-sql_" + scalaVersion + "-" + spark_version + ".jar") + assert(Files.exists(path), s"$path does not exist") + path.toFile + } + + private val clientJar = { + val path = Paths.get( + sparkHome, + "connector", + "connect", + "client", + "jvm", + "target", + "scala-" + scalaVersion, + "spark-connect-client-jvm_" + scalaVersion + "-" + spark_version + ".jar") + assert(Files.exists(path), s"$path does not exist") + path.toFile + } + def main(args: Array[String]): Unit = { var resultWriter: Writer = null try { resultWriter = Files.newBufferedWriter( Paths.get(s"$sparkHome/.connect-mima-check-result"), StandardCharsets.UTF_8) - val clientJar: File = - findJar( - "connector/connect/client/jvm", - "spark-connect-client-jvm-assembly", - "spark-connect-client-jvm") - val sqlJar: File = findJar("sql/core", "spark-sql", "spark-sql") val problemsWithSqlModule = checkMiMaCompatibilityWithSqlModule(clientJar, sqlJar) appendMimaCheckErrorMessageIfNeeded( resultWriter, diff --git a/connector/connect/common/pom.xml b/connector/connect/common/pom.xml index fb261c284d9..40de7414051 100644 --- a/connector/connect/common/pom.xml +++ b/connector/connect/common/pom.xml @@ -38,7 +38,6 @@ <groupId>org.apache.spark</groupId> <artifactId>spark-sql-api_${scala.binary.version}</artifactId> <version>${project.version}</version> - <scope>provided</scope> </dependency> <dependency> <groupId>org.scala-lang</groupId> @@ -47,7 +46,6 @@ <dependency> <groupId>com.google.protobuf</groupId> <artifactId>protobuf-java</artifactId> - <scope>compile</scope> </dependency> <dependency> <groupId>com.google.guava</groupId> @@ -85,25 +83,21 @@ <groupId>io.netty</groupId> <artifactId>netty-codec-http2</artifactId> <version>${netty.version}</version> - <scope>provided</scope> </dependency> <dependency> <groupId>io.netty</groupId> <artifactId>netty-handler-proxy</artifactId> <version>${netty.version}</version> - <scope>provided</scope> </dependency> <dependency> <groupId>io.netty</groupId> <artifactId>netty-transport-native-unix-common</artifactId> <version>${netty.version}</version> - <scope>provided</scope> </dependency> <dependency> <!-- necessary for Java 9+ --> <groupId>org.apache.tomcat</groupId> <artifactId>annotations-api</artifactId> <version>${tomcat.annotations.api.version}</version> - <scope>provided</scope> </dependency> <!-- This spark-tags test-dep is needed even though it isn't used in this module, diff --git a/dev/connect-jvm-client-mima-check b/dev/connect-jvm-client-mima-check index ac4b95935b9..4e0ccbb2367 100755 --- a/dev/connect-jvm-client-mima-check +++ b/dev/connect-jvm-client-mima-check @@ -35,7 +35,7 @@ fi rm -f .connect-mima-check-result echo "Build required modules..." -build/sbt "sql/package;connect-client-jvm/assembly;connect-client-jvm/Test/package;avro/package;protobuf/assembly" +build/sbt "sql/package;connect-client-jvm/package;connect-client-jvm/Test/package;avro/package;protobuf/assembly" CONNECT_TEST_CLASSPATH="$(build/sbt -DcopyDependencies=false "export connect-client-jvm/Test/fullClasspath" | grep jar | tail -n1)" SQL_CLASSPATH="$(build/sbt -DcopyDependencies=false "export sql/fullClasspath" | grep jar | tail -n1)" --------------------------------------------------------------------- To unsubscribe, e-mail: commits-unsubscr...@spark.apache.org For additional commands, e-mail: commits-h...@spark.apache.org