This is an automated email from the ASF dual-hosted git repository.
hvanhovell pushed a commit to branch branch-3.5
in repository https://gitbox.apache.org/repos/asf/spark.git
The following commit(s) were added to refs/heads/branch-3.5 by this push:
new fd07239505c [SPARK-44832][CONNECT] Make transitive dependencies work
properly for Scala Client
fd07239505c is described below
commit fd07239505cff8baafa4f1684034278d41234de7
Author: Herman van Hovell <her...@databricks.com>
AuthorDate: Mon Aug 28 19:53:40 2023 +0200
[SPARK-44832][CONNECT] Make transitive dependencies work properly for Scala
Client
### What changes were proposed in this pull request?
This PR cleans up the Maven build for the Spark Connect Client and Spark
Connect Common. The most important change is that we move `sql-api` from a
`provided` to `compile` dependency. The net effect of this is that when a user
takes a dependency on the client, all of its required (transitive) dependencies
are automatically added.
Please note that this does not address concerns around creating an überjar
and shading. That is for a different day :)
### Why are the changes needed?
When you take a dependency on the connect scala client you need to manually
add the `sql-api` module as a dependency. This is rather poor UX.
### Does this PR introduce _any_ user-facing change?
No.
### How was this patch tested?
Manually running maven, checking dependency tree, ...
Closes #42518 from hvanhovell/SPARK-44832.
Authored-by: Herman van Hovell <her...@databricks.com>
Signed-off-by: Herman van Hovell <her...@databricks.com>
(cherry picked from commit 50d9a56f824ae51d10543f4573753ff60dc9053b)
Signed-off-by: Herman van Hovell <her...@databricks.com>
---
connector/connect/client/jvm/pom.xml | 48 ----------------------
.../CheckConnectJvmClientCompatibility.scala | 33 ++++++++++++---
connector/connect/common/pom.xml | 6 ---
dev/connect-jvm-client-mima-check | 2 +-
4 files changed, 28 insertions(+), 61 deletions(-)
diff --git a/connector/connect/client/jvm/pom.xml
b/connector/connect/client/jvm/pom.xml
index a7e5c5c2bab..67227ef38eb 100644
--- a/connector/connect/client/jvm/pom.xml
+++ b/connector/connect/client/jvm/pom.xml
@@ -39,55 +39,21 @@
<groupId>org.apache.spark</groupId>
<artifactId>spark-connect-common_${scala.binary.version}</artifactId>
<version>${project.version}</version>
- <exclusions>
- <exclusion>
- <groupId>com.google.guava</groupId>
- <artifactId>guava</artifactId>
- </exclusion>
- </exclusions>
</dependency>
<dependency>
<groupId>org.apache.spark</groupId>
<artifactId>spark-sql-api_${scala.binary.version}</artifactId>
<version>${project.version}</version>
- <scope>provided</scope>
</dependency>
<dependency>
<groupId>org.apache.spark</groupId>
<artifactId>spark-sketch_${scala.binary.version}</artifactId>
<version>${project.version}</version>
</dependency>
- <dependency>
- <groupId>com.google.protobuf</groupId>
- <artifactId>protobuf-java</artifactId>
- <scope>compile</scope>
- </dependency>
<dependency>
<groupId>com.google.guava</groupId>
<artifactId>guava</artifactId>
<version>${connect.guava.version}</version>
- <scope>compile</scope>
- </dependency>
- <dependency>
- <groupId>com.google.guava</groupId>
- <artifactId>failureaccess</artifactId>
- <version>${guava.failureaccess.version}</version>
- <scope>compile</scope>
- </dependency>
- <dependency>
- <groupId>io.netty</groupId>
- <artifactId>netty-codec-http2</artifactId>
- <version>${netty.version}</version>
- </dependency>
- <dependency>
- <groupId>io.netty</groupId>
- <artifactId>netty-handler-proxy</artifactId>
- <version>${netty.version}</version>
- </dependency>
- <dependency>
- <groupId>io.netty</groupId>
- <artifactId>netty-transport-native-unix-common</artifactId>
- <version>${netty.version}</version>
</dependency>
<dependency>
<groupId>com.lihaoyi</groupId>
@@ -95,19 +61,6 @@
<version>${ammonite.version}</version>
<scope>provided</scope>
</dependency>
- <dependency>
- <groupId>org.apache.spark</groupId>
- <artifactId>spark-connect-common_${scala.binary.version}</artifactId>
- <version>${project.version}</version>
- <type>test-jar</type>
- <scope>test</scope>
- <exclusions>
- <exclusion>
- <groupId>com.google.guava</groupId>
- <artifactId>guava</artifactId>
- </exclusion>
- </exclusions>
- </dependency>
<dependency>
<groupId>org.scalacheck</groupId>
<artifactId>scalacheck_${scala.binary.version}</artifactId>
@@ -148,7 +101,6 @@
<include>org.codehaus.mojo:*</include>
<include>org.checkerframework:*</include>
<include>org.apache.spark:spark-connect-common_${scala.binary.version}</include>
-
<include>org.apache.spark:spark-common-utils_${scala.binary.version}</include>
</includes>
</artifactSet>
<relocations>
diff --git
a/connector/connect/client/jvm/src/test/scala/org/apache/spark/sql/connect/client/CheckConnectJvmClientCompatibility.scala
b/connector/connect/client/jvm/src/test/scala/org/apache/spark/sql/connect/client/CheckConnectJvmClientCompatibility.scala
index 1f599f2346e..72b0f02f378 100644
---
a/connector/connect/client/jvm/src/test/scala/org/apache/spark/sql/connect/client/CheckConnectJvmClientCompatibility.scala
+++
b/connector/connect/client/jvm/src/test/scala/org/apache/spark/sql/connect/client/CheckConnectJvmClientCompatibility.scala
@@ -24,6 +24,7 @@ import java.util.regex.Pattern
import com.typesafe.tools.mima.core._
import com.typesafe.tools.mima.lib.MiMaLib
+import org.apache.spark.SparkBuildInfo.spark_version
import org.apache.spark.sql.test.IntegrationTestUtils._
/**
@@ -46,18 +47,38 @@ object CheckConnectJvmClientCompatibility {
sys.env("SPARK_HOME")
}
+ private val sqlJar = {
+ val path = Paths.get(
+ sparkHome,
+ "sql",
+ "core",
+ "target",
+ "scala-" + scalaVersion,
+ "spark-sql_" + scalaVersion + "-" + spark_version + ".jar")
+ assert(Files.exists(path), s"$path does not exist")
+ path.toFile
+ }
+
+ private val clientJar = {
+ val path = Paths.get(
+ sparkHome,
+ "connector",
+ "connect",
+ "client",
+ "jvm",
+ "target",
+ "scala-" + scalaVersion,
+ "spark-connect-client-jvm_" + scalaVersion + "-" + spark_version +
".jar")
+ assert(Files.exists(path), s"$path does not exist")
+ path.toFile
+ }
+
def main(args: Array[String]): Unit = {
var resultWriter: Writer = null
try {
resultWriter = Files.newBufferedWriter(
Paths.get(s"$sparkHome/.connect-mima-check-result"),
StandardCharsets.UTF_8)
- val clientJar: File =
- findJar(
- "connector/connect/client/jvm",
- "spark-connect-client-jvm-assembly",
- "spark-connect-client-jvm")
- val sqlJar: File = findJar("sql/core", "spark-sql", "spark-sql")
val problemsWithSqlModule =
checkMiMaCompatibilityWithSqlModule(clientJar, sqlJar)
appendMimaCheckErrorMessageIfNeeded(
resultWriter,
diff --git a/connector/connect/common/pom.xml b/connector/connect/common/pom.xml
index fb261c284d9..40de7414051 100644
--- a/connector/connect/common/pom.xml
+++ b/connector/connect/common/pom.xml
@@ -38,7 +38,6 @@
<groupId>org.apache.spark</groupId>
<artifactId>spark-sql-api_${scala.binary.version}</artifactId>
<version>${project.version}</version>
- <scope>provided</scope>
</dependency>
<dependency>
<groupId>org.scala-lang</groupId>
@@ -47,7 +46,6 @@
<dependency>
<groupId>com.google.protobuf</groupId>
<artifactId>protobuf-java</artifactId>
- <scope>compile</scope>
</dependency>
<dependency>
<groupId>com.google.guava</groupId>
@@ -85,25 +83,21 @@
<groupId>io.netty</groupId>
<artifactId>netty-codec-http2</artifactId>
<version>${netty.version}</version>
- <scope>provided</scope>
</dependency>
<dependency>
<groupId>io.netty</groupId>
<artifactId>netty-handler-proxy</artifactId>
<version>${netty.version}</version>
- <scope>provided</scope>
</dependency>
<dependency>
<groupId>io.netty</groupId>
<artifactId>netty-transport-native-unix-common</artifactId>
<version>${netty.version}</version>
- <scope>provided</scope>
</dependency>
<dependency> <!-- necessary for Java 9+ -->
<groupId>org.apache.tomcat</groupId>
<artifactId>annotations-api</artifactId>
<version>${tomcat.annotations.api.version}</version>
- <scope>provided</scope>
</dependency>
<!--
This spark-tags test-dep is needed even though it isn't used in this
module,
diff --git a/dev/connect-jvm-client-mima-check
b/dev/connect-jvm-client-mima-check
index ac4b95935b9..4e0ccbb2367 100755
--- a/dev/connect-jvm-client-mima-check
+++ b/dev/connect-jvm-client-mima-check
@@ -35,7 +35,7 @@ fi
rm -f .connect-mima-check-result
echo "Build required modules..."
-build/sbt
"sql/package;connect-client-jvm/assembly;connect-client-jvm/Test/package;avro/package;protobuf/assembly"
+build/sbt
"sql/package;connect-client-jvm/package;connect-client-jvm/Test/package;avro/package;protobuf/assembly"
CONNECT_TEST_CLASSPATH="$(build/sbt -DcopyDependencies=false "export
connect-client-jvm/Test/fullClasspath" | grep jar | tail -n1)"
SQL_CLASSPATH="$(build/sbt -DcopyDependencies=false "export sql/fullClasspath"
| grep jar | tail -n1)"
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscr...@spark.apache.org
For additional commands, e-mail: commits-h...@spark.apache.org
