This is an automated email from the ASF dual-hosted git repository.
srowen pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/spark.git
The following commit(s) were added to refs/heads/master by this push:
new b23185080cc [SPARK-43489][BUILD] Remove protobuf 2.5.0
b23185080cc is described below
commit b23185080cc3e5a00b88496cec70c2b3cd7019f5
Author: Cheng Pan <cheng...@apache.org>
AuthorDate: Sun May 14 08:09:37 2023 -0500
[SPARK-43489][BUILD] Remove protobuf 2.5.0
### What changes were proposed in this pull request?
Spark does not use protobuf 2.5.0 directly, instead, it comes from other
dependencies, with the following changes, now, Spark does not require protobuf
2.5.0 (please let me know if I miss something),
- SPARK-40323 upgraded ORC 1.8.0, which moved from protobuf 2.5.0 to a
shaded protobuf 3
- SPARK-33212 switched from Hadoop vanilla client to Hadoop shaded client,
also removed the protobuf 2 dependency. SPARK-42452 removed the support for
Hadoop 2.
- SPARK-14421 shaded and relocated protobuf 2.6.1, which is required by the
kinesis client, into the kinesis assembly jar
- Spark itself's core/connect/protobuf modules use protobuf 3, also shaded
and relocated all protobuf 3 deps.
### Why are the changes needed?
Remove the obsolete dependency, which is EOL long ago, and has CVEs
[CVE-2022-3510](https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2022-3510)
[CVE-2022-3509](https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2022-3509)
[CVE-2022-3171](https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2022-3171)
[CVE-2021-22569](https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2021-22569)
### Does this PR introduce _any_ user-facing change?
No.
### How was this patch tested?
Pass GA.
Closes #41153 from pan3793/remove-protobuf-2.
Authored-by: Cheng Pan <cheng...@apache.org>
Signed-off-by: Sean Owen <sro...@gmail.com>
---
connector/connect/client/jvm/pom.xml | 1 -
connector/connect/common/pom.xml | 1 -
connector/connect/server/pom.xml | 2 --
connector/protobuf/pom.xml | 2 --
core/pom.xml | 3 +--
dev/deps/spark-deps-hadoop-3-hive-2.3 | 1 -
pom.xml | 18 ++++++++----------
sql/core/pom.xml | 1 -
sql/hive/pom.xml | 11 -----------
9 files changed, 9 insertions(+), 31 deletions(-)
diff --git a/connector/connect/client/jvm/pom.xml
b/connector/connect/client/jvm/pom.xml
index 8543057d0c0..413764d0ea2 100644
--- a/connector/connect/client/jvm/pom.xml
+++ b/connector/connect/client/jvm/pom.xml
@@ -65,7 +65,6 @@
<dependency>
<groupId>com.google.protobuf</groupId>
<artifactId>protobuf-java</artifactId>
- <version>${protobuf.version}</version>
<scope>compile</scope>
</dependency>
<dependency>
diff --git a/connector/connect/common/pom.xml b/connector/connect/common/pom.xml
index e457620e593..06076646df7 100644
--- a/connector/connect/common/pom.xml
+++ b/connector/connect/common/pom.xml
@@ -57,7 +57,6 @@
<dependency>
<groupId>com.google.protobuf</groupId>
<artifactId>protobuf-java</artifactId>
- <version>${protobuf.version}</version>
<scope>compile</scope>
</dependency>
<dependency>
diff --git a/connector/connect/server/pom.xml b/connector/connect/server/pom.xml
index a62c420bcc0..8313f21f4ba 100644
--- a/connector/connect/server/pom.xml
+++ b/connector/connect/server/pom.xml
@@ -170,13 +170,11 @@
<dependency>
<groupId>com.google.protobuf</groupId>
<artifactId>protobuf-java</artifactId>
- <version>${protobuf.version}</version>
<scope>compile</scope>
</dependency>
<dependency>
<groupId>com.google.protobuf</groupId>
<artifactId>protobuf-java-util</artifactId>
- <version>${protobuf.version}</version>
<scope>compile</scope>
</dependency>
<dependency>
diff --git a/connector/protobuf/pom.xml b/connector/protobuf/pom.xml
index 6feef54ce71..e85f07841df 100644
--- a/connector/protobuf/pom.xml
+++ b/connector/protobuf/pom.xml
@@ -79,13 +79,11 @@
<dependency>
<groupId>com.google.protobuf</groupId>
<artifactId>protobuf-java</artifactId>
- <version>${protobuf.version}</version>
<scope>compile</scope>
</dependency>
<dependency>
<groupId>com.google.protobuf</groupId>
<artifactId>protobuf-java-util</artifactId>
- <version>${protobuf.version}</version>
<scope>compile</scope>
</dependency>
</dependencies>
diff --git a/core/pom.xml b/core/pom.xml
index 66e41837d52..09b0a2af96f 100644
--- a/core/pom.xml
+++ b/core/pom.xml
@@ -536,7 +536,6 @@
<dependency>
<groupId>com.google.protobuf</groupId>
<artifactId>protobuf-java</artifactId>
- <version>${protobuf.version}</version>
<scope>compile</scope>
</dependency>
</dependencies>
@@ -627,7 +626,7 @@
<overWriteIfNewer>true</overWriteIfNewer>
<useSubDirectoryPerType>true</useSubDirectoryPerType>
<includeArtifactIds>
-
guava,jetty-io,jetty-servlet,jetty-servlets,jetty-continuation,jetty-http,jetty-plus,jetty-util,jetty-server,jetty-security,jetty-proxy,jetty-client
+
guava,protobuf-java,jetty-io,jetty-servlet,jetty-servlets,jetty-continuation,jetty-http,jetty-plus,jetty-util,jetty-server,jetty-security,jetty-proxy,jetty-client
</includeArtifactIds>
<silent>true</silent>
</configuration>
diff --git a/dev/deps/spark-deps-hadoop-3-hive-2.3
b/dev/deps/spark-deps-hadoop-3-hive-2.3
index c23bb89c983..7e702e44c40 100644
--- a/dev/deps/spark-deps-hadoop-3-hive-2.3
+++ b/dev/deps/spark-deps-hadoop-3-hive-2.3
@@ -221,7 +221,6 @@
parquet-format-structures/1.13.0//parquet-format-structures-1.13.0.jar
parquet-hadoop/1.13.0//parquet-hadoop-1.13.0.jar
parquet-jackson/1.13.0//parquet-jackson-1.13.0.jar
pickle/1.3//pickle-1.3.jar
-protobuf-java/2.5.0//protobuf-java-2.5.0.jar
py4j/0.10.9.7//py4j-0.10.9.7.jar
remotetea-oncrpc/1.1.2//remotetea-oncrpc-1.1.2.jar
rocksdbjni/8.0.0//rocksdbjni-8.0.0.jar
diff --git a/pom.xml b/pom.xml
index 35037932243..457d96c3c7b 100644
--- a/pom.xml
+++ b/pom.xml
@@ -121,9 +121,6 @@
<log4j.version>2.20.0</log4j.version>
<!-- make sure to update IsolatedClientLoader whenever this version is
changed -->
<hadoop.version>3.3.5</hadoop.version>
- <!-- Protobuf version for building with Hadoop/Yarn dependencies -->
-
<protobuf.hadoopDependency.version>2.5.0</protobuf.hadoopDependency.version>
- <!-- Actual Protobuf version in Spark modules like Spark Connect, protobuf
connector, etc. -->
<!-- SPARK-41247: When updating `protobuf.version`, also need to update
`protoVersion` in `SparkBuild.scala` -->
<protobuf.version>3.22.3</protobuf.version>
<protoc-jar-maven-plugin.version>3.11.4</protoc-jar-maven-plugin.version>
@@ -802,16 +799,17 @@
</exclusion>
</exclusions>
</dependency>
- <!-- In theory we need not directly depend on protobuf since Spark does
not directly
- use it. However, when building with Hadoop/YARN 2.2 Maven doesn't
correctly bump
- the protobuf version up from the one Mesos gives. For now we
include this variable
- to explicitly bump the version when building with YARN. It would be
nice to figure
- out why Maven can't resolve this correctly (like SBT does). -->
<dependency>
<groupId>com.google.protobuf</groupId>
<artifactId>protobuf-java</artifactId>
- <version>${protobuf.hadoopDependency.version}</version>
- <scope>${hadoop.deps.scope}</scope>
+ <version>${protobuf.version}</version>
+ <scope>provided</scope>
+ </dependency>
+ <dependency>
+ <groupId>com.google.protobuf</groupId>
+ <artifactId>protobuf-java-util</artifactId>
+ <version>${protobuf.version}</version>
+ <scope>provided</scope>
</dependency>
<dependency>
<groupId>org.roaringbitmap</groupId>
diff --git a/sql/core/pom.xml b/sql/core/pom.xml
index 87ab2306660..7f4c2a4cfa5 100644
--- a/sql/core/pom.xml
+++ b/sql/core/pom.xml
@@ -150,7 +150,6 @@
<dependency>
<groupId>com.google.protobuf</groupId>
<artifactId>protobuf-java</artifactId>
- <version>${protobuf.version}</version>
</dependency>
<dependency>
<groupId>org.scalacheck</groupId>
diff --git a/sql/hive/pom.xml b/sql/hive/pom.xml
index b1f438864bc..16d915c233e 100644
--- a/sql/hive/pom.xml
+++ b/sql/hive/pom.xml
@@ -85,17 +85,6 @@
<artifactId>scala-parallel-collections_${scala.binary.version}</artifactId>
</dependency>
--><!-- #endif scala-2.13 -->
-<!--
- <dependency>
- <groupId>com.google.guava</groupId>
- <artifactId>guava</artifactId>
- </dependency>
- <dependency>
- <groupId>com.google.protobuf</groupId>
- <artifactId>protobuf-java</artifactId>
- <version>${protobuf.version}</version>
- </dependency>
--->
<dependency>
<groupId>${hive.group}</groupId>
<artifactId>hive-common</artifactId>
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscr...@spark.apache.org
For additional commands, e-mail: commits-h...@spark.apache.org
