This is an automated email from the ASF dual-hosted git repository. mridulm80 pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/spark.git
The following commit(s) were added to refs/heads/master by this push: new 4f09b6cc101 [SPARK-45376][CORE] Add netty-tcnative-boringssl-static dependency 4f09b6cc101 is described below commit 4f09b6cc101bd17f283fe1f9013579c6529795ba Author: Hasnain Lakhani <hasnain.lakh...@databricks.com> AuthorDate: Tue Oct 3 18:59:18 2023 -0500 [SPARK-45376][CORE] Add netty-tcnative-boringssl-static dependency ### What changes were proposed in this pull request? This PR adds a new dependency on this library so that we can use it for SSL functionality. We also include the network-common test helper library in a few places, which will be needed for reusing some test configuration files in the future. ### Why are the changes needed? These dependency changes are needed so we can use this library to provide SSL functionality, and for test helpers. ### Does this PR introduce _any_ user-facing change? No ### How was this patch tested? CI. This is used in unit tests and functionality as part of https://github.com/apache/spark/pull/42685, from which this is being split out ### Was this patch authored or co-authored using generative AI tooling? No Closes #43164 from hasnain-db/spark-tls-deps. Authored-by: Hasnain Lakhani <hasnain.lakh...@databricks.com> Signed-off-by: Mridul Muralidharan <mridul<at>gmail.com> --- common/network-common/pom.xml | 34 +++++++++++++++++++++++++++++++++- core/pom.xml | 7 +++++++ dev/deps/spark-deps-hadoop-3-hive-2.3 | 6 ++++++ pom.xml | 25 +++++++++++++++++++++++++ resource-managers/yarn/pom.xml | 7 +++++++ 5 files changed, 78 insertions(+), 1 deletion(-) diff --git a/common/network-common/pom.xml b/common/network-common/pom.xml index c8bacf96aed..bec9531fcbe 100644 --- a/common/network-common/pom.xml +++ b/common/network-common/pom.xml @@ -61,6 +61,26 @@ <artifactId>netty-transport-native-kqueue</artifactId> <classifier>osx-x86_64</classifier> </dependency> + <dependency> + <groupId>io.netty</groupId> + <artifactId>netty-tcnative-boringssl-static</artifactId> + <classifier>linux-x86_64</classifier> + </dependency> + <dependency> + <groupId>io.netty</groupId> + <artifactId>netty-tcnative-boringssl-static</artifactId> + <classifier>linux-aarch_64</classifier> + </dependency> + <dependency> + <groupId>io.netty</groupId> + <artifactId>netty-tcnative-boringssl-static</artifactId> + <classifier>osx-aarch_64</classifier> + </dependency> + <dependency> + <groupId>io.netty</groupId> + <artifactId>netty-tcnative-boringssl-static</artifactId> + <classifier>osx-x86_64</classifier> + </dependency> <!-- Netty End --> <dependency> @@ -141,12 +161,24 @@ <artifactId>log4j-slf4j2-impl</artifactId> <scope>test</scope> </dependency> + <dependency> + <groupId>org.bouncycastle</groupId> + <artifactId>bcprov-jdk15on</artifactId> + <version>${bouncycastle.version}</version> + <scope>test</scope> + </dependency> + <dependency> + <groupId>org.bouncycastle</groupId> + <artifactId>bcpkix-jdk15on</artifactId> + <version>${bouncycastle.version}</version> + <scope>test</scope> + </dependency> <dependency> <groupId>org.apache.spark</groupId> <artifactId>spark-common-utils_${scala.binary.version}</artifactId> <version>${project.version}</version> - </dependency> + </dependency> <!-- This spark-tags test-dep is needed even though it isn't used in this module, otherwise testing-cmds that exclude diff --git a/core/pom.xml b/core/pom.xml index bbfac34117b..5ac3d5bb4de 100644 --- a/core/pom.xml +++ b/core/pom.xml @@ -446,6 +446,13 @@ <classifier>tests</classifier> <scope>test</scope> </dependency> + <dependency> + <groupId>org.apache.spark</groupId> + <artifactId>spark-network-common_${scala.binary.version}</artifactId> + <version>${project.version}</version> + <type>test-jar</type> + <scope>test</scope> + </dependency> <dependency> <groupId>org.apache.spark</groupId> <artifactId>spark-network-shuffle_${scala.binary.version}</artifactId> diff --git a/dev/deps/spark-deps-hadoop-3-hive-2.3 b/dev/deps/spark-deps-hadoop-3-hive-2.3 index 38f05e39dcd..f4e3e2006f8 100644 --- a/dev/deps/spark-deps-hadoop-3-hive-2.3 +++ b/dev/deps/spark-deps-hadoop-3-hive-2.3 @@ -197,6 +197,12 @@ netty-common/4.1.99.Final//netty-common-4.1.99.Final.jar netty-handler-proxy/4.1.99.Final//netty-handler-proxy-4.1.99.Final.jar netty-handler/4.1.99.Final//netty-handler-4.1.99.Final.jar netty-resolver/4.1.99.Final//netty-resolver-4.1.99.Final.jar +netty-tcnative-boringssl-static/2.0.61.Final/linux-aarch_64/netty-tcnative-boringssl-static-2.0.61.Final-linux-aarch_64.jar +netty-tcnative-boringssl-static/2.0.61.Final/linux-x86_64/netty-tcnative-boringssl-static-2.0.61.Final-linux-x86_64.jar +netty-tcnative-boringssl-static/2.0.61.Final/osx-aarch_64/netty-tcnative-boringssl-static-2.0.61.Final-osx-aarch_64.jar +netty-tcnative-boringssl-static/2.0.61.Final/osx-x86_64/netty-tcnative-boringssl-static-2.0.61.Final-osx-x86_64.jar +netty-tcnative-boringssl-static/2.0.61.Final/windows-x86_64/netty-tcnative-boringssl-static-2.0.61.Final-windows-x86_64.jar +netty-tcnative-classes/2.0.61.Final//netty-tcnative-classes-2.0.61.Final.jar netty-transport-classes-epoll/4.1.99.Final//netty-transport-classes-epoll-4.1.99.Final.jar netty-transport-classes-kqueue/4.1.99.Final//netty-transport-classes-kqueue-4.1.99.Final.jar netty-transport-native-epoll/4.1.99.Final/linux-aarch_64/netty-transport-native-epoll-4.1.99.Final-linux-aarch_64.jar diff --git a/pom.xml b/pom.xml index fc3297c96bd..ef98ecf1291 100644 --- a/pom.xml +++ b/pom.xml @@ -223,6 +223,7 @@ <bouncycastle.version>1.70</bouncycastle.version> <tink.version>1.9.0</tink.version> <netty.version>4.1.99.Final</netty.version> + <netty-tcnative.version>2.0.61.Final</netty-tcnative.version> <!-- If you are changing Arrow version specification, please check ./python/pyspark/sql/pandas/utils.py, and ./python/setup.py too. @@ -928,6 +929,30 @@ <version>${netty.version}</version> <classifier>osx-x86_64</classifier> </dependency> + <dependency> + <groupId>io.netty</groupId> + <artifactId>netty-tcnative-boringssl-static</artifactId> + <version>${netty-tcnative.version}</version> + <classifier>linux-x86_64</classifier> + </dependency> + <dependency> + <groupId>io.netty</groupId> + <artifactId>netty-tcnative-boringssl-static</artifactId> + <version>${netty-tcnative.version}</version> + <classifier>linux-aarch_64</classifier> + </dependency> + <dependency> + <groupId>io.netty</groupId> + <artifactId>netty-tcnative-boringssl-static</artifactId> + <version>${netty-tcnative.version}</version> + <classifier>osx-aarch_64</classifier> + </dependency> + <dependency> + <groupId>io.netty</groupId> + <artifactId>netty-tcnative-boringssl-static</artifactId> + <version>${netty-tcnative.version}</version> + <classifier>osx-x86_64</classifier> + </dependency> <!-- Netty End --> <dependency> diff --git a/resource-managers/yarn/pom.xml b/resource-managers/yarn/pom.xml index 073661e9ac6..75ab7bc6e2f 100644 --- a/resource-managers/yarn/pom.xml +++ b/resource-managers/yarn/pom.xml @@ -91,6 +91,13 @@ <type>test-jar</type> <scope>test</scope> </dependency> + <dependency> + <groupId>org.apache.spark</groupId> + <artifactId>spark-network-common_${scala.binary.version}</artifactId> + <version>${project.version}</version> + <type>test-jar</type> + <scope>test</scope> + </dependency> <dependency> <groupId>org.apache.hadoop</groupId> <artifactId>hadoop-client-api</artifactId> --------------------------------------------------------------------- To unsubscribe, e-mail: commits-unsubscr...@spark.apache.org For additional commands, e-mail: commits-h...@spark.apache.org