This is an automated email from the ASF dual-hosted git repository.
ethanfeng pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/celeborn.git
The following commit(s) were added to refs/heads/main by this push:
new 589100ea9 [CELEBORN-1600] Enable check server dependencies
589100ea9 is described below
commit 589100ea917fb106609352522fe73531dd8054aa
Author: sychen <[email protected]>
AuthorDate: Fri Sep 20 15:14:56 2024 +0800
[CELEBORN-1600] Enable check server dependencies
### What changes were proposed in this pull request?
### Why are the changes needed?
Server module missing checks.
### Does this PR introduce _any_ user-facing change?
No
### How was this patch tested?
GA
Closes #2742 from cxzl25/check_server_deps.
Authored-by: sychen <[email protected]>
Signed-off-by: mingji <[email protected]>
---
dev/dependencies.sh | 39 ++++++++++++++++++---------------------
dev/deps/dependencies-client-mr | 8 ++++----
dev/deps/dependencies-server | 9 ++++-----
openapi/openapi-client/pom.xml | 16 ----------------
pom.xml | 22 ++++++++++++++++++++++
project/CelebornBuild.scala | 20 ++++++++++++++------
service/pom.xml | 15 ---------------
7 files changed, 62 insertions(+), 67 deletions(-)
diff --git a/dev/dependencies.sh b/dev/dependencies.sh
index b7971a3c9..625b12edf 100755
--- a/dev/dependencies.sh
+++ b/dev/dependencies.sh
@@ -63,6 +63,22 @@ function sbt_build_client_classpath() {
awk 'NR==2' | \
tr ":" "\n"
)
+ sbt_process_classpath "$deps"
+ }
+
+function sbt_build_server_classpath() {
+ deps=$(
+ $SBT "error; clean; export managedClasspath" | \
+ awk '/managedClasspath/ { found=1 } found { print }' | \
+ awk 'NR % 2 == 0' | \
+ # This will skip the last line
+ sed '$d' | \
+ tr ":" "\n"
+ )
+ sbt_process_classpath "$deps"
+}
+
+function sbt_process_classpath() {
deps1=$(echo "$deps" | grep -v ".sbt")
deps2=$(echo "$deps" | grep ".sbt" || true)
result1=$(
@@ -95,24 +111,6 @@ function sbt_build_client_classpath() {
echo "${result[@]}" | tr ' ' '\n' | sort -u >> "${DEP_PR}"
}
-function sbt_build_server_classpath() {
- $SBT "error; clean; export managedClasspath" | \
- awk '/managedClasspath/ { found=1 } found { print }' | \
- awk 'NR % 2 == 0' | \
- # This will skip the last line
- sed '$d' | \
- tr ":" "\n" | \
- awk -F '/' '{
- artifact_id=$(NF-2);
- version=$(NF-1);
- jar_name=$NF;
- classifier_start_index=length(artifact_id"-"version"-") + 1;
- classifier_end_index=index(jar_name, ".jar") - 1;
- classifier=substr(jar_name, classifier_start_index, classifier_end_index
- classifier_start_index + 1);
- print artifact_id"/"version"/"classifier"/"jar_name
- }' | sort -u >> "${DEP_PR}"
-}
-
function check_diff() {
set +e
the_diff=$(diff "${DEP}" "${DEP_PR}")
@@ -208,7 +206,7 @@ case "$MODULE" in
;;
*)
MODULE="server"
- MVN_MODULES="worker,master"
+ MVN_MODULES="worker,master,cli"
;;
esac
@@ -258,7 +256,6 @@ if [ "$REPLACE" == "true" ]; then
exit 0
fi
-# skip check for sbt server now until sbt_build_server_classpath is refactored
-if [ "$CHECK" == "true" -a "$MODULE" != "server" ]; then
+if [ "$CHECK" == "true" ]; then
check_diff
fi
diff --git a/dev/deps/dependencies-client-mr b/dev/deps/dependencies-client-mr
index ac08f3ae6..cc979eb28 100644
--- a/dev/deps/dependencies-client-mr
+++ b/dev/deps/dependencies-client-mr
@@ -72,13 +72,13 @@ jackson-annotations/2.15.3//jackson-annotations-2.15.3.jar
jackson-core-asl/1.9.13//jackson-core-asl-1.9.13.jar
jackson-core/2.15.3//jackson-core-2.15.3.jar
jackson-databind/2.15.3//jackson-databind-2.15.3.jar
-jackson-jaxrs-base/2.12.7//jackson-jaxrs-base-2.12.7.jar
-jackson-jaxrs-json-provider/2.12.7//jackson-jaxrs-json-provider-2.12.7.jar
+jackson-jaxrs-base/2.15.3//jackson-jaxrs-base-2.15.3.jar
+jackson-jaxrs-json-provider/2.15.3//jackson-jaxrs-json-provider-2.15.3.jar
jackson-mapper-asl/1.9.13//jackson-mapper-asl-1.9.13.jar
-jackson-module-jaxb-annotations/2.12.7//jackson-module-jaxb-annotations-2.12.7.jar
+jackson-module-jaxb-annotations/2.15.3//jackson-module-jaxb-annotations-2.15.3.jar
jackson-module-scala_2.12/2.15.3//jackson-module-scala_2.12-2.15.3.jar
jakarta.activation-api/1.2.1//jakarta.activation-api-1.2.1.jar
-jakarta.xml.bind-api/2.3.2//jakarta.xml.bind-api-2.3.2.jar
+jakarta.xml.bind-api/2.3.3//jakarta.xml.bind-api-2.3.3.jar
javax-websocket-client-impl/9.4.51.v20230217//javax-websocket-client-impl-9.4.51.v20230217.jar
javax-websocket-server-impl/9.4.51.v20230217//javax-websocket-server-impl-9.4.51.v20230217.jar
javax.annotation-api/1.3.2//javax.annotation-api-1.3.2.jar
diff --git a/dev/deps/dependencies-server b/dev/deps/dependencies-server
index afaf9b7d1..9e5bceeb4 100644
--- a/dev/deps/dependencies-server
+++ b/dev/deps/dependencies-server
@@ -19,7 +19,6 @@ HikariCP/4.0.3//HikariCP-4.0.3.jar
RoaringBitmap/1.0.6//RoaringBitmap-1.0.6.jar
aopalliance-repackaged/2.6.1//aopalliance-repackaged-2.6.1.jar
ap-loader-all/3.0-8//ap-loader-all-3.0-8.jar
-aws-java-sdk-bundle/1.12.367//aws-java-sdk-bundle-1.12.367.jar
classgraph/4.8.138//classgraph-4.8.138.jar
commons-cli/1.5.0//commons-cli-1.5.0.jar
commons-crypto/1.0.0//commons-crypto-1.0.0.jar
@@ -28,7 +27,6 @@ commons-lang3/3.13.0//commons-lang3-3.13.0.jar
commons-logging/1.1.3//commons-logging-1.1.3.jar
failureaccess/1.0.2//failureaccess-1.0.2.jar
guava/33.1.0-jre//guava-33.1.0-jre.jar
-hadoop-aws/3.3.6//hadoop-aws-3.3.6.jar
hadoop-client-api/3.3.6//hadoop-client-api-3.3.6.jar
hadoop-client-runtime/3.3.6//hadoop-client-runtime-3.3.6.jar
hk2-api/2.6.1//hk2-api-2.6.1.jar
@@ -122,18 +120,19 @@
netty-transport-udt/4.1.109.Final//netty-transport-udt-4.1.109.Final.jar
netty-transport/4.1.109.Final//netty-transport-4.1.109.Final.jar
osgi-resource-locator/1.0.3//osgi-resource-locator-1.0.3.jar
paranamer/2.8//paranamer-2.8.jar
+picocli/4.7.6//picocli-4.7.6.jar
protobuf-java/3.21.7//protobuf-java-3.21.7.jar
ratis-client/3.1.0//ratis-client-3.1.0.jar
ratis-common/3.1.0//ratis-common-3.1.0.jar
ratis-grpc/3.1.0//ratis-grpc-3.1.0.jar
-ratis-metrics-default/3.1.0/ratis-metrics-default-3.1.0.jar
+ratis-metrics-api/3.1.0//ratis-metrics-api-3.1.0.jar
+ratis-metrics-default/3.1.0//ratis-metrics-default-3.1.0.jar
ratis-netty/3.1.0//ratis-netty-3.1.0.jar
ratis-proto/3.1.0//ratis-proto-3.1.0.jar
ratis-server-api/3.1.0//ratis-server-api-3.1.0.jar
ratis-server/3.1.0//ratis-server-3.1.0.jar
ratis-shell/3.1.0//ratis-shell-3.1.0.jar
-ratis-thirdparty-misc/1.0.5//ratis-thirdparty-misc-1.0.5.jar
-reflections/0.10.2//reflections-0.10.2.jar
+ratis-thirdparty-misc/1.0.6//ratis-thirdparty-misc-1.0.6.jar
rocksdbjni/8.11.3//rocksdbjni-8.11.3.jar
scala-library/2.12.18//scala-library-2.12.18.jar
scala-reflect/2.12.18//scala-reflect-2.12.18.jar
diff --git a/openapi/openapi-client/pom.xml b/openapi/openapi-client/pom.xml
index 8b05ae0b0..e608d3e30 100644
--- a/openapi/openapi-client/pom.xml
+++ b/openapi/openapi-client/pom.xml
@@ -29,22 +29,6 @@
<packaging>jar</packaging>
<name>Celeborn OpenAPI Client</name>
- <dependencyManagement>
- <dependencies>
- <dependency>
- <groupId>com.fasterxml.jackson.jaxrs</groupId>
- <artifactId>jackson-jaxrs-json-provider</artifactId>
- <version>${jackson.version}</version>
- <exclusions>
- <exclusion>
- <groupId>jakarta.activation</groupId>
- <artifactId>jakarta.activation-api</artifactId>
- </exclusion>
- </exclusions>
- </dependency>
- </dependencies>
- </dependencyManagement>
-
<dependencies>
<dependency>
<groupId>com.fasterxml.jackson.core</groupId>
diff --git a/pom.xml b/pom.xml
index 9f91c1552..48069d96e 100644
--- a/pom.xml
+++ b/pom.xml
@@ -692,6 +692,28 @@
<artifactId>jackson-datatype-jsr310</artifactId>
<version>${jackson.version}</version>
</dependency>
+ <dependency>
+ <groupId>com.fasterxml.jackson.jaxrs</groupId>
+ <artifactId>jackson-jaxrs-json-provider</artifactId>
+ <version>${jackson.version}</version>
+ <exclusions>
+ <exclusion>
+ <groupId>jakarta.activation</groupId>
+ <artifactId>jakarta.activation-api</artifactId>
+ </exclusion>
+ </exclusions>
+ </dependency>
+ <dependency>
+ <groupId>com.fasterxml.jackson.module</groupId>
+ <artifactId>jackson-module-jaxb-annotations</artifactId>
+ <version>${jackson.version}</version>
+ <exclusions>
+ <exclusion>
+ <groupId>jakarta.activation</groupId>
+ <artifactId>jakarta.activation-api</artifactId>
+ </exclusion>
+ </exclusions>
+ </dependency>
<dependency>
<groupId>org.xerial.snappy</groupId>
<artifactId>snappy-java</artifactId>
diff --git a/project/CelebornBuild.scala b/project/CelebornBuild.scala
index eff7349d3..5bc1291e7 100644
--- a/project/CelebornBuild.scala
+++ b/project/CelebornBuild.scala
@@ -63,6 +63,7 @@ object Dependencies {
val roaringBitmapVersion = "1.0.6"
val rocksdbJniVersion = "8.11.3"
val jacksonVersion = "2.15.3"
+ val jakartaActivationApiVersion = "1.2.1"
val scalatestMockitoVersion = "1.17.14"
val scalatestVersion = "3.2.16"
val slf4jVersion = "1.7.36"
@@ -79,7 +80,7 @@ object Dependencies {
val openApiToolsJacksonBindNullableVersion = "0.2.6"
val httpClient5Version = "5.3.1"
val httpCore5Version = "5.2.4"
- val javaxAnnotationApiVersion = "1.3.2"
+ val jakartaAnnotationApiVersion = "1.3.5"
val picocliVersion = "4.7.6"
// For SSL support
@@ -111,6 +112,7 @@ object Dependencies {
ExclusionRule("com.google.guava", "guava"),
ExclusionRule("com.fasterxml.jackson.core", "jackson-annotations"),
ExclusionRule("com.fasterxml.jackson.core", "jackson-databind"),
+ ExclusionRule("jakarta.activation", "jakarta.activation-api"),
ExclusionRule("jline", "jline"),
ExclusionRule("log4j", "log4j"),
ExclusionRule("org.slf4j", "slf4j-log4j12"))
@@ -152,8 +154,11 @@ object Dependencies {
val jacksonDataTypeJsr310 = "com.fasterxml.jackson.datatype" %
"jackson-datatype-jsr310" % jacksonVersion
val jacksonDataFormatYam = "com.fasterxml.jackson.dataformat" %
"jackson-dataformat-yaml" % jacksonVersion
val jacksonJaxrsBase = "com.fasterxml.jackson.jaxrs" % "jackson-jaxrs-base"
% jacksonVersion
- val jacksonJaxrsJsonProvider = "com.fasterxml.jackson.jaxrs" %
"jackson-jaxrs-json-provider" % jacksonVersion
- val jacksonModuleJaxbAnnotations = "com.fasterxml.jackson.module" %
"jackson-module-jaxb-annotations" % jacksonVersion
+ val jacksonJaxrsJsonProvider = "com.fasterxml.jackson.jaxrs" %
"jackson-jaxrs-json-provider" % jacksonVersion excludeAll (
+ ExclusionRule("jakarta.activation", "jakarta.activation-api"))
+ val jacksonModuleJaxbAnnotations = "com.fasterxml.jackson.module" %
"jackson-module-jaxb-annotations" % jacksonVersion excludeAll (
+ ExclusionRule("jakarta.activation", "jakarta.activation-api"))
+ val jakartaActivationApi = "jakarta.activation" % "jakarta.activation-api" %
jakartaActivationApiVersion
val scalaReflect = "org.scala-lang" % "scala-reflect" % projectScalaVersion
val slf4jApi = "org.slf4j" % "slf4j-api" % slf4jVersion
val slf4jJulToSlf4j = "org.slf4j" % "jul-to-slf4j" % slf4jVersion
@@ -185,7 +190,7 @@ object Dependencies {
val httpClient5 = "org.apache.httpcomponents.client5" % "httpclient5" %
httpClient5Version
val httpCore5 = "org.apache.httpcomponents.core5" % "httpcore5" %
httpCore5Version
val httpCore5H2 = "org.apache.httpcomponents.core5" % "httpcore5-h2" %
httpCore5Version
- val javaxAnnotationApi = "javax.annotation" % "javax.annotation-api" %
javaxAnnotationApiVersion
+ val jakartaAnnotationApi = "jakarta.annotation" % "jakarta.annotation-api" %
jakartaAnnotationApiVersion
// Test dependencies
// https://www.scala-sbt.org/1.x/docs/Testing.html
@@ -585,6 +590,7 @@ object CelebornService {
Dependencies.slf4jApi,
Dependencies.mybatis,
Dependencies.hikaricp,
+ Dependencies.jacksonDataFormatYam,
Dependencies.swaggerJaxrs2,
Dependencies.swaggerUi,
Dependencies.jakartaServletApi,
@@ -1200,7 +1206,9 @@ object MRClientProjects {
libraryDependencies ++= Seq(
Dependencies.hadoopClientApi,
Dependencies.hadoopClientRuntime,
- Dependencies.hadoopMapreduceClientApp
+ Dependencies.hadoopMapreduceClientApp,
+ Dependencies.jacksonJaxrsJsonProvider,
+ Dependencies.jakartaActivationApi
) ++ commonUnitTestDependencies,
dependencyOverrides += Dependencies.commonsCompress
)
@@ -1392,7 +1400,7 @@ object CelebornOpenApi {
Dependencies.jacksonDataTypeJsr310,
Dependencies.jacksonJaxrsJsonProvider,
Dependencies.findbugsJsr305,
- Dependencies.javaxAnnotationApi,
+ Dependencies.jakartaAnnotationApi,
Dependencies.httpClient5,
Dependencies.httpCore5,
Dependencies.httpCore5H2,
diff --git a/service/pom.xml b/service/pom.xml
index a95a8b820..9ec2d7a5e 100644
--- a/service/pom.xml
+++ b/service/pom.xml
@@ -56,21 +56,6 @@
<artifactId>jetty-proxy</artifactId>
<version>${jetty.version}</version>
</dependency>
- <dependency>
- <groupId>com.fasterxml.jackson.jaxrs</groupId>
- <artifactId>jackson-jaxrs-base</artifactId>
- <version>${jackson.version}</version>
- </dependency>
- <dependency>
- <groupId>com.fasterxml.jackson.jaxrs</groupId>
- <artifactId>jackson-jaxrs-json-provider</artifactId>
- <version>${jackson.version}</version>
- </dependency>
- <dependency>
- <groupId>com.fasterxml.jackson.module</groupId>
- <artifactId>jackson-module-jaxb-annotations</artifactId>
- <version>${jackson.version}</version>
- </dependency>
</dependencies>
</dependencyManagement>
