This is an automated email from the ASF dual-hosted git repository.
sivabalan pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/hudi.git
The following commit(s) were added to refs/heads/master by this push:
new 55a489c [1568] Fixing spark3 bundles (#2625)
55a489c is described below
commit 55a489c769c7681d2732aa99f963d02d26ed991b
Author: Sivabalan Narayanan <[email protected]>
AuthorDate: Fri Mar 19 14:21:36 2021 -0400
[1568] Fixing spark3 bundles (#2625)
- [1568] Fixing spark3 bundles
---
docker/hoodie/hadoop/hive_base/pom.xml | 2 +-
docker/hoodie/hadoop/pom.xml | 2 +-
packaging/hudi-spark-bundle/pom.xml | 2 +-
pom.xml | 4 ++++
scripts/release/deploy_staging_jars.sh | 6 ++++--
5 files changed, 11 insertions(+), 5 deletions(-)
diff --git a/docker/hoodie/hadoop/hive_base/pom.xml
b/docker/hoodie/hadoop/hive_base/pom.xml
index 163fac2..62ea4c1 100644
--- a/docker/hoodie/hadoop/hive_base/pom.xml
+++ b/docker/hoodie/hadoop/hive_base/pom.xml
@@ -58,7 +58,7 @@
<tasks>
<copy
file="${project.basedir}/../../../../packaging/hudi-hadoop-mr-bundle/target/hudi-hadoop-mr-bundle-${project.version}.jar"
tofile="target/hoodie-hadoop-mr-bundle.jar" />
<copy
file="${project.basedir}/../../../../packaging/hudi-hive-sync-bundle/target/hudi-hive-sync-bundle-${project.version}.jar"
tofile="target/hoodie-hive-sync-bundle.jar" />
- <copy
file="${project.basedir}/../../../../packaging/hudi-spark-bundle/target/hudi-spark-bundle_${scala.binary.version}-${project.version}.jar"
tofile="target/hoodie-spark-bundle.jar" />
+ <copy
file="${project.basedir}/../../../../packaging/hudi-spark-bundle/target/hudi-spark${sparkbundle.version}-bundle_${scala.binary.version}-${project.version}.jar"
tofile="target/hoodie-spark-bundle.jar" />
<copy
file="${project.basedir}/../../../../packaging/hudi-utilities-bundle/target/hudi-utilities-bundle_${scala.binary.version}-${project.version}.jar"
tofile="target/hoodie-utilities.jar" />
</tasks>
</configuration>
diff --git a/docker/hoodie/hadoop/pom.xml b/docker/hoodie/hadoop/pom.xml
index 64b9342..612067d 100644
--- a/docker/hoodie/hadoop/pom.xml
+++ b/docker/hoodie/hadoop/pom.xml
@@ -42,7 +42,7 @@
<dependencies>
<dependency>
<groupId>org.apache.hudi</groupId>
- <artifactId>hudi-spark-bundle_${scala.binary.version}</artifactId>
+
<artifactId>hudi-spark${sparkbundle.version}-bundle_${scala.binary.version}</artifactId>
<version>${project.version}</version>
</dependency>
</dependencies>
diff --git a/packaging/hudi-spark-bundle/pom.xml
b/packaging/hudi-spark-bundle/pom.xml
index e20123e..21d8638 100644
--- a/packaging/hudi-spark-bundle/pom.xml
+++ b/packaging/hudi-spark-bundle/pom.xml
@@ -23,7 +23,7 @@
<relativePath>../../pom.xml</relativePath>
</parent>
<modelVersion>4.0.0</modelVersion>
- <artifactId>hudi-spark-bundle_${scala.binary.version}</artifactId>
+
<artifactId>hudi-spark${sparkbundle.version}-bundle_${scala.binary.version}</artifactId>
<packaging>jar</packaging>
<properties>
diff --git a/pom.xml b/pom.xml
index ad726c0..36775e2 100644
--- a/pom.xml
+++ b/pom.xml
@@ -106,9 +106,12 @@
<prometheus.version>0.8.0</prometheus.version>
<http.version>4.4.1</http.version>
<spark.version>${spark2.version}</spark.version>
+ <sparkbundle.version>${spark2bundle.version}</sparkbundle.version>
<flink.version>1.11.2</flink.version>
<spark2.version>2.4.4</spark2.version>
<spark3.version>3.0.0</spark3.version>
+ <spark2bundle.version></spark2bundle.version>
+ <spark3bundle.version>3</spark3bundle.version>
<avro.version>1.8.2</avro.version>
<scala11.version>2.11.12</scala11.version>
<scala12.version>2.12.10</scala12.version>
@@ -1459,6 +1462,7 @@
<id>spark3</id>
<properties>
<spark.version>${spark3.version}</spark.version>
+ <sparkbundle.version>${spark3bundle.version}</sparkbundle.version>
<scala.version>${scala12.version}</scala.version>
<scala.binary.version>2.12</scala.binary.version>
<kafka.version>2.4.1</kafka.version>
diff --git a/scripts/release/deploy_staging_jars.sh
b/scripts/release/deploy_staging_jars.sh
index 885c25f..b2c5bf3 100755
--- a/scripts/release/deploy_staging_jars.sh
+++ b/scripts/release/deploy_staging_jars.sh
@@ -21,7 +21,7 @@
## Variables with defaults (if not overwritten by environment)
##
MVN=${MVN:-mvn}
-
+SPARK_VERSION=2
# fail immediately
set -o errexit
set -o nounset
@@ -44,6 +44,8 @@ else
do
if [[ $param =~ --scala_version\=(2\.1[1-2]) ]]; then
SCALA_VERSION=${BASH_REMATCH[1]}
+ elif [[ $param =~ --spark_version\=([2-3]) ]]; then
+ SPARK_VERSION=${BASH_REMATCH[0]}
fi
done
fi
@@ -54,5 +56,5 @@ cd ..
echo "Deploying to repository.apache.org with scala version ${SCALA_VERSION}"
-COMMON_OPTIONS="-Dscala-${SCALA_VERSION} -Prelease -DskipTests
-DretryFailedDeploymentCount=10 -DdeployArtifacts=true"
+COMMON_OPTIONS="-Dscala-${SCALA_VERSION} -Dspark${SPARK_VERSION} -Prelease
-DskipTests -DretryFailedDeploymentCount=10 -DdeployArtifacts=true"
$MVN clean deploy $COMMON_OPTIONS
