This is an automated email from the ASF dual-hosted git repository.
stevel pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/hadoop-release-support.git
The following commit(s) were added to refs/heads/main by this push:
new fad0dc7 Hadoop 3.4.0 RC validation
fad0dc7 is described below
commit fad0dc79c41f9e5e598056680aee5ba1de230258
Author: Steve Loughran <ste...@cloudera.com>
AuthorDate: Wed Feb 28 18:03:17 2024 +0000
Hadoop 3.4.0 RC validation
* spark builds with SPARK-41392 applied
* cloudera cloud integration module compiles against spark 4/scala
2.13/3.4.0. test failures
---
README.md | 13 +++-
build.xml | 77 ++++++++++++++++++----
....1.properties => release-info-3.3.6.properties} | 14 ++--
src/releases/release-info-3.4.1.properties | 4 +-
4 files changed, 85 insertions(+), 23 deletions(-)
diff --git a/README.md b/README.md
index 30bfdbf..5dfac2b 100644
--- a/README.md
+++ b/README.md
@@ -500,10 +500,19 @@ Validates hadoop client artifacts; the cloud tests cover
hadoop cloud storage cl
ant spark.build
```
-And to to run the `hadoop-cloud` tests
+To view the `hadoop-cloud` dependencies
```bash
-ant spark.test.hadoop-cloud
+ant spark.hadoop-cloud.dependencies
+```
+
+Review this to look for conflict.
+
+
+To run the `hadoop-cloud` tests
+
+```bash
+ant spark.hadoop-cloud.test
```
A full spark test run takes so long that CI infrastructure should be used.
diff --git a/build.xml b/build.xml
index b4d194e..950602d 100644
--- a/build.xml
+++ b/build.xml
@@ -86,8 +86,8 @@
<!-- set immutable properties for this build -->
- <setpath name="dist.dir" location="${downloads.dir}/dist"/>
<setpath name="downloads.dir" location="downloads"/>
+ <setpath name="dist.dir" location="${downloads.dir}/dist"/>
<setpath name="home" location="${user.home}"/>
<setpath name="incoming.dir" location="${downloads.dir}/incoming"/>
<setpath name="src" location="src"/>
@@ -183,6 +183,9 @@
<setpath name="arm.binary.sha512" location="${arm.binary}.sha512" />
<setpath name="arm.binary.asc" location="${arm.binary}.asc" />
+ <!-- for spark builds -->
+ <set name="spark.version" value="4.0.0-SNAPSHOT"/>
+
<echo>
subsidiary build options only if explicitly set
@@ -192,6 +195,7 @@
spark.version=${spark.version}
cloudstore.dir=${cloudstore.dir}
+ cloudstore.profile=${cloudstore.profile}
bigdata-interop.dir=${bigdata-interop.dir}
hboss.dir=${hboss.dir}
cloud-examples.dir=${cloud-examples.dir}
@@ -327,7 +331,7 @@
<mvn output="${mvndeps.out}">
<arg value="dependency:tree"/>
<arg value="-Pstaging"/>
- <arg value="-Dverbose"/>
+ <arg value="-Ddetail=true"/>
<arg value="-Dhadoop.version=${hadoop.version}"/>
</mvn>
<!-- load and print the output -->
@@ -628,19 +632,20 @@ Message is in file ${message.out}
depends="init">
- <!-- for spark builds -->
- <set name="spark.version" value="3.5.0-SNAPSHOT"/>
+
<!-- spark excludes hadoop-aws dependency and forces in their own
this fixes it to be in sync with hadoop
see https://issues.apache.org/jira/browse/SPARK-39969
-->
<set name="spark.aws.version" value="1.12.316"/>
+ <set name="cloud.examples.spark.profile" value="spark-master"/>
+ <set name="cloud.examples.hadoop.profile" value="hadoop-3.4"/>
</target>
<target name="spark.build" if="spark.dir"
depends="spark.init"
- description="build the spark release in spark.dir">
+ description="build the spark release in spark.dir. Requires java 17">
<echo>
Note: this build includes kinesis and hadoop cloud artifacts;
@@ -666,8 +671,31 @@ Message is in file ${message.out}
</mvn>
</target>
- <target name="spark.test.hadoop-cloud" if="spark.dir"
- depends="init"
+ <target name="spark.hadoop-cloud.dependencies" if="spark.dir"
+ depends="spark.init"
+ description="run the spark-hadoop-cloud tests in spark.dir">
+ <echo>
+ Collect and report spark hadoop-cloud dependencies
+ Requires the full spark build to have already been executed
+ </echo>
+ <!-- execute mvn dependency:tree saving the output to a file -->
+ <setpath name="mvndeps-spark.txt" location="${target}/mvndeps-spark.txt"/>
+ <mvn dir="${spark.dir}" output="${mvndeps-spark.txt}">
+ <arg value="-Psnapshots-and-staging"/>
+ <arg value="-Phadoop-cloud"/> <!-- cloud module -->
+ <arg value="--pl"/>
+ <arg value="hadoop-cloud"/>
+ <arg value="-Dmaven.javadoc.skip=true"/>
+ <arg value="-Dhadoop.version=${hadoop.version}"/>
+ <arg value="dependency:tree"/>
+ <arg value="-Ddetail=true"/>
+ </mvn>
+ <loadfile property="mvndeps-spark" srcFile="${mvndeps-spark.txt}"/>
+ <echo>${mvndeps-spark}</echo>
+ </target>
+
+ <target name="spark.hadoop-cloud.test" if="spark.dir"
+ depends="spark.init"
description="run the spark-hadoop-cloud tests in spark.dir">
<echo>
Test spark hadoop-cloud.
@@ -686,15 +714,16 @@ Message is in file ${message.out}
<target name="cloud-examples.build" if="cloud-examples.dir"
- depends="init"
+ depends="spark.init"
description="build the cloud examples release">
<echo>
Build the cloud examples;
spark MUST have been built against this release first.
</echo>
<mvn dir="${cloud-examples.dir}">
- <arg value="-Psnapshots-and-staging"/>
- <arg value="-Dspark-3.4"/>
+ <arg value="-Pstaging"/>
+ <arg value="-P${cloud.examples.spark.profile}"/>
+ <arg value="-P${cloud.examples.hadoop.profile}"/>
<arg value="-Dspark.version=${spark.version}"/>
<arg value="-Dhadoop.version=${hadoop.version}"/>
<arg value="clean"/>
@@ -703,6 +732,29 @@ Message is in file ${message.out}
</mvn>
</target>
+ <target name="cloud-examples.dependencies" if="cloud-examples.dir"
+ depends="spark.init"
+ description="build the cloud examples release">
+ <echo>
+ Dependencies of the cloud examples;
+ spark MUST have been built against this release first.
+ </echo>
+ <!-- execute mvn dependency:tree saving the output to a file -->
+ <setpath name="mvndeps-cloud-examples.txt"
location="${target}/mvndeps-cloud-examples.txt"/>
+ <mvn dir="${cloud-examples.dir}"
+ output="${mvndeps-cloud-examples.txt}">
+ <arg value="-Pstaging"/>
+ <arg value="-P${cloud.examples.spark.profile}"/>
+ <arg value="-P${cloud.examples.hadoop.profile}"/>
+ <arg value="-Dspark.version=${spark.version}"/>
+ <arg value="-Dhadoop.version=${hadoop.version}"/>
+ <arg value="dependency:tree"/>
+ <arg value="-Ddetail=true"/>
+ </mvn>
+ <loadfile property="mvndeps-cloud-examples"
srcFile="${mvndeps-cloud-examples.txt}"/>
+ <echo>${mvndeps-cloud-examples}</echo>
+ </target>
+
<target name="cloud-examples.test"
if="cloud-examples.dir"
@@ -716,8 +768,9 @@ Message is in file ${message.out}
</echo>
<require p="cloud.test.configuration.file" />
<mvn dir="${cloud-examples.dir}">
- <arg value="-Psnapshots-and-staging"/>
- <arg value="-Dspark-3.4"/>
+ <arg value="-Pstaging"/>
+ <arg value="-P${cloud.examples.spark.profile}"/>
+ <arg value="-P${cloud.examples.hadoop.profile}"/>
<arg value="-Pscale"/>
<arg value="-Dscale.test.enabled=true"/>
<arg value="-Dspark.version=${spark.version}"/>
diff --git a/src/releases/release-info-3.4.1.properties
b/src/releases/release-info-3.3.6.properties
similarity index 85%
copy from src/releases/release-info-3.4.1.properties
copy to src/releases/release-info-3.3.6.properties
index a2f48d9..766b1ba 100644
--- a/src/releases/release-info-3.4.1.properties
+++ b/src/releases/release-info-3.3.6.properties
@@ -14,15 +14,15 @@
# See the License for the specific language governing permissions and
# limitations under the License.
-# property file for 3.4.1
-hadoop.version=3.4.1-SNAPSHOT
+# property file for3.3.6
+hadoop.version=3.3.6
rc=RC0
-previous.version=3.4.1
-release.branch=3.4
-git.commit.id=branch-3.4
+previous.version=3.4.0
+release.branch=branch-3.3.6
+git.commit.id=1be78238728
-jira.id=HADOOP-XXXXXX
-jira.title=Release 3.4.1
+jira.id=HADOOP-12345
+jira.title=Release 3.3.6
amd.src.dir=https://dist.apache.org/repos/dist/dev/hadoop/hadoop-3.4.0-RC2
arm.src.dir=${amd.src.dir}
diff --git a/src/releases/release-info-3.4.1.properties
b/src/releases/release-info-3.4.1.properties
index a2f48d9..463b063 100644
--- a/src/releases/release-info-3.4.1.properties
+++ b/src/releases/release-info-3.4.1.properties
@@ -17,11 +17,11 @@
# property file for 3.4.1
hadoop.version=3.4.1-SNAPSHOT
rc=RC0
-previous.version=3.4.1
+previous.version=3.4.0
release.branch=3.4
git.commit.id=branch-3.4
-jira.id=HADOOP-XXXXXX
+jira.id=HADOOP-19087
jira.title=Release 3.4.1
amd.src.dir=https://dist.apache.org/repos/dist/dev/hadoop/hadoop-3.4.0-RC2
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org
For additional commands, e-mail: common-commits-h...@hadoop.apache.org
