This is an automated email from the ASF dual-hosted git repository. stevel pushed a commit to branch main in repository https://gitbox.apache.org/repos/asf/hadoop-release-support.git
The following commit(s) were added to refs/heads/main by this push: new e818784 HADOOP-19565. Release Hadoop 3.4.2: improving release commands e818784 is described below commit e8187849d60c6cda8f30fc654d6fc34cca2c325e Author: Steve Loughran <ste...@cloudera.com> AuthorDate: Tue Jun 3 15:01:22 2025 +0100 HADOOP-19565. Release Hadoop 3.4.2: improving release commands --- README.md | 29 ++++++++++++----- build.xml | 37 +++++++--------------- pom.xml | 16 ++++++++-- src/releases/release-info-3.4.2.properties | 1 - .../apache/hadoop/validator/TestRuntimeValid.java | 5 +-- 5 files changed, 47 insertions(+), 41 deletions(-) diff --git a/README.md b/README.md index 895f6ca..8cfda36 100644 --- a/README.md +++ b/README.md @@ -469,10 +469,15 @@ locally. # How to download and build a staged release candidate -This relies on the release-info file pointing to the source directory +This project can be used to download and validated a release created by other people, +downloading the staged artifacts and validating their signatures before +executing some (minimal) commands. + +This relies on the relevant `release-info-` file declaring the URL to download the artifacts from, and the maven staging repository. + ```properties -http.source=https://dist.apache.org/repos/dist/dev/hadoop/hadoop-${hadoop.version}-RC${rc}/ +amd.src.dir=https://dist.apache.org/repos/dist/dev/hadoop/hadoop-${hadoop.version}-RC${rc}/ ``` ### Targets of Relevance @@ -490,10 +495,9 @@ http.source=https://dist.apache.org/repos/dist/dev/hadoop/hadoop-${hadoop.versio | `release.bin.commands` | execute a series of commands against the untarred binaries | | `release.site.untar` | untar the downloaded site artifact | | `release.site.validate` | perform minimal validation of the site. | -| `release.arm.untar` | untar the ARM binary file | -| `release.arm.commands` | execute commands against the arm binaries | | `release.lean.tar` | create a release of the x86 binary tar without the AWS SDK | + set `check.native.binaries` to false to skip native binary checks on platforms without them ### Download the Staged RC files from the Apache http servers @@ -539,11 +543,11 @@ in by declaring them in the property `source.compile.maven.args` ```properties source.compile.maven.args=-Pnative ``` -These are added at the end of the hard-coded arguments (`clean install -DskipTests`) +These are added at the end of the hard-coded arguments (`ant clean install -DskipTests`) Testing is also possible through the target `release.src.test` -``` +```bash ant release.src.test ``` Again, the options set in `source.compile.maven.args` are passed down. @@ -594,10 +598,19 @@ If `check.native.binaries` is false, the `bin/hadoop checknative` is still executed, with the outcome printed (reporting a failure if the binaries are not present). -The and build itself is successful. +The ant build itself will succeed, even if the `checknative` command reports a failure. ## Testing ARM binaries +There are ARM variants of the commands to fetch and validate the ARM binaries. + +| target | action | +|-------------------------|------------------------------------------------------------| +| `release.fetch.arm` | fetch ARM artifacts | +| `gpg.arm.verify` | verify ARM artifacts | +| `release.arm.untar` | untar the ARM binary file | +| `release.arm.commands` | execute commands against the arm binaries | + ```bash # untars the `-aarch64.tar.gz` binary ant release.arm.untar @@ -680,7 +693,6 @@ Review this to make sure there are no unexpected artifacts coming in, ant cloudstore.build ``` -Note: this does not include the AWS V1 SDK `-Pextra` profile. ## Build and test Google GCS @@ -747,6 +759,7 @@ ant parquet.test ``` + # After the Vote Succeeds: publishing the release ## Update the announcement and create site/email notifications diff --git a/build.xml b/build.xml index 9c1c220..0441213 100644 --- a/build.xml +++ b/build.xml @@ -218,7 +218,6 @@ spark.version=${spark.version} cloudstore.dir=${cloudstore.dir} - cloudstore.profile=${cloudstore.profile} bigdata-interop.dir=${bigdata-interop.dir} hboss.dir=${hboss.dir} cloud-examples.dir=${cloud-examples.dir} @@ -248,7 +247,7 @@ <macrodef name="gpg-verify-file" > <attribute name="dir" /> - <attribute name="file" /> + <attribute name="name" /> <sequential> <echo>Verifying GPG signature of @{dir}/@{name}</echo> <gpg dir="@{dir}"> @@ -485,6 +484,13 @@ <gpgverify name="${release}-site.tar.gz"/> <gpgverify name="${release}.tar.gz"/> <gpgverify name="${release}-rat.txt"/> + + </target> + + <target name="gpg.arm.verify" depends="release.dir.check" + description="verify the downloaded arm artifacts"> + + <echo>Verifying GPG signatures of arm artifacts in ${release.dir}</echo> <gpgverify name="${arm.binary.filename}"/> </target> @@ -940,11 +946,8 @@ Message is in file ${message.out} if this is done with java11, it shouldn't be released. </echo> - <require p="cloudstore.profile"/> - <mvn dir="${cloudstore.dir}"> <arg value="-Psnapshots-and-staging"/> - <arg value="-P${cloudstore.profile}"/> <arg value="-Dhadoop.version=${hadoop.version}"/> <arg value="clean"/> <arg value="package"/> @@ -952,26 +955,6 @@ Message is in file ${message.out} </mvn> </target> - <!-- ========================================================= --> - <!-- Hadoop FS API shim WiP --> - <!-- ========================================================= --> - - - <target name="fsapi.test" if="fs-api-shim.dir" - depends="init" - description="Build and test fs-api-shim"> - <echo> - Build the fs api shim module. - This MUST build against hadoop-3.2.0; the test version is - what we want here. - </echo> - <mvn dir="${fs-api-shim.dir}"> - <arg value="-Psnapshots-and-staging"/> - <arg value="-Dhadoop.test.version=${hadoop.version}"/> - <arg value="clean"/> - <arg value="test"/> - </mvn> - </target> <!-- ========================================================= --> <!-- Parquet --> @@ -1120,7 +1103,6 @@ Message is in file ${message.out} <download artifact="CHANGELOG.md"/> <download artifact="RELEASENOTES.md"/> <download artifact="${release}.tar.gz"/> - <download artifact="${release}-aarch64.tar.gz"/> <download artifact="${release}-site.tar.gz"/> <download artifact="${release}-src.tar.gz"/> <download artifact="${release}-rat.txt"/> @@ -1224,6 +1206,9 @@ Message is in file ${message.out} <echo>checknative</echo> + <echo> + Checking for native binaries; check.native.binaries=${check.native.binaries} + </echo> <hadoopq failonerror="${check.native.binaries}"> <arg value="checknative" /> </hadoopq> diff --git a/pom.xml b/pom.xml index 9ee9681..3db252a 100644 --- a/pom.xml +++ b/pom.xml @@ -29,8 +29,9 @@ <enforced.java.version>${javac.version}</enforced.java.version> <maven-antrun-plugin.version>3.1.0</maven-antrun-plugin.version> + <aws-java-sdk-v2.version>2.29.52</aws-java-sdk-v2.version> - <hadoop.version>3.4.0</hadoop.version> + <hadoop.version>3.4.2</hadoop.version> </properties> @@ -242,6 +243,17 @@ <version>${hadoop.version}</version> </dependency> + <dependency> + <groupId>software.amazon.awssdk</groupId> + <artifactId>bundle</artifactId> + <version>${aws-java-sdk-v2.version}</version> + <exclusions> + <exclusion> + <groupId>*</groupId> + <artifactId>*</artifactId> + </exclusion> + </exclusions> + </dependency> <dependency> <groupId>junit</groupId> <artifactId>junit</artifactId> @@ -309,7 +321,7 @@ <profile> <id>branch-3.4</id> <properties> - <hadoop.version>3.4.1</hadoop.version> + <hadoop.version>3.4.3-SNAPSHOT</hadoop.version> </properties> </profile> diff --git a/src/releases/release-info-3.4.2.properties b/src/releases/release-info-3.4.2.properties index cd1099d..a3656b2 100644 --- a/src/releases/release-info-3.4.2.properties +++ b/src/releases/release-info-3.4.2.properties @@ -28,5 +28,4 @@ amd.src.dir=https://dist.apache.org/repos/dist/dev/hadoop/hadoop-3.4.2-RC1 arm.src.dir=${amd.src.dir} http.source=${amd.src.dir} asf.staging.url=https://repository.apache.org/content/repositories/orgapachehadoop-1437 -cloudstore.profile=sdk2 diff --git a/src/test/java/org/apache/hadoop/validator/TestRuntimeValid.java b/src/test/java/org/apache/hadoop/validator/TestRuntimeValid.java index 87e56ad..ec3336a 100644 --- a/src/test/java/org/apache/hadoop/validator/TestRuntimeValid.java +++ b/src/test/java/org/apache/hadoop/validator/TestRuntimeValid.java @@ -22,10 +22,7 @@ import org.junit.Test; import org.apache.hadoop.fs.azurebfs.AzureBlobFileSystem; import org.apache.hadoop.fs.s3a.S3AFileSystem; -import org.apache.hadoop.hdfs.DistributedFileSystem; -import org.apache.hadoop.validator.CompileFS; - -/** +import org.apache.hadoop.hdfs.DistributedFileSystem;/** * Let's test that runtime. */ public class TestRuntimeValid { --------------------------------------------------------------------- To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org For additional commands, e-mail: common-commits-h...@hadoop.apache.org