This is an automated email from the ASF dual-hosted git repository. echobravo pushed a commit to branch release/1.12.0 in repository https://gitbox.apache.org/repos/asf/geode.git
The following commit(s) were added to refs/heads/release/1.12.0 by this push: new 6293da1 GEODE-7915: rc pipeline fixes and improvements (#4853) 6293da1 is described below commit 6293da1b7280e6e8811403b871bcaa51b19b08cb Author: Owen Nichols <34043438+onichols-pivo...@users.noreply.github.com> AuthorDate: Fri Mar 27 16:17:50 2020 -0700 GEODE-7915: rc pipeline fixes and improvements (#4853) * remove examples.zip test from pipeline as .zip is no longer produced * check all source artifacts to ensure they do not contain any binaries * check that no unexpected artifacts are present * align release retention with "N-2" proposal (cherry picked from commit 80ffe65afeb84223faa75eebca77ad39e29f7f5a) --- dev-tools/release/deploy_rc_pipeline.sh | 114 +++++++++++++++++++++----------- dev-tools/release/finalize_release.sh | 4 +- 2 files changed, 78 insertions(+), 40 deletions(-) diff --git a/dev-tools/release/deploy_rc_pipeline.sh b/dev-tools/release/deploy_rc_pipeline.sh index daf9b62..e5a11d5 100755 --- a/dev-tools/release/deploy_rc_pipeline.sh +++ b/dev-tools/release/deploy_rc_pipeline.sh @@ -138,7 +138,7 @@ jobs: cd apache-geode-${VERSION}-src java -version ./gradlew test - - name: run-geode-examples + - name: run-geode-examples-jdk11 serial: true plan: - aggregate: @@ -151,7 +151,7 @@ jobs: type: docker-image source: repository: openjdk - tag: 8 + tag: 11 inputs: - name: geode-examples platform: linux @@ -164,7 +164,7 @@ jobs: cd geode-examples java -version ./gradlew runAll - - name: run-geode-examples-from-src-tar-gz + - name: run-geode-examples-from-src-tar-gz-jdk8 serial: true plan: - aggregate: @@ -187,7 +187,7 @@ jobs: - -ec - | set -ex - FULL_VERSION=$(cat geode-examples/gradle.properties | grep geodeReleaseUrl | sed -e 's#.*/geode/##') + FULL_VERSION=$(cd geode-examples && git describe --tags | sed -e 's#^rel/v##' -e 's#-.*##') VERSION=$(echo $FULL_VERSION|sed -e 's/\.RC.*//') STAGING_MAVEN=$(cat geode-examples/gradle.properties | grep geodeRepositoryUrl | awk '{print $3}') curl -s https://dist.apache.org/repos/dist/dev/geode/${FULL_VERSION}/apache-geode-examples-${VERSION}.tar.gz > src.tgz @@ -195,37 +195,6 @@ jobs: cd apache-geode-examples-${VERSION} java -version ./gradlew -PgeodeReleaseUrl=https://dist.apache.org/repos/dist/dev/geode/${FULL_VERSION} -PgeodeRepositoryUrl=${STAGING_MAVEN} build runAll - - name: run-geode-examples-from-src-zip-11 - serial: true - plan: - - aggregate: - - get: geode-examples - trigger: true - - task: validate - timeout: 1h - config: - image_resource: - type: docker-image - source: - repository: openjdk - tag: 11 - inputs: - - name: geode-examples - platform: linux - run: - path: /bin/sh - args: - - -ec - - | - set -ex - FULL_VERSION=$(cat geode-examples/gradle.properties | grep geodeReleaseUrl | sed -e 's#.*/geode/##') - VERSION=$(echo $FULL_VERSION|sed -e 's/\.RC.*//') - STAGING_MAVEN=$(cat geode-examples/gradle.properties | grep geodeRepositoryUrl | awk '{print $3}') - curl -s https://dist.apache.org/repos/dist/dev/geode/${FULL_VERSION}/apache-geode-examples-${VERSION}.zip > src.zip - unzip src.zip - cd apache-geode-examples-${VERSION} - java -version - ./gradlew -PgeodeReleaseUrl=https://dist.apache.org/repos/dist/dev/geode/${FULL_VERSION} -PgeodeRepositoryUrl=${STAGING_MAVEN} build runAll - name: build-geode-native-from-tag serial: true plan: @@ -284,6 +253,7 @@ jobs: repository: openjdk tag: 8 inputs: + - name: geode-native - name: geode platform: linux run: @@ -292,7 +262,7 @@ jobs: - -ec - | set -ex - FULL_VERSION=$(cd geode && git describe --tags | sed -e 's#^rel/v##') + FULL_VERSION=$(cd geode-native && git describe --tags | sed -e 's#^rel/v##') VERSION=$(echo $FULL_VERSION|sed -e 's/\.RC.*//') # build geode from source cd geode @@ -331,6 +301,7 @@ jobs: repository: openjdk tag: 8 inputs: + - name: geode - name: upthewaterspout-tests - name: geode-examples platform: linux @@ -340,7 +311,7 @@ jobs: - -ec - | set -ex - FULL_VERSION=$(cd geode-examples && git describe --tags | sed -e 's#^rel/v##') + FULL_VERSION=$(cd geode && git describe --tags | sed -e 's#^rel/v##') VERSION=$(echo $FULL_VERSION|sed -e 's/\.RC.*//') STAGING_MAVEN=$(cat geode-examples/gradle.properties | grep geodeRepositoryUrl | awk '{print $3}') cd upthewaterspout-tests @@ -384,7 +355,7 @@ jobs: echo "StrictHostKeyChecking no" >> /etc/ssh/ssh_config service ssh start ./gradlew build test - - name: verify-keys + - name: verify-expected-files-and-keys serial: true plan: - aggregate: @@ -423,12 +394,79 @@ jobs: curl -s $url/$sha > $sha gpg --verify $asc $sum -c $sha + echo $file >> exp + echo $asc >> exp + echo $sha >> exp } verifyArtifactSignature apache-geode-${VERSION}-src.tgz 256 verifyArtifactSignature apache-geode-${VERSION}.tgz 256 verifyArtifactSignature apache-geode-examples-${VERSION}.tar.gz 256 verifyArtifactSignature apache-geode-native-${VERSION}-src.tar.gz 512 verifyArtifactSignature apache-geode-benchmarks-${VERSION}-src.tgz 256 + curl -s ${url}/ | awk '/>..</{next}/<li>/{gsub(/ *<[^>]*>/,"");print}' | sort > actual-file-list + sort < exp > expected-file-list + set +x + echo "" + if diff -q expected-file-list actual-file-list ; then + echo "The file list at $url matches what is expected and all signatures were verified :)" + else + echo "Expected:" + cat expected-file-list + echo "" + echo "Actual:" + cat actual-file-list + echo "" + echo "Diff:" + diff expected-file-list actual-file-list + exit 1 + fi + - name: verify-no-binaries + serial: true + plan: + - aggregate: + - get: geode + trigger: true + - task: validate + timeout: 1h + config: + image_resource: + type: docker-image + source: + repository: openjdk + tag: 8 + inputs: + - name: geode + platform: linux + run: + path: /bin/bash + args: + - -ec + - | + set -e + FULL_VERSION=$(cd geode && git describe --tags | sed -e 's#^rel/v##') + VERSION=$(echo $FULL_VERSION|sed -e 's/\.RC.*//') + url=https://dist.apache.org/repos/dist/dev/geode/${FULL_VERSION} + BINARY_EXTENSIONS="jar|war|class|exe|dll|o|so|obj|bin|out|pyc" + echo "Source artifacts should not contain any files ending in$(echo "|${BINARY_EXTENSIONS}"|sed 's/[^a-z]/ ./g')" + echo "" + function verifyNoBinaries { + file=$1 + echo "" + echo Checking $file... + curl -s $url/$file | tar tvzf - | egrep '\.('"${BINARY_EXTENSIONS}"')$' | tee -a bins + } + verifyNoBinaries apache-geode-${VERSION}-src.tgz + verifyNoBinaries apache-geode-examples-${VERSION}.tar.gz + verifyNoBinaries apache-geode-native-${VERSION}-src.tar.gz + verifyNoBinaries apache-geode-benchmarks-${VERSION}-src.tgz + echo "" + echo "" + if grep -q . bins ; then + echo Binary files were found! + exit 1 + else + echo All good + fi EOF fly -t concourse.apachegeode-ci.info-main login --team-name main --concourse-url https://concourse.apachegeode-ci.info/ fly -t concourse.apachegeode-ci.info-main set-pipeline -p apache-release-${VERSION//./-}-rc -c $PIPEYML diff --git a/dev-tools/release/finalize_release.sh b/dev-tools/release/finalize_release.sh index 564c8ac..2801c99 100755 --- a/dev-tools/release/finalize_release.sh +++ b/dev-tools/release/finalize_release.sh @@ -174,8 +174,8 @@ echo "============================================================" set -x cd $SVN_RELEASE_DIR svn update --set-depth immediates -#identify the latest patch release for the latest 2 major.minor releases, remove anything else from mirrors (all releases remain available on non-mirrored archive site) -RELEASES_TO_KEEP=2 +#identify the latest patch release for "N-2" (the latest 3 major.minor releases), remove anything else from mirrors (all releases remain available on non-mirrored archive site) +RELEASES_TO_KEEP=3 set +x ls | awk -F. '/KEYS/{next}{print 1000000*$1+1000*$2+$3,$1"."$2"."$3}'| sort -n | awk '{mm=$2;sub(/\.[^.]*$/,"",mm);V[mm]=$2}END{for(v in V){print V[v]}}'|tail -$RELEASES_TO_KEEP > ../keep echo Keeping releases: $(cat ../keep)