This is an automated email from the ASF dual-hosted git repository.
ibzib pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/beam.git
The following commit(s) were added to refs/heads/master by this push:
new 4c81134 [BEAM-9541] Create a Gradle task to push all docker images,
and use it instead of bash when building the release candidate.
new f4f4611 Merge pull request #13811 from ibzib/flink-docker-push
4c81134 is described below
commit 4c81134dd9206fd6cb9060a9b973fc111b8fbd64
Author: Kyle Weaver <kcwea...@google.com>
AuthorDate: Tue Jan 26 13:05:26 2021 -0800
[BEAM-9541] Create a Gradle task to push all docker images, and use it
instead of bash when building the release candidate.
Bash arrays make no sense and I've seen the ones in this file cause at
least two bugs already.
Delegating to Gradle also lets as many of the tasks run in parallel as the
OS allows.
Also use LOCAL_CLONE_DIR instead of LOCAL_PYTHON_STAGING_DIR for clarity,
and rm it afterwards.
---
build.gradle | 9 +++++
.../src/main/scripts/build_release_candidate.sh | 38 ++++------------------
sdks/java/container/build.gradle | 5 +++
sdks/python/container/build.gradle | 6 ++++
4 files changed, 26 insertions(+), 32 deletions(-)
diff --git a/build.gradle b/build.gradle
index 12f355d..d1c5857 100644
--- a/build.gradle
+++ b/build.gradle
@@ -307,6 +307,15 @@ task typescriptPreCommit() {
dependsOn ":sdks:python:test-suites:tox:py38:jest"
}
+task pushAllDockerImages() {
+ dependsOn ":runners:spark:job-server:container:dockerPush"
+ dependsOn ":sdks:java:container:pushAll"
+ dependsOn ":sdks:python:container:pushAll"
+ for (version in project.ext.allFlinkVersions) {
+ dependsOn ":runners:flink:${version}:job-server-container:dockerPush"
+ }
+}
+
// Configure the release plugin to do only local work; the release manager
determines what, if
// anything, to push. On failure, the release manager can reset the branch
without pushing.
release {
diff --git a/release/src/main/scripts/build_release_candidate.sh
b/release/src/main/scripts/build_release_candidate.sh
index 1c618ca..8c57261 100755
--- a/release/src/main/scripts/build_release_candidate.sh
+++ b/release/src/main/scripts/build_release_candidate.sh
@@ -57,13 +57,6 @@ PYTHON_ARTIFACTS_DIR=python
BEAM_ROOT_DIR=beam
WEBSITE_ROOT_DIR=beam-site
-DOCKER_IMAGE_DEFAULT_REPO_ROOT=apache
-DOCKER_IMAGE_DEFAULT_REPO_PREFIX=beam_
-
-JAVA_VER=("java8" "java11")
-PYTHON_VER=("python3.6" "python3.7" "python3.8")
-FLINK_VER=("1.8" "1.9" "1.10")
-
echo "================Setting Up Environment Variables==========="
echo "Which release version are you working on: "
read RELEASE
@@ -257,39 +250,20 @@ read confirmation
if [[ $confirmation = "y" ]]; then
echo "============Staging SDK docker images on docker hub========="
cd ~
- if [[ -d ${LOCAL_PYTHON_STAGING_DIR} ]]; then
- rm -rf ${LOCAL_PYTHON_STAGING_DIR}
+ if [[ -d ${LOCAL_CLONE_DIR} ]]; then
+ rm -rf ${LOCAL_CLONE_DIR}
fi
- mkdir -p ${LOCAL_PYTHON_STAGING_DIR}
- cd ${LOCAL_PYTHON_STAGING_DIR}
+ mkdir -p ${LOCAL_CLONE_DIR}
+ cd ${LOCAL_CLONE_DIR}
echo '-------------------Cloning Beam Release Branch-----------------'
git clone ${GIT_REPO_URL}
cd ${BEAM_ROOT_DIR}
git checkout ${RELEASE_BRANCH}
- echo '-------------------Generating and Pushing Python
images-----------------'
- ./gradlew :sdks:python:container:buildAll -Pdocker-pull-licenses
-Pdocker-tag=${RELEASE}_rc${RC_NUM}
- for ver in "${PYTHON_VER[@]}"; do
- docker push
${DOCKER_IMAGE_DEFAULT_REPO_ROOT}/${DOCKER_IMAGE_DEFAULT_REPO_PREFIX}${ver}_sdk:${RELEASE}_rc${RC_NUM}
&
- done
+ ./gradlew :pushAllDockerImages -Pdocker-pull-licenses
-Pdocker-tag=${RELEASE}_rc${RC_NUM}
- echo '-------------------Generating and Pushing Java images-----------------'
- echo "Building containers for the following Java versions:" "${JAVA_VER[@]}"
- for ver in "${JAVA_VER[@]}"; do
- ./gradlew :sdks:java:container:${ver}:dockerPush -Pdocker-pull-licenses
-Pdocker-tag=${RELEASE}_rc${RC_NUM}
- done
-
- echo '-------------Generating and Pushing Flink job server
images-------------'
- echo "Building containers for the following Flink versions:"
"${FLINK_VER[@]}"
- for ver in "${FLINK_VER[@]}"; do
- ./gradlew ":runners:flink:${ver}:job-server-container:dockerPush"
-Pdocker-tag="${RELEASE}_rc${RC_NUM}"
- done
-
- echo '-------------Generating and Pushing Spark job server
image-------------'
- ./gradlew ":runners:spark:job-server:container:dockerPush"
-Pdocker-tag="${RELEASE}_rc${RC_NUM}"
-
- rm -rf ~/${PYTHON_ARTIFACTS_DIR}
+ rm -rf ~/${LOCAL_CLONE_DIR}
fi
echo "[Current Step]: Update beam-site"
diff --git a/sdks/java/container/build.gradle b/sdks/java/container/build.gradle
index c2a9ef1..e0867eb 100644
--- a/sdks/java/container/build.gradle
+++ b/sdks/java/container/build.gradle
@@ -87,3 +87,8 @@ task pullLicenses(type: Exec) {
artifacts {
sdkHarnessLauncher file: file('./build/target'), builtBy: goBuild
}
+
+task pushAll {
+ dependsOn ":sdks:java:container:java8:dockerPush"
+ dependsOn ":sdks:java:container:java11:dockerPush"
+}
diff --git a/sdks/python/container/build.gradle
b/sdks/python/container/build.gradle
index 6093a14..b36ecc32 100644
--- a/sdks/python/container/build.gradle
+++ b/sdks/python/container/build.gradle
@@ -56,6 +56,12 @@ task buildAll {
dependsOn ':sdks:python:container:py38:docker'
}
+task pushAll {
+ dependsOn ':sdks:python:container:py36:dockerPush'
+ dependsOn ':sdks:python:container:py37:dockerPush'
+ dependsOn ':sdks:python:container:py38:dockerPush'
+}
+
artifacts {
sdkHarnessLauncher file: file('./build/target/launcher'), builtBy: goBuild
}
