This is an automated email from the ASF dual-hosted git repository. ibzib pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/beam.git
The following commit(s) were added to refs/heads/master by this push: new 8d75633 [BEAM-13220] Update release instructions. new 5fc55f3 Merge pull request #16065 from ibzib/BEAM-13220 8d75633 is described below commit 8d75633d0c082b1151405cf364c9370bfc95f596 Author: Kyle Weaver <kcwea...@google.com> AuthorDate: Wed Nov 24 15:24:04 2021 -0800 [BEAM-13220] Update release instructions. The biggest change here is to run Java wordcount and mobile gaming examples on Jenkins instead of the bash script, since getting the script to work properly took too long. --- .../main/scripts/publish_github_release_notes.sh | 2 +- release/src/main/scripts/run_rc_validation.sh | 128 +-------------------- release/src/main/scripts/script.config | 2 - .../site/content/en/contribute/release-guide.md | 20 ++-- 4 files changed, 14 insertions(+), 138 deletions(-) diff --git a/release/src/main/scripts/publish_github_release_notes.sh b/release/src/main/scripts/publish_github_release_notes.sh index 7f58648..09fec15 100755 --- a/release/src/main/scripts/publish_github_release_notes.sh +++ b/release/src/main/scripts/publish_github_release_notes.sh @@ -33,7 +33,7 @@ RELEASE_NOTES=$( ) # Escape notes' content to work with JSON -ESCAPED_NOTES=$(printf '%s' "${RELEASE_NOTES}" | python -c 'import json,sys; print(json.dumps(sys.stdin.read()))') +ESCAPED_NOTES=$(printf '%s' "${RELEASE_NOTES}" | python3 -c 'import json,sys; print(json.dumps(sys.stdin.read()))') # Build JSON for the API request REQUEST_JSON="$(cat <<-EOF diff --git a/release/src/main/scripts/run_rc_validation.sh b/release/src/main/scripts/run_rc_validation.sh index baff09b..28ba2eb 100755 --- a/release/src/main/scripts/run_rc_validation.sh +++ b/release/src/main/scripts/run_rc_validation.sh @@ -207,133 +207,6 @@ kubectl version echo "" -echo "" -echo "====================Starting Java Quickstart=======================" -echo "[Current task] Java quickstart with direct runner" -if [[ "$java_quickstart_direct" = true ]]; then - echo "*************************************************************" - echo "* Running Java Quickstart with DirectRunner" - echo "*************************************************************" - ./gradlew :runners:direct-java:runQuickstartJavaDirect \ - -Prepourl=${REPO_URL} \ - -Pver=${RELEASE_VER} -else - echo "* Skip Java quickstart with direct runner" -fi - -echo "[Current task] Java quickstart with Flink local runner" -if [[ "$java_quickstart_flink_local" = true ]]; then - echo "*************************************************************" - echo "* Running Java Quickstart with Flink local runner" - echo "*************************************************************" - ./gradlew :runners:flink:1.13:runQuickstartJavaFlinkLocal \ - -Prepourl=${REPO_URL} \ - -Pver=${RELEASE_VER} -else - echo "* Skip Java quickstart with Flink local runner" -fi - -echo "[Current task] Java quickstart with Spark local runner" -if [[ "$java_quickstart_spark_local" = true ]]; then - echo "*************************************************************" - echo "* Running Java Quickstart with Spark local runner" - echo "*************************************************************" - ./gradlew :runners:spark:2:runQuickstartJavaSpark \ - -Prepourl=${REPO_URL} \ - -Pver=${RELEASE_VER} -else - echo "* Skip Java quickstart with Spark local runner" -fi - -echo "[Current task] Java quickstart with Dataflow runner" -if [[ "$java_quickstart_dataflow" = true && ! -z `which gcloud` ]]; then - echo "*************************************************************" - echo "* Running Java Quickstart with DataflowRunner" - echo "*************************************************************" - ./gradlew :runners:google-cloud-dataflow-java:runQuickstartJavaDataflow \ - -Prepourl=${REPO_URL} \ - -Pver=${RELEASE_VER} \ - -PgcpProject=${USER_GCP_PROJECT} \ - -PgcsBucket=${USER_GCS_BUCKET:5} # skip 'gs://' prefix -else - echo "* Skip Java quickstart with Dataflow runner. Google Cloud SDK is required." -fi - -echo "[Current task] Java quickstart with Twister2 local runner" -if [[ "$java_quickstart_twister2_local" = true ]]; then - echo "*************************************************************" - echo "* Running Java Quickstart with Twister2 local runner" - echo "*************************************************************" - ./gradlew :runners:twister2:runQuickstartJavaTwister2 \ - -Prepourl=${REPO_URL} \ - -Pver=${RELEASE_VER} -else - echo "* Skip Java quickstart with Twister2 local runner" -fi - -echo "" -echo "====================Starting Java Mobile Game=====================" -if [[ ("$java_mobile_game_direct" = true || "$java_mobile_game_dataflow" = true) \ - && ! -z `which gcloud` ]]; then - MOBILE_GAME_DATASET=${USER}_java_validations_$(date +%m%d)_$RANDOM - MOBILE_GAME_PUBSUB_TOPIC=leader_board-${USER}-java-topic-$(date +%m%d)_$RANDOM - echo "Using GCP project: ${USER_GCP_PROJECT}" - echo "Will create BigQuery dataset: ${MOBILE_GAME_DATASET}" - echo "Will create Pubsub topic: ${MOBILE_GAME_PUBSUB_TOPIC}" - - echo "-----------------Creating BigQuery Dataset-----------------" - bq mk --project_id=${USER_GCP_PROJECT} ${MOBILE_GAME_DATASET} - - echo "-----------------Creating Pubsub Topic-----------------" - gcloud pubsub topics create --project=${USER_GCP_PROJECT} ${MOBILE_GAME_PUBSUB_TOPIC} - - if [[ "$java_mobile_game_direct" = true ]]; then - echo "**************************************************************************" - echo "* Java mobile game on DirectRunner: UserScore, HourlyTeamScore, Leaderboard" - echo "**************************************************************************" - ./gradlew :runners:direct-java:runMobileGamingJavaDirect \ - -Prepourl=${REPO_URL} \ - -Pver=${RELEASE_VER} \ - -PgcpProject=${USER_GCP_PROJECT} \ - -PbqDataset=${MOBILE_GAME_DATASET} \ - -PpubsubTopic=${MOBILE_GAME_PUBSUB_TOPIC} \ - -PgcsBucket=${USER_GCS_BUCKET:5} # skip 'gs://' prefix - else - echo "* Skip Java Mobile Game on DirectRunner." - fi - - if [[ "$java_mobile_game_dataflow" = true ]]; then - echo "**************************************************************************" - echo "* Java mobile game on DataflowRunner: UserScore, HourlyTeamScore, Leaderboard" - echo "**************************************************************************" - ./gradlew :runners:google-cloud-dataflow-java:runMobileGamingJavaDataflow \ - -Prepourl=${REPO_URL} \ - -Pver=${RELEASE_VER} \ - -PgcpProject=${USER_GCP_PROJECT} \ - -PbqDataset=${MOBILE_GAME_DATASET} \ - -PpubsubTopic=${MOBILE_GAME_PUBSUB_TOPIC} \ - -PgcsBucket=${USER_GCS_BUCKET:5} # skip 'gs://' prefix - - echo "**************************************************************************" - echo "* Java mobile game on DataflowRunner using Beam GCP BOM: UserScore, HourlyTeamScore, Leaderboard" - echo "**************************************************************************" - ./gradlew :runners:google-cloud-dataflow-java:runMobileGamingJavaDataflowBom \ - -Prepourl=${REPO_URL} \ - -Pver=${RELEASE_VER} \ - -PgcpProject=${USER_GCP_PROJECT} \ - -PbqDataset=${MOBILE_GAME_DATASET} \ - -PpubsubTopic=${MOBILE_GAME_PUBSUB_TOPIC} \ - -PgcsBucket=${USER_GCS_BUCKET:5} # skip 'gs://' prefix - else - echo "* Skip Java Mobile Game on DataflowRunner." - fi - - echo "-----------------Cleaning up BigQuery & Pubsub-----------------" - bq rm -r -f --project_id=${USER_GCP_PROJECT} ${MOBILE_GAME_DATASET} - gcloud pubsub topics delete projects/${USER_GCP_PROJECT}/topics/${MOBILE_GAME_PUBSUB_TOPIC} -fi - -echo "" echo "====================Starting Python Quickstart and MobileGame===================" echo "This task will create a PR against apache/beam, trigger a jenkins job to run:" echo "1. Python quickstart validations(batch & streaming)" @@ -357,6 +230,7 @@ else echo "* Skip Python Quickstart and MobileGame. Hub is required." fi +# TODO(BEAM-13220) Run the remaining tests on Jenkins. echo "" echo "====================Starting Python Leaderboard & GameStates Validations===============" if [[ ("$python_leaderboard_direct" = true \ diff --git a/release/src/main/scripts/script.config b/release/src/main/scripts/script.config index 97e9dd3..52836f9 100755 --- a/release/src/main/scripts/script.config +++ b/release/src/main/scripts/script.config @@ -41,7 +41,6 @@ RC_VALIDATE_CONFIGS=( INSTALL_KUBECTL LOCAL_BEAM_DIR java_quickstart_direct - java_quickstart_apex_local java_quickstart_flink_local java_quickstart_spark_local java_quickstart_dataflow @@ -153,7 +152,6 @@ LOCAL_BEAM_DIR="$(mktemp -d -t beam-release.${RELEASE_VER}.XXXXXX)" # Whether to start certain validation pipeline. ############################################################################## java_quickstart_direct=true -java_quickstart_apex_local=true java_quickstart_flink_local=true java_quickstart_spark_local=true java_quickstart_dataflow=true diff --git a/website/www/site/content/en/contribute/release-guide.md b/website/www/site/content/en/contribute/release-guide.md index 748af95..8a8b607 100644 --- a/website/www/site/content/en/contribute/release-guide.md +++ b/website/www/site/content/en/contribute/release-guide.md @@ -135,11 +135,11 @@ __NOTE__: When generating the key, please make sure you choose the key type as _ This will list your GPG keys. One of these should reflect your Apache account, for example: -------------------------------------------------- - pub 2048R/845E6689 2016-02-23 + pub rsa4096/845E6689845E6689 2016-02-23 uid Nomen Nescio <anonym...@apache.org> - sub 2048R/BA4D50BE 2016-02-23 + sub rsa4096/BA4D50BEBA4D50BE 2016-02-23 - Here, the key ID is the 8-digit hex string in the `pub` line: `845E6689`. + Here, the key ID is the 16-digit hex string in the `pub` line: `845E6689845E6689`. #### Access to Apache Nexus repository @@ -548,7 +548,7 @@ The final state of the repository should match this diagram: ./beam/release/src/main/scripts/choose_rc_commit.sh \ --release "${RELEASE_VERSION}" \ --rc "${RC_NUM}" \ - --commit "${COMMIT_REF}" \ + --commit "${COMMIT_REF}" \ --clone \ --push-tag @@ -569,7 +569,7 @@ Before you start, run this command to make sure you'll be using the latest docke * **Usage** - ./beam/release/src/main/scripts/build_release_candidate.sh + ./beam/release/src/main/scripts/build_release_candidate.sh --release "${RELEASE_VERSION}" --rc "${RC_NUM}" --github-user "${GITHUB_USER}" * **The script will:** 1. Clone the repo at the selected RC tag. @@ -839,8 +839,6 @@ versions to run all of the tests. See Python installation tips in [Developer Wik ``` * **Tasks included** - 1. Run Java quickstart with Direct Runner, Flink local runner, Spark local runner and Dataflow runner. - 1. Run Java Mobile Games(UserScore, HourlyTeamScore, Leaderboard) with Dataflow runner. 1. Create a PR to trigger python validation job, including * Python quickstart in batch and streaming mode with direct runner and Dataflow runner. * Python Mobile Games(UserScore, HourlyTeamScore) with direct runner and Dataflow runner. @@ -854,6 +852,12 @@ versions to run all of the tests. See Python installation tips in [Developer Wik * **Tasks you need to do manually** 1. Check whether validations succeed by following console output instructions. 1. Terminate streaming jobs and java injector. + 1. Run Java quickstart (wordcount) and mobile game examples with the staged artifacts. The easiest way to do this is by running the tests on Jenkins. + 1. Log in to Jenkins. + 1. Go to https://ci-beam.apache.org/job/beam_PostRelease_NightlySnapshot/. + 1. Click "Build with Parameters". + 1. Set `snapshot_version` to `2.xx.0`, and set `snapshot_url` to point to the staged artifacts in Maven central (https://repository.apache.org/content/repositories/orgapachebeam-NNNN/). + 1. Click "Build". 1. Sign up [spreadsheet](https://s.apache.org/beam-release-validation). 1. Vote in the release thread. @@ -1184,7 +1188,7 @@ After pushing the tag, the tag should be visible on Github's [Tags](https://gith Once the tag is uploaded, publish the release notes to Github, as follows: ``` -cd beam/release/src/main/scripts && ./publish_github_release_notes.sh +./beam/release/src/main/scripts/publish_github_release_notes.sh ``` Note this script reads the release notes from the blog post, so you should make sure to run this from master _after_ merging the blog post PR.