This is an automated email from the ASF dual-hosted git repository. altay pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/beam.git
The following commit(s) were added to refs/heads/master by this push: new b944f66 Automate "Run validation tests" step (#6081) b944f66 is described below commit b944f6631de05625b21226af4e448378826aaf89 Author: Boyuan Zhang <36090911+boyua...@users.noreply.github.com> AuthorDate: Fri Aug 3 15:26:15 2018 -0700 Automate "Run validation tests" step (#6081) * Automate java quickstart validation tests * Automate java mobile game validations * Automate "Create a PR to trigger python validations" --- release/src/main/scripts/run_rc_validation.sh | 541 ++++++++++++++++++++++++++ 1 file changed, 541 insertions(+) diff --git a/release/src/main/scripts/run_rc_validation.sh b/release/src/main/scripts/run_rc_validation.sh new file mode 100755 index 0000000..0bd3e23 --- /dev/null +++ b/release/src/main/scripts/run_rc_validation.sh @@ -0,0 +1,541 @@ +#!/bin/bash +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +function clean_up(){ + echo "======================Stopping Pubsub Java Injector=========================" + echo "Please stop java injector manually." + echo "===========================Signing up Spreadsheet===========================" + echo "Please open this spreadsheet: https://s.apache.org/beam-release-validation" + echo "Please sign up your name in the tests you have ran." + + echo "===========================Final Cleanup===========================" + echo "* Restoring ~/.m2/settings.xml" + if [[ ! -z `ls -a ~/.m2/settings_backup.xml` ]]; then + rm ~/.m2/settings.xml + cp ~/.m2/settings_backup.xml ~/.m2/settings.xml + fi + echo "* Restoring ~/.bashrc" + if [[ ! -z `ls -a ~/.bashrc_backup` ]]; then + rm ~/.bashrc + cp ~/.bashrc_backup ~/.bashrc + fi + + rm -rf ~/{LOCAL_CLONE_DIR} +} + +RELEASE= +REPO_URL= +RC_NUM= +RELEASE_BRANCH= +WORKING_BRANCH= +LOCAL_CLONE_DIR=rc_validations +BEAM_ROOT_DIR=beam +GIT_REPO_URL=https://github.com/apache/beam.git +PYTHON_RC_DOWNLOAD_URL=https://dist.apache.org/repos/dist/dev/beam + +echo "[Input Required] Please enter the release version: " +read RELEASE +RELEASE_BRANCH=release-${RELEASE} +WORKING_BRANCH=release-${RELEASE}-RC${RC_NUM}_validations +echo "[Input Required] Please enter the release candidate number(e.g. 1): " +read RC_NUM +echo "[Input Required] Please copy the repo URL from the vote email sent out by Release Manager:" +echo "The URL should look like: https://repository.apache.org/content/repositories/orgapachebeam-0000" +read REPO_URL + +echo "====================Checking Environment Variables=================" +echo "running validations on release ${RELEASE} RC${RC_NUM}." +echo "repo URL for this RC: ${REPO_URL}" +echo "[Confirmation Required] Do you confirm all information above are correct? [y|N]" +read confirmation +if [[ $confirmation != "y" ]]; then + echo "Please rerun this script and make sure you have the right inputs." + exit +fi + +echo "====================Cloning Beam Release Branch====================" +cd ~ +mkdir ${LOCAL_CLONE_DIR} +cd ${LOCAL_CLONE_DIR} +git clone ${GIT_REPO_URL} +cd ${BEAM_ROOT_DIR} +git checkout ${RELEASE_BRANCH} +git checkout -b ${WORKING_BRANCH} + +echo "====================Starting Java Quickstart=======================" +echo "[Current task] Java quickstart with direct runner" +echo "[Confirmation Required] Do you want to start this task? [y|N]" +read confirmation +if [[ $confirmation = "y" ]]; then + echo "*************************************************************" + echo "* Running Java Quickstart with DirectRunner" + echo "*************************************************************" + ./gradlew :beam-runners-direct-java:runQuickstartJavaDirect \ + -Prepourl=${REPO_URL} \ + -Pver=${RELEASE} +fi + +echo "[Current task] Java quickstart with Apex local runner" +echo "[Confirmation Required] Do you want to start this task? [y|N]" +read confirmation +if [[ $confirmation = "y" ]]; then + echo "*************************************************************" + echo "* Running Java Quickstart with Apex local runner" + echo "*************************************************************" + ./gradlew :beam-runners-apex:runQuickstartJavaApex \ + -Prepourl=${REPO_URL} \ + -Pver=${RELEASE} +fi + +echo "[Current task] Java quickstart with Flink local runner" +echo "[Confirmation Required] Do you want to start this task? [y|N]" +read confirmation +if [[ $confirmation = "y" ]]; then + echo "*************************************************************" + echo "* Running Java Quickstart with Flink local runner" + echo "*************************************************************" + ./gradlew :beam-runners-flink_2.11:runQuickstartJavaFlinkLocal \ + -Prepourl=${REPO_URL} \ + -Pver=${RELEASE} +fi + +echo "[Current task] Java quickstart with Spark local runner" +echo "[Confirmation Required] Do you want to start this task? [y|N]" +read confirmation +if [[ $confirmation = "y" ]]; then + echo "*************************************************************" + echo "* Running Java Quickstart with Spark local runner" + echo "*************************************************************" + ./gradlew :beam-runners-spark:runQuickstartJavaSpark \ + -Prepourl=${REPO_URL} \ + -Pver=${RELEASE} +fi + +echo "====================Checking Google Cloud SDK======================" +if [[ -z `which gcloud` ]]; then + echo "You don't have Google Cloud SDK installed." + echo " Do you want to install gcloud with sudo permission? [y|N]" + read confirmation + if [[ $confirmation != 'y' ]]; then + echo "Exit script without running rest validations." + exit + fi + sudo apt-get install google-cloud-sdk +fi +gcloud --version + +echo "[Current task] Java quickstart with Dataflow runner" +echo "[Confirmation Required] Do you want to start this task? [y|N]" +read confirmation +if [[ $confirmation = "y" ]]; then + echo "[GCP Project Required] Please input your GCP project:" + read USER_GCP_PROJECT + echo "[GCP GCS Bucket Required] Please input your GCS bucket: " + read USER_GCS_BUCKET + echo "[gcloud Login Required] Please login into your gcp account: " + gcloud auth application-default login + GOOGLE_APPLICATION_CREDENTIALS=~/.config/gcloud/application_default_credentials.json + + echo "*************************************************************" + echo "* Running Java Quickstart with DataflowRunner" + echo "*************************************************************" + gcloud auth application-default login + gcloud config set project ${USER_GCP_PROJECT} + ./gradlew :beam-runners-google-cloud-dataflow-java:runQuickstartJavaDataflow \ + -Prepourl=${REPO_URL} \ + -Pver=${RELEASE} \ + -PgcpProject=${USER_GCP_PROJECT} \ + -PgcsBucket=${USER_GCS_BUCKET} +fi + +echo "===================Starting Java Mobile Game=====================" +echo "[Confirmation Required] This task asks for GCP resources." +echo "Do you want to proceed? [y|N]" +read confirmation +if [[ $confirmation = "y" ]]; then + echo "[GCP Project Required] Please input your GCP project:" + read USER_GCP_PROJECT + echo "[GCP GCS Bucket Required] Please input your GCS bucket: " + read USER_GCS_BUCKET + MOBILE_GAME_DATASET=${USER}_java_validations + MOBILE_GAME_PUBSUB_TOPIC=leader_board-${USER}-java-topic-1 + echo "Please review following GCP sources setup: " + echo "Using GCP project: ${USER_GCP_PROJECT}" + echo "Will create BigQuery dataset: ${MOBILE_GAME_DATASET}" + echo "Will create Pubsub topic: ${MOBILE_GMAE_PUBSUB_TOPIC}" + echo "[Confirmation Required] Do you want to run validations with configurations above? [y|N]" + read confirmation + if [[ $confirmation = "y" ]]; then + gcloud auth login + gcloud config set project ${USER_GCP_PROJECT} + echo "-----------------Setting Up Service Account------------------------" + echo "Please go to GCP IAM console under your project(${USER_GCP_PROJECT})." + echo "Create a service account as project owner, if you don't have one." + echo "[Input Required] Please enter your service account email:" + read USER_SERVICE_ACCOUNT_EMAIL + SERVICE_ACCOUNT_KEY_JSON=${USER}_json_key.json + gcloud iam service-accounts keys create ${SERVICE_ACCOUNT_KEY_JSON} --iam-account ${USER_SERVICE_ACCOUNT_EMAIL} + export GOOGLE_APPLICATION_CREDENTIALS=$(pwd)/${SERVICE_ACCOUNT_KEY_JSON} + + echo "-------------------Creating BigQuery Dataset-----------------------" + bq rm -rf --project=${USER_GCP_PROJECT} ${MOBILE_GAME_DATASET} + bq mk --project=${USER_GCP_PROJECT} ${MOBILE_GAME_DATASET} + + echo "----------------------Creating Pubsub Topic------------------------" + gcloud alpha pubsub topics delete projects/${USER_GCP_PROJECT}/topics/${MOBILE_GAME_PUBSUB_TOPIC} + gcloud alpha pubsub topics create --project=${USER_GCP_PROJECT} ${MOBILE_GAME_PUBSUB_TOPIC} + + echo "**************************************************************************" + echo "* Java mobile game validations: UserScore, HourlyTeamScore, Leaderboard" + echo "**************************************************************************" + ./gradlew :beam-runners-google-cloud-dataflow-java:runMobileGamingJavaDataflow \ + -Prepourl=${REPO_URL} \ + -Pver=${RELEASE} \ + -PgcpProject=${USER_GCP_PROJECT} \ + -PgcsBucket=${USER_GCS_BUCKET} \ + -PbqDataset=${MOBILE_GAME_DATASET} -PpubsubTopic=${MOBILE_GAME_PUBSUB_TOPIC} + fi +fi + +echo "==================Starting Python Quickstart and MobileGame===================" +echo "This task will create a PR against apache/beam, trigger a jenkins job to run:" +echo "1. Python quickstart validations(batch & streaming)" +echo "2. Python MobileGame validations(UserScore, HourlyTeamScore)" +echo "[Confirmation Required] Do you want to proceed? [y|N]" +read confirmation +if [[ $confirmation = "y" ]]; then + echo "[Input Required] Please enter your github repo URL forked from apache/beam:" + read USER_REMOTE_URL + WORKING_BRANCH=python_validatoin_pr + git checkout -b ${WORKING_BRANCH} + touch empty_file.txt + git add empty_file.txt + git commit -m "Add empty file in order to create PR" + git push -f ${USER_REMOTE_URL} + hub pull-request -b apache:${RELEASE_BRANCH} -h boyuanzz:${WORKING_BRANCH} -F- <<<"[DO NOT MERGE]Run Python RC Validation Tests + + + Run Python ReleaseCandidate" + + echo "[NOTE] If there is no jenkins job started, please comment generated PR with: Run Python ReleaseCandidate" +fi + +echo "==============Starting Python Leaderboard & GameStates Validations===============" +echo "This task asks for GCP resources. Do you want to proceed? [y|N]" +read confirmation +if [[ $confirmation = "y" ]]; then + cd ~/${LOCAL_CLONE_DIR} + + echo "---------------------Checking gnome-terminal----------------------------------" + if [[ -z `which gnome-terminal` ]]; then + echo "You don't have gnome-terminal installed." + echo "Do you want to install gnome-terminal with sudo permission? [y|N]" + read confirmation + if [[ $confirmation != 'y' ]]; then + echo "Exit this script without proceeding to the next step." + exit + fi + fi + gnome-terminal --version + + echo "---------------------Downloading Python Staging RC----------------------------" + wget ${PYTHON_RC_DOWNLOAD_URL}/${RELEASE}/python/apache-beam-${RELEASE}.zip + wget ${PYTHON_RC_DOWNLOAD_URL}/${RELEASE}/python/apache-beam-${RELEASE}.zip.sha512 + + echo "--------------------------Verifying Hashes------------------------------------" + sha512sum -c apache-beam-${RELEASE}.zip.sha512 + + echo "---------------------------Setting up virtualenv------------------------------" + sudo pip install --upgrade pip + sudo pip install --upgrade setuptools + sudo pip install --upgrade virtualenv + virtualenv beam_env + . beam_env/bin/activate + + echo "--------------------------Installing Python SDK-------------------------------" + pip install apache-beam-${RELEASE}.zip + pip install apache-beam-${RELEASE}.zip[gcp] + + echo "----------------------------Setting up GCP Sources----------------------------" + echo "[GCP Project Required] Please input your GCP project:" + read USER_GCP_PROJECT + gcloud auth login + gcloud config set project ${USER_GCP_PROJECT} + + MOBILE_GAME_GCS_BUCKET=gs://${USER}_python_validations_bucket + MOBILE_GAME_DATASET=${USER}_python_validations + MOBILE_GAME_PUBSUB_TOPIC=leader_board-${USER}-python-topic-1 + gsutil mb -p ${USER_GCP_PROJECT} ${MOBILE_GAME_GCS_BUCKET} + gcloud alpha pubsub topics delete projects/${USER_GCP_PROJECT}/topics/${MOBILE_GAME_PUBSUB_TOPIC} + gcloud alpha pubsub topics create --project=${USER_GCP_PROJECT} ${MOBILE_GAME_PUBSUB_TOPIC} + + echo "-----------------------Setting Up Service Account-----------------------------" + echo "Please go to GCP IAM console under your project(${USER_GCP_PROJECT})." + echo "Create a service account as project owner, if you don't have one." + echo "[Input Required] Please enter your service account email:" + read USER_SERVICE_ACCOUNT_EMAIL + SERVICE_ACCOUNT_KEY_JSON= ${USER}_json_key.json + gcloud iam service-accounts keys create ${SERVICE_ACCOUNT_KEY_JSON} --iam-account ${USER_SERVICE_ACCOUNT_EMAIL} + export GOOGLE_APPLICATION_CREDENTIALS=$(pwd)/${SERVICE_ACCOUNT_KEY_JSON} + + echo "-----------------------Setting up Shell Env Vars------------------------------" + cp ~/.bashrc ~/.bashrc_backup + echo "export USER_GCP_PROJECT=${USER_GCP_PROJECT}" >> ~/.bashrc + echo "export MOBILE_GAME_DATASET=${MOBILE_GAME_DATASET}" >> ~/.bashrc + echo "export MOBILE_GAME_PUBSUB_TOPIC=${MOBILE_GAME_PUBSUB_TOPIC}" >> ~/.bashrc + echo "export MOBILE_GAME_GCS_BUCKET=${MOBILE_GAME_GCS_BUCKET}" >> ~/.bashrc + echo "export GOOGLE_APPLICATION_CREDENTIALS=${GOOGLE_APPLICATION_CREDENTIALS}" >> ~/.bashrc + echo "export RELEASE=${RELEASE}" >> ~/.bashrc + + echo "--------------------------Updating ~/.m2/settings.xml-------------------------" + cd ~ + if [[ -z `ls -a ~ | grep ".m2"` ]]; then + mkdir .m2 + fi + cd .m2 + if [[ ! -z `ls -a ~/.m2/ | grep "settings.xml"` ]]; then + mv settings.xml settings_backup.xml + fi + touch settings.xml + echo "<settings>" >> settings.xml + echo " <profiles>" >> settings.xml + echo " <profile>" >> settings.xml + echo " <id>release-repo</id>" >> settings.xml + echo " <activation>" >> settings.xml + echo " <activeByDefault>true</activeByDefault>" >> settings.xml + echo " </activation>" >> settings.xml + echo " <repositories>" >> settings.xml + echo " <repository>" >> settings.xml + echo " <id>Release ${RELEASE} RC${RC_NUM}</id>" >> settings.xml + echo " <name>Release ${RELEASE} RC${RC_NUM}</name>" >> settings.xml + echo " <url>${REPO_URL}</url>" >> settings.xml + echo " </repository>" >> settings.xml + echo " </repositories>" >> settings.xml + echo " </profile>" >> settings.xml + echo " </profiles>" >> settings.xml + echo "</settings>" >> settings.xml + + echo "----------------------Starting Pubsub Java Injector--------------------------" + cd ~/${LOCAL_CLONE_DIR} + mvn archetype:generate \ + -DarchetypeGroupId=org.apache.beam \ + -DarchetypeArtifactId=beam-sdks-java-maven-archetypes-examples \ + -DarchetypeVersion=${RELEASE} \ + -DgroupId=org.example \ + -DartifactId=word-count-beam \ + -Dversion="0.1" \ + -Dpackage=org.apache.beam.examples \ + -DinteractiveMode=false \ + -DarchetypeCatalog=internal + + cd word-count-beam + echo "A new terminal will pop up and start a java top injector." + gnome-terminal -x sh -c \ + "echo '******************************************************'; + echo '* Running Pubsub Java Injector'; + echo '******************************************************'; + mvn compile exec:java -Dexec.mainClass=org.apache.beam.examples.complete.game.injector.Injector \ + -Dexec.args='${USER_GCP_PROJECT} ${MOBILE_GAME_PUBSUB_TOPIC} none'; + exec bash" + + echo "[Confirmation Required] Please enter y to confirm injector running:" + read confirmation + if [[ $confirmation != "y" ]]; then + echo "Following tests only can be ran when java injector running." + clean_up + exit + fi + + cd ~/${LOCAL_CLONE_DIR}/ + + echo "----------------Starting Leaderboard with DirectRunner-----------------------" + echo "[Confirmation Required] Do you want to proceed? [y|N]" + read confirmation + if [[ $confirmation = "y" ]]; then + bq rm -rf --project=${USER_GCP_PROJECT} ${MOBILE_GAME_DATASET} + bq mk --project=${USER_GCP_PROJECT} ${MOBILE_GAME_DATASET} + echo "This is a streaming job. This task will be launched in a separate terminal." + gnome-terminal -x sh -c \ + "echo '*****************************************************'; + echo '* Running Python Leaderboard with DirectRunner'; + echo '*****************************************************'; + python -m apache_beam.examples.complete.game.leader_board \ + --project=${USER_GCP_PROJECT} \ + --topic projects/${USER_GCP_PROJECT}/topics/${MOBILE_GAME_PUBSUB_TOPIC} \ + --dataset ${MOBILE_GAME_DATASET}; + exec bash" + + echo "***************************************************************" + echo "* Please wait for at least 5 mins to let results get populated." + echo "* Sleeping for 5 mins" + sleep 5m + echo "***************************************************************" + echo "* How to verify results:" + echo "* 1. Check whether there is any error messages in the task running terminal." + echo "* 2. Goto your BigQuery console and check whether your ${MOBILE_GAME_DATASET} has leader_board_users and leader_board_teams table." + echo "* 3. Check whether leader_board_users has data, retrieving BigQuery data as below: " + bq head -n 10 ${MOBILE_GAME_DATASET}.leader_board_users + echo "* 4. Check whether leader_board_teams has data, retrieving BigQuery data as below:" + bq head -n 10 ${MOBILE_GAME_DATASET}.leader_board_teams + echo "***************************************************************" + + echo "If you have verified all items listed above, please terminate the python job." + echo "[Confirmation Required] Please confirm whether you have stopped this job: [y|N]" + read confirmation + if [[ $confirmation != "y" ]]; then + echo "Current job must be terminated in order to proceed into next test." + clean_up + exit + fi + fi + + echo "----------------Starting Leaderboard with DataflowRunner---------------------" + echo "[Confirmation Required] Do you want to proceed? [y|N]" + read confirmation + if [[ $confirmation = "y" ]]; then + bq rm -rf --project=${USER_GCP_PROJECT} ${MOBILE_GAME_DATASET} + bq mk --project=${USER_GCP_PROJECT} ${MOBILE_GAME_DATASET} + echo "This is a streaming job. This task will be launched in a separate terminal." + gnome-terminal -x sh -c \ + "echo '*****************************************************'; + echo '* Running Python Leaderboard with DataflowRunner'; + echo '*****************************************************'; + python -m apache_beam.examples.complete.game.leader_board \ + --project=${USER_GCP_PROJECT} \ + --topic projects/${USER_GCP_PROJECT}/topics/${MOBILE_GAME_PUBSUB_TOPIC} \ + --dataset ${MOBILE_GAME_DATASET} \ + --runner DataflowRunner \ + --temp_location=${MOBILE_GAME_GCS_BUCKET}/temp/ \ + --sdk_location apache-beam-${RELEASE}.zip; \ + exec bash" + + echo "***************************************************************" + echo "* Please wait for at least 10 mins to let Dataflow job be launched and results get populated." + echo "* Sleeping for 10 mins" + sleep 10m + echo "* How to verify results:" + echo "* 1. Goto your Dataflow job console and check whether there is any error." + echo "* 2. Goto your BigQuery console and check whether your ${MOBILE_GAME_DATASET} has leader_board_users and leader_board_teams table." + echo "* 3. Check whether leader_board_users has data, retrieving BigQuery data as below: " + bq head -n 10 ${MOBILE_GAME_DATASET}.leader_board_users + echo "* 4. Check whether leader_board_teams has data, retrieving BigQuery data as below:" + bq head -n 10 ${MOBILE_GAME_DATASET}.leader_board_teams + echo "***************************************************************" + + echo "If you have verified all items listed above, please terminate this job in Dataflow Console." + echo "[Confirmation Required] Please confirm whether you have stopped this job: [y|N]" + read confirmation + if [[ $confirmation != "y" ]]; then + echo "Current job must be terminated in order to proceed into next test." + clean_up + exit + fi + fi + + # [BEAM-4518] + FIXED_WINDOW_DURATION=20 + + echo "------------------Starting GameStats with DirectRunner-----------------------" + echo "[Confirmation Required] Do you want to proceed? [y|N]" + read confirmation + if [[ $confirmation = "y" ]]; then + bq rm -rf --project=${USER_GCP_PROJECT} ${MOBILE_GAME_DATASET} + bq mk --project=${USER_GCP_PROJECT} ${MOBILE_GAME_DATASET} + + echo "This is a streaming job. This task will be launched in a separate terminal." + echo "Streaming job is running with fixed_window_duration=${FIXED_WINDOW_DURATION}" + gnome-terminal -x sh -c \ + "echo '*****************************************************'; + echo '* Running GameStats with DirectRunner'; + echo '*****************************************************'; + python -m apache_beam.examples.complete.game.game_stats \ + --project=${USER_GCP_PROJECT} \ + --topic projects/${USER_GCP_PROJECT}/topics/${MOBILE_GAME_PUBSUB_TOPIC} \ + --dataset ${MOBILE_GAME_DATASET} \ + --fixed_window_duration ${FIXED_WINDOW_DURATION}; \ + exec bash" + + echo "***************************************************************" + echo "* Please wait for at least 25 mins to let results get populated." + echo "* Sleeping for 25mins" + sleep 25m + echo "* How to verify results:" + echo "* 1. Check whether there is any error messages in the task running terminal." + echo "* 2. Goto your BigQuery console and check whether your ${MOBILE_GAME_DATASET} has game_stats_teams and game_stats_sessions table." + echo "* 3. Check whether game_stats_teams has data, retrieving BigQuery data as below: " + bq head -n 10 ${MOBILE_GAME_DATASET}.game_stats_teams + echo "* 4. Check whether game_stats_sessions has data, retrieving BigQuery data as below:" + bq head -n 10 ${MOBILE_GAME_DATASET}.game_stats_sessions + echo "***************************************************************" + + echo "If you have verified all items listed above, please terminate the python job." + echo "[Confirmation Required] Please confirm whether you have stopped this job: [y|N]" + read confirmation + if [[ $confirmation != "y" ]]; then + echo "Current job must be terminated in order to proceed into next test." + clean_up + exit + fi + fi + + echo "-------------------Starting GameStats with DataflowRunner--------------------" + echo "[Confirmation Required] Do you want to proceed? [y|N]" + read confirmation + if [[ $confirmation = "y" ]]; then + bq rm -rf --project=${USER_GCP_PROJECT} ${MOBILE_GAME_DATASET} + bq mk --project=${USER_GCP_PROJECT} ${MOBILE_GAME_DATASET} + echo "This is a streaming job. This task will be launched in a separate terminal." + echo "Streaming job is running with fixed_window_duration=${FIXED_WINDOW_DURATION}" + gnome-terminal -x sh -c \ + "echo '*****************************************************'; + echo '* Running GameStats with DataflowRunner'; + echo '*****************************************************'; + python -m apache_beam.examples.complete.game.game_stats \ + --project=${USER_GCP_PROJECT} \ + --topic projects/${USER_GCP_PROJECT}/topics/${MOBILE_GAME_PUBSUB_TOPIC} \ + --dataset ${MOBILE_GAME_DATASET} \ + --runner DataflowRunner \ + --temp_location=${MOBILE_GAME_GCS_BUCKET}/temp/ \ + --sdk_location apache-beam-${RELEASE}.zip \ + --fixed_window_duration ${FIXED_WINDOW_DURATION}; exec bash" + + echo "***************************************************************" + echo "* Please wait for at least 30 mins to let results get populated." + echo "* Sleeping for 30 mins" + sleep 30m + echo "* How to verify results:" + echo "* 1. Goto your Dataflow job console and check whether there is any error." + echo "* 2. Goto your BigQuery console and check whether your ${MOBILE_GAME_DATASET} has game_stats_teams and game_stats_sessions table." + echo "* 3. Check whether game_stats_teams has data, retrieving BigQuery data as below: " + bq head -n 10 ${MOBILE_GAME_DATASET}.game_stats_teams + echo "* 4. Check whether game_stats_sessions has data, retrieving BigQuery data as below:" + bq head -n 10 ${MOBILE_GAME_DATASET}.game_stats_sessions + echo "***************************************************************" + + echo "If you have verified all items listed above, please terminate the python job." + echo "[Confirmation Required] Please confirm whether you have stopped this job: [y|N]" + read confirmation + if [[ $confirmation != "y" ]]; then + echo "Current job must be terminated in order to proceed into next test." + clean_up + exit + fi + fi +fi + +clean_up