This is an automated email from the ASF dual-hosted git repository.
altay pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/beam.git
The following commit(s) were added to refs/heads/master by this push:
new d853b69 [BEAM-6596] Use multiple interpreter versions for SDK release
validation. (#7914)
d853b69 is described below
commit d853b696642e6eb237dfb4eda07358cf504dbb04
Author: tvalentyn <[email protected]>
AuthorDate: Thu Feb 28 13:12:46 2019 -0800
[BEAM-6596] Use multiple interpreter versions for SDK release validation.
(#7914)
* Use multiple interpreter versions for SDK release validation.
* Provide an option to the select the interpreter version to use.
* Support mutliple interpreter version in python-release verification
script suite.
---
.../python-release/python_release_automation.sh | 13 +-
.../python_release_automation_utils.sh | 47 +-
.../run_release_candidate_python_mobile_gaming.sh | 25 +-
.../run_release_candidate_python_quickstart.sh | 28 +-
release/src/main/scripts/run_rc_validation.sh | 515 +++++++++++----------
5 files changed, 325 insertions(+), 303 deletions(-)
diff --git a/release/src/main/python-release/python_release_automation.sh
b/release/src/main/python-release/python_release_automation.sh
old mode 100644
new mode 100755
index bc81cf5..8b305ae
--- a/release/src/main/python-release/python_release_automation.sh
+++ b/release/src/main/python-release/python_release_automation.sh
@@ -19,7 +19,12 @@
source
release/src/main/python-release/run_release_candidate_python_quickstart.sh
source
release/src/main/python-release/run_release_candidate_python_mobile_gaming.sh
-run_release_candidate_python_quickstart "tar"
-run_release_candidate_python_mobile_gaming "tar"
-run_release_candidate_python_quickstart "wheel"
-run_release_candidate_python_mobile_gaming "wheel"
+run_release_candidate_python_quickstart "tar" "python2.7"
+run_release_candidate_python_mobile_gaming "tar" "python2.7"
+run_release_candidate_python_quickstart "wheel" "python2.7"
+run_release_candidate_python_mobile_gaming "wheel" "python2.7"
+
+run_release_candidate_python_quickstart "tar" "python3.5"
+run_release_candidate_python_mobile_gaming "tar" "python3.5"
+run_release_candidate_python_quickstart "wheel" "python3.5"
+run_release_candidate_python_mobile_gaming "wheel" "python3.5"
diff --git a/release/src/main/python-release/python_release_automation_utils.sh
b/release/src/main/python-release/python_release_automation_utils.sh
index f45428c..07653e6 100644
--- a/release/src/main/python-release/python_release_automation_utils.sh
+++ b/release/src/main/python-release/python_release_automation_utils.sh
@@ -78,11 +78,32 @@ function get_version() {
# BEAM_PYTHON_SDK*
# Arguments:
# $1 - SDK type: tar, wheel
+# $2 - python interpreter version: python2.7, python3.5, ...
#######################################
function download_files() {
+ VERSION=$(get_version)
+
if [[ $1 = *"wheel"* ]]; then
+ if [[ $2 == "python2.7" ]]; then
+
BEAM_PYTHON_SDK_WHL="apache_beam-$VERSION*-cp27-cp27mu-manylinux1_x86_64.whl"
+ elif [[ $2 == "python3.5" ]]; then
+
BEAM_PYTHON_SDK_WHL="apache_beam-$VERSION*-cp35-cp35m-manylinux1_x86_64.whl"
+ elif [[ $2 == "python3.6" ]]; then
+
BEAM_PYTHON_SDK_WHL="apache_beam-$VERSION*-cp36-cp36m-manylinux1_x86_64.whl"
+ elif [[ $2 == "python3.7" ]]; then
+
BEAM_PYTHON_SDK_WHL="apache_beam-$VERSION*-cp37-cp37m-manylinux1_x86_64.whl"
+ elif [[ $2 == "python3.8" ]]; then
+
BEAM_PYTHON_SDK_WHL="apache_beam-$VERSION*-cp38-cp38m-manylinux1_x86_64.whl"
+ elif [[ $2 == "python3.9" ]]; then
+
BEAM_PYTHON_SDK_WHL="apache_beam-$VERSION*-cp39-cp39m-manylinux1_x86_64.whl"
+ else
+ echo "Unable to determine a Beam wheel for interpreter version $2."
+ exit 1
+ fi
+
wget -r -l2 --no-parent -nd -A "$BEAM_PYTHON_SDK_WHL*" $RC_STAGING_URL
else
+ BEAM_PYTHON_SDK_ZIP="apache-beam-$VERSION.zip"
wget -r -l2 --no-parent -nd -A "$BEAM_PYTHON_SDK_ZIP*" $RC_STAGING_URL
fi
}
@@ -133,6 +154,30 @@ function get_asc_name() {
#######################################
+# Create a new virtualenv and install the SDK
+# Globals:
+# BEAM_PYTHON_SDK
+# Arguments:
+# $1 - SDK type: tar, wheel
+# $2 - python interpreter version: [python2.7, python3.5, ...]
+#######################################
+function install_sdk() {
+ sdk_file=$(get_sdk_name $1)
+ print_separator "Creating new virtualenv with $2 interpreter and installing
the SDK from $sdk_file."
+ gsutil version -l
+ rm -rf ./temp_virtualenv_${2}
+ virtualenv temp_virtualenv_${2} -p $2
+ . temp_virtualenv_${2}/bin/activate
+ gcloud_version=$(gcloud --version | head -1 | awk '{print $4}')
+ if [[ "$gcloud_version" < "189" ]]; then
+ update_gcloud
+ fi
+ pip install google-compute-engine
+ pip install $sdk_file[gcp]
+}
+
+
+#######################################
# Publish data to Pubsub topic for streaming wordcount examples.
# Arguments:
# None
@@ -277,8 +322,6 @@ function verify_hourly_team_score() {
# Python RC configurations
VERSION=$(get_version)
RC_STAGING_URL="https://dist.apache.org/repos/dist/dev/beam/$VERSION/";
-BEAM_PYTHON_SDK_ZIP="apache-beam-$VERSION.zip"
-BEAM_PYTHON_SDK_WHL="apache_beam-$VERSION*-cp27-cp27mu-manylinux1_x86_64.whl"
# Cloud Configurations
PROJECT_ID='apache-beam-testing'
diff --git
a/release/src/main/python-release/run_release_candidate_python_mobile_gaming.sh
b/release/src/main/python-release/run_release_candidate_python_mobile_gaming.sh
index 2bfe974..a9c9272 100755
---
a/release/src/main/python-release/run_release_candidate_python_mobile_gaming.sh
+++
b/release/src/main/python-release/run_release_candidate_python_mobile_gaming.sh
@@ -48,26 +48,6 @@ function complete() {
#######################################
-# Download files from RC staging location, install python sdk
-# Globals:
-# BEAM_PYTHON_SDK
-# Arguments:
-# None
-#######################################
-function install_sdk() {
- print_separator "Creating new virtualenv and installing the SDK"
- virtualenv temp_virtualenv
- . temp_virtualenv/bin/activate
- gcloud_version=$(gcloud --version | head -1 | awk '{print $4}')
- if [[ "$gcloud_version" < "189" ]]; then
- update_gcloud
- fi
- pip install google-compute-engine
- pip install $BEAM_PYTHON_SDK[gcp]
-}
-
-
-#######################################
# Run UserScore with DirectRunner
# Globals:
# USERSCORE_OUTPUT_PREFIX, DATASET, BUCKET_NAME
@@ -164,6 +144,7 @@ function verify_hourlyteamscore_dataflow() {
# VERSION
# Arguments:
# $1 - sdk types: [tar, wheel]
+# $2 - python interpreter version: [python2.7, python3.5, ...]
#######################################
function run_release_candidate_python_mobile_gaming() {
print_separator "Start Mobile Gaming Examples"
@@ -173,11 +154,11 @@ function run_release_candidate_python_mobile_gaming() {
echo $TMPDIR
pushd $TMPDIR
- download_files $1
+ download_files $1 $2
# get exact names of sdk and other files
BEAM_PYTHON_SDK=$(get_sdk_name $1)
- install_sdk
+ install_sdk $1 $2
verify_userscore_direct
verify_userscore_dataflow
verify_hourlyteamscore_direct
diff --git
a/release/src/main/python-release/run_release_candidate_python_quickstart.sh
b/release/src/main/python-release/run_release_candidate_python_quickstart.sh
index 9656a14..4700d30 100755
--- a/release/src/main/python-release/run_release_candidate_python_quickstart.sh
+++ b/release/src/main/python-release/run_release_candidate_python_quickstart.sh
@@ -72,34 +72,11 @@ function verify_hash() {
wget https://dist.apache.org/repos/dist/dev/beam/KEYS
gpg --import KEYS
gpg --verify $ASC_FILE_NAME $BEAM_PYTHON_SDK
- echo "test place 1"
gsutil version -l
}
#######################################
-# Create a new virtualenv and install the SDK
-# Globals:
-# BEAM_PYTHON_SDK
-# Arguments:
-# None
-#######################################
-function install_sdk() {
- print_separator "Creating new virtualenv and installing the SDK"
- echo "test place 2"
- gsutil version -l
- virtualenv temp_virtualenv
- . temp_virtualenv/bin/activate
- gcloud_version=$(gcloud --version | head -1 | awk '{print $4}')
- if [[ "$gcloud_version" < "300" ]]; then
- update_gcloud
- fi
- pip install google-compute-engine
- pip install $BEAM_PYTHON_SDK[gcp]
-}
-
-
-#######################################
# Run wordcount with DirectRunner
# Arguments:
# None
@@ -234,6 +211,7 @@ function verify_streaming_wordcount_dataflow() {
# VERSION
# Arguments:
# $1 - sdk types: [tar, wheel]
+# $2 - python interpreter version: [python2.7, python3.5, ...]
#######################################
function run_release_candidate_python_quickstart(){
print_separator "Start Quickstarts Examples"
@@ -242,14 +220,14 @@ function run_release_candidate_python_quickstart(){
echo $TMPDIR
pushd $TMPDIR
- download_files $1
+ download_files $1 $2
# get exact names of sdk and other files
BEAM_PYTHON_SDK=$(get_sdk_name $1)
ASC_FILE_NAME=$(get_asc_name $1)
SHA512_FILE_NAME=$(get_sha512_name $1)
verify_hash
- install_sdk
+ install_sdk $1 $2
verify_wordcount_direct
verify_wordcount_dataflow
verify_streaming_wordcount_direct
diff --git a/release/src/main/scripts/run_rc_validation.sh
b/release/src/main/scripts/run_rc_validation.sh
index 1fc0d54..393e2a0 100755
--- a/release/src/main/scripts/run_rc_validation.sh
+++ b/release/src/main/scripts/run_rc_validation.sh
@@ -51,6 +51,7 @@ GIT_REPO_URL=https://github.com/apache/beam.git
PYTHON_RC_DOWNLOAD_URL=https://dist.apache.org/repos/dist/dev/beam
HUB_VERSION=2.5.0
HUB_ARTIFACTS_NAME=hub-linux-amd64-${HUB_VERSION}
+declare -a DEFAULT_PYTHON_VERSIONS_TO_VALIDATE=("python2.7" "python3.5")
echo "[Input Required] Please enter the release version: "
read RELEASE
@@ -289,283 +290,297 @@ if [[ $confirmation = "y" ]]; then
echo "--------------------------Verifying
Hashes------------------------------------"
sha512sum -c apache-beam-${RELEASE}.zip.sha512
- echo "---------------------------Setting up
virtualenv------------------------------"
sudo `which pip` install --upgrade pip
sudo `which pip` install --upgrade setuptools
sudo `which pip` install --upgrade virtualenv
- virtualenv beam_env
- . beam_env/bin/activate
- echo "--------------------------Installing Python
SDK-------------------------------"
- pip install apache-beam-${RELEASE}.zip
- pip install apache-beam-${RELEASE}.zip[gcp]
+ echo "[Input Required] Please enter Python interpreter(s) separated by space
to use for running validation steps."
+ echo "Sample input: python2.7"
+ echo "Enter empty line to repeat validation steps using all of
${DEFAULT_PYTHON_VERSIONS_TO_VALIDATE[@]}."
- echo "----------------------------Setting up GCP
Sources----------------------------"
- echo "[GCP Project Required] Please input your GCP project:"
- read USER_GCP_PROJECT
- gcloud auth login
- gcloud config set project ${USER_GCP_PROJECT}
-
- MOBILE_GAME_GCS_BUCKET=gs://${USER}_python_validations_bucket
- MOBILE_GAME_DATASET=${USER}_python_validations
- MOBILE_GAME_PUBSUB_TOPIC=leader_board-${USER}-python-topic-1
- gsutil mb -p ${USER_GCP_PROJECT} ${MOBILE_GAME_GCS_BUCKET}
- gcloud alpha pubsub topics delete
projects/${USER_GCP_PROJECT}/topics/${MOBILE_GAME_PUBSUB_TOPIC}
- gcloud alpha pubsub topics create --project=${USER_GCP_PROJECT}
${MOBILE_GAME_PUBSUB_TOPIC}
-
- echo "-----------------------Setting Up Service
Account-----------------------------"
- echo "Please go to GCP IAM console under your project(${USER_GCP_PROJECT})."
- echo "Create a service account as project owner, if you don't have one."
- echo "[Input Required] Please enter your service account email:"
- read USER_SERVICE_ACCOUNT_EMAIL
- SERVICE_ACCOUNT_KEY_JSON=${USER}_json_key.json
- gcloud iam service-accounts keys create ${SERVICE_ACCOUNT_KEY_JSON}
--iam-account ${USER_SERVICE_ACCOUNT_EMAIL}
- export GOOGLE_APPLICATION_CREDENTIALS=$(pwd)/${SERVICE_ACCOUNT_KEY_JSON}
-
- echo "-----------------------Setting up Shell Env
Vars------------------------------"
- # [BEAM-4518]
- FIXED_WINDOW_DURATION=20
- cp ~/.bashrc ~/.bashrc_backup
- echo "export USER_GCP_PROJECT=${USER_GCP_PROJECT}" >> ~/.bashrc
- echo "export MOBILE_GAME_DATASET=${MOBILE_GAME_DATASET}" >> ~/.bashrc
- echo "export MOBILE_GAME_PUBSUB_TOPIC=${MOBILE_GAME_PUBSUB_TOPIC}" >>
~/.bashrc
- echo "export MOBILE_GAME_GCS_BUCKET=${MOBILE_GAME_GCS_BUCKET}" >> ~/.bashrc
- echo "export
GOOGLE_APPLICATION_CREDENTIALS=${GOOGLE_APPLICATION_CREDENTIALS}" >> ~/.bashrc
- echo "export RELEASE=${RELEASE}" >> ~/.bashrc
- echo "export FIXED_WINDOW_DURATION=${FIXED_WINDOW_DURATION}" >> ~/.bashrc
-
-
- echo "--------------------------Updating
~/.m2/settings.xml-------------------------"
- cd ~
- if [[ -d .m2 ]]; then
- mkdir .m2
- fi
- cd .m2
- if [[ -f ~/.m2/settings.xml ]]; then
- mv settings.xml settings_backup.xml
- fi
- touch settings.xml
- echo "<settings>" >> settings.xml
- echo " <profiles>" >> settings.xml
- echo " <profile>" >> settings.xml
- echo " <id>release-repo</id>" >> settings.xml
- echo " <activation>" >> settings.xml
- echo " <activeByDefault>true</activeByDefault>" >> settings.xml
- echo " </activation>" >> settings.xml
- echo " <repositories>" >> settings.xml
- echo " <repository>" >> settings.xml
- echo " <id>Release ${RELEASE} RC${RC_NUM}</id>" >> settings.xml
- echo " <name>Release ${RELEASE} RC${RC_NUM}</name>" >> settings.xml
- echo " <url>${REPO_URL}</url>" >> settings.xml
- echo " </repository>" >> settings.xml
- echo " </repositories>" >> settings.xml
- echo " </profile>" >> settings.xml
- echo " </profiles>" >> settings.xml
- echo "</settings>" >> settings.xml
-
- echo "----------------------Starting Pubsub Java
Injector--------------------------"
- cd ~/${LOCAL_CLONE_DIR}
- mvn archetype:generate \
- -DarchetypeGroupId=org.apache.beam \
- -DarchetypeArtifactId=beam-sdks-java-maven-archetypes-examples \
- -DarchetypeVersion=${RELEASE} \
- -DgroupId=org.example \
- -DartifactId=word-count-beam \
- -Dversion="0.1" \
- -Dpackage=org.apache.beam.examples \
- -DinteractiveMode=false \
- -DarchetypeCatalog=internal
-
- cd word-count-beam
- echo "A new terminal will pop up and start a java top injector."
- gnome-terminal -x sh -c \
- "echo '******************************************************';
- echo '* Running Pubsub Java Injector';
- echo '******************************************************';
- mvn compile exec:java
-Dexec.mainClass=org.apache.beam.examples.complete.game.injector.Injector \
- -Dexec.args='${USER_GCP_PROJECT} ${MOBILE_GAME_PUBSUB_TOPIC} none';
- exec bash"
-
- echo "[Confirmation Required] Please enter y to confirm injector running:"
- read confirmation
- if [[ $confirmation != "y" ]]; then
- echo "Following tests only can be ran when java injector running."
- clean_up
- exit
+ read -a PYTHON_VERSIONS_TO_VALIDATE
+ if [[ -z "$PYTHON_VERSIONS_TO_VALIDATE" ]]; then
+ PYTHON_VERSIONS_TO_VALIDATE=${DEFAULT_PYTHON_VERSIONS_TO_VALIDATE[@]}
fi
- cd ~/${LOCAL_CLONE_DIR}/
+ for py_version in "${PYTHON_VERSIONS_TO_VALIDATE[@]}"
+ do
+ rm -rf ./beam_env_${py_version}
+ echo "--------------Setting up virtualenv with $py_version
interpreter----------------"
+ virtualenv beam_env_${py_version} -p $py_version
+ . beam_env_${py_version}/bin/activate
- echo "----------------Starting Leaderboard with
DirectRunner-----------------------"
- echo "[Confirmation Required] Do you want to proceed? [y|N]"
- read confirmation
- if [[ $confirmation = "y" ]]; then
- bq rm -rf --project=${USER_GCP_PROJECT} ${MOBILE_GAME_DATASET}
- bq mk --project=${USER_GCP_PROJECT} ${MOBILE_GAME_DATASET}
- echo "This is a streaming job. This task will be launched in a separate
terminal."
+ echo "--------------------------Installing Python
SDK-------------------------------"
+ pip install apache-beam-${RELEASE}.zip
+ pip install apache-beam-${RELEASE}.zip[gcp]
+
+ echo "----------------------------Setting up GCP
Sources----------------------------"
+ echo "[GCP Project Required] Please input your GCP project:"
+ read USER_GCP_PROJECT
+ gcloud auth login
+ gcloud config set project ${USER_GCP_PROJECT}
+
+ MOBILE_GAME_GCS_BUCKET=gs://${USER}_python_validations_bucket
+ MOBILE_GAME_DATASET=${USER}_python_validations
+ MOBILE_GAME_PUBSUB_TOPIC=leader_board-${USER}-python-topic-1
+ gsutil mb -p ${USER_GCP_PROJECT} ${MOBILE_GAME_GCS_BUCKET}
+ gcloud alpha pubsub topics delete
projects/${USER_GCP_PROJECT}/topics/${MOBILE_GAME_PUBSUB_TOPIC}
+ gcloud alpha pubsub topics create --project=${USER_GCP_PROJECT}
${MOBILE_GAME_PUBSUB_TOPIC}
+
+ echo "-----------------------Setting Up Service
Account-----------------------------"
+ echo "Please go to GCP IAM console under your
project(${USER_GCP_PROJECT})."
+ echo "Create a service account as project owner, if you don't have one."
+ echo "[Input Required] Please enter your service account email:"
+ read USER_SERVICE_ACCOUNT_EMAIL
+ SERVICE_ACCOUNT_KEY_JSON=${USER}_json_key.json
+ gcloud iam service-accounts keys create ${SERVICE_ACCOUNT_KEY_JSON}
--iam-account ${USER_SERVICE_ACCOUNT_EMAIL}
+ export GOOGLE_APPLICATION_CREDENTIALS=$(pwd)/${SERVICE_ACCOUNT_KEY_JSON}
+
+ echo "-----------------------Setting up Shell Env
Vars------------------------------"
+ # [BEAM-4518]
+ FIXED_WINDOW_DURATION=20
+ cp ~/.bashrc ~/.bashrc_backup
+ echo "export USER_GCP_PROJECT=${USER_GCP_PROJECT}" >> ~/.bashrc
+ echo "export MOBILE_GAME_DATASET=${MOBILE_GAME_DATASET}" >> ~/.bashrc
+ echo "export MOBILE_GAME_PUBSUB_TOPIC=${MOBILE_GAME_PUBSUB_TOPIC}" >>
~/.bashrc
+ echo "export MOBILE_GAME_GCS_BUCKET=${MOBILE_GAME_GCS_BUCKET}" >> ~/.bashrc
+ echo "export
GOOGLE_APPLICATION_CREDENTIALS=${GOOGLE_APPLICATION_CREDENTIALS}" >> ~/.bashrc
+ echo "export RELEASE=${RELEASE}" >> ~/.bashrc
+ echo "export FIXED_WINDOW_DURATION=${FIXED_WINDOW_DURATION}" >> ~/.bashrc
+
+
+ echo "--------------------------Updating
~/.m2/settings.xml-------------------------"
+ cd ~
+ if [[ -d .m2 ]]; then
+ mkdir .m2
+ fi
+ cd .m2
+ if [[ -f ~/.m2/settings.xml ]]; then
+ mv settings.xml settings_backup.xml
+ fi
+ touch settings.xml
+ echo "<settings>" >> settings.xml
+ echo " <profiles>" >> settings.xml
+ echo " <profile>" >> settings.xml
+ echo " <id>release-repo</id>" >> settings.xml
+ echo " <activation>" >> settings.xml
+ echo " <activeByDefault>true</activeByDefault>" >> settings.xml
+ echo " </activation>" >> settings.xml
+ echo " <repositories>" >> settings.xml
+ echo " <repository>" >> settings.xml
+ echo " <id>Release ${RELEASE} RC${RC_NUM}</id>" >> settings.xml
+ echo " <name>Release ${RELEASE} RC${RC_NUM}</name>" >>
settings.xml
+ echo " <url>${REPO_URL}</url>" >> settings.xml
+ echo " </repository>" >> settings.xml
+ echo " </repositories>" >> settings.xml
+ echo " </profile>" >> settings.xml
+ echo " </profiles>" >> settings.xml
+ echo "</settings>" >> settings.xml
+
+ echo "----------------------Starting Pubsub Java
Injector--------------------------"
+ cd ~/${LOCAL_CLONE_DIR}
+ mvn archetype:generate \
+ -DarchetypeGroupId=org.apache.beam \
+ -DarchetypeArtifactId=beam-sdks-java-maven-archetypes-examples \
+ -DarchetypeVersion=${RELEASE} \
+ -DgroupId=org.example \
+ -DartifactId=word-count-beam \
+ -Dversion="0.1" \
+ -Dpackage=org.apache.beam.examples \
+ -DinteractiveMode=false \
+ -DarchetypeCatalog=internal
+
+ cd word-count-beam
+ echo "A new terminal will pop up and start a java top injector."
gnome-terminal -x sh -c \
- "echo '*****************************************************';
- echo '* Running Python Leaderboard with DirectRunner';
- echo '*****************************************************';
- python -m apache_beam.examples.complete.game.leader_board \
- --project=${USER_GCP_PROJECT} \
- --topic projects/${USER_GCP_PROJECT}/topics/${MOBILE_GAME_PUBSUB_TOPIC} \
- --dataset ${MOBILE_GAME_DATASET};
+ "echo '******************************************************';
+ echo '* Running Pubsub Java Injector';
+ echo '******************************************************';
+ mvn compile exec:java
-Dexec.mainClass=org.apache.beam.examples.complete.game.injector.Injector \
+ -Dexec.args='${USER_GCP_PROJECT} ${MOBILE_GAME_PUBSUB_TOPIC} none';
exec bash"
- echo "***************************************************************"
- echo "* Please wait for at least 5 mins to let results get populated."
- echo "* Sleeping for 5 mins"
- sleep 5m
- echo "***************************************************************"
- echo "* How to verify results:"
- echo "* 1. Check whether there is any error messages in the task running
terminal."
- echo "* 2. Goto your BigQuery console and check whether your
${MOBILE_GAME_DATASET} has leader_board_users and leader_board_teams table."
- echo "* 3. Check whether leader_board_users has data, retrieving BigQuery
data as below: "
- bq head -n 10 ${MOBILE_GAME_DATASET}.leader_board_users
- echo "* 4. Check whether leader_board_teams has data, retrieving BigQuery
data as below:"
- bq head -n 10 ${MOBILE_GAME_DATASET}.leader_board_teams
- echo "***************************************************************"
-
- echo "If you have verified all items listed above, please terminate the
python job."
- echo "[Confirmation Required] Please confirm whether you have stopped this
job: [y|N]"
+ echo "[Confirmation Required] Please enter y to confirm injector running:"
read confirmation
if [[ $confirmation != "y" ]]; then
- echo "Current job must be terminated in order to proceed into next test."
+ echo "Following tests only can be ran when java injector running."
clean_up
exit
fi
- fi
- echo "----------------Starting Leaderboard with
DataflowRunner---------------------"
- echo "[Confirmation Required] Do you want to proceed? [y|N]"
- read confirmation
- if [[ $confirmation = "y" ]]; then
- bq rm -rf --project=${USER_GCP_PROJECT} ${MOBILE_GAME_DATASET}
- bq mk --project=${USER_GCP_PROJECT} ${MOBILE_GAME_DATASET}
- echo "This is a streaming job. This task will be launched in a separate
terminal."
- gnome-terminal -x sh -c \
- "echo '*****************************************************';
- echo '* Running Python Leaderboard with DataflowRunner';
- echo '*****************************************************';
- python -m apache_beam.examples.complete.game.leader_board \
- --project=${USER_GCP_PROJECT} \
- --topic projects/${USER_GCP_PROJECT}/topics/${MOBILE_GAME_PUBSUB_TOPIC} \
- --dataset ${MOBILE_GAME_DATASET} \
- --runner DataflowRunner \
- --temp_location=${MOBILE_GAME_GCS_BUCKET}/temp/ \
- --sdk_location apache-beam-${RELEASE}.zip; \
- exec bash"
+ cd ~/${LOCAL_CLONE_DIR}/
- echo "***************************************************************"
- echo "* Please wait for at least 10 mins to let Dataflow job be launched
and results get populated."
- echo "* Sleeping for 10 mins"
- sleep 10m
- echo "* How to verify results:"
- echo "* 1. Goto your Dataflow job console and check whether there is any
error."
- echo "* 2. Goto your BigQuery console and check whether your
${MOBILE_GAME_DATASET} has leader_board_users and leader_board_teams table."
- echo "* 3. Check whether leader_board_users has data, retrieving BigQuery
data as below: "
- bq head -n 10 ${MOBILE_GAME_DATASET}.leader_board_users
- echo "* 4. Check whether leader_board_teams has data, retrieving BigQuery
data as below:"
- bq head -n 10 ${MOBILE_GAME_DATASET}.leader_board_teams
- echo "***************************************************************"
-
- echo "If you have verified all items listed above, please terminate this
job in Dataflow Console."
- echo "[Confirmation Required] Please confirm whether you have stopped this
job: [y|N]"
+ echo "----------------Starting Leaderboard with
DirectRunner-----------------------"
+ echo "[Confirmation Required] Do you want to proceed? [y|N]"
read confirmation
- if [[ $confirmation != "y" ]]; then
- echo "Current job must be terminated in order to proceed into next test."
- clean_up
- exit
+ if [[ $confirmation = "y" ]]; then
+ bq rm -rf --project=${USER_GCP_PROJECT} ${MOBILE_GAME_DATASET}
+ bq mk --project=${USER_GCP_PROJECT} ${MOBILE_GAME_DATASET}
+ echo "This is a streaming job. This task will be launched in a separate
terminal."
+ gnome-terminal -x sh -c \
+ "echo '*****************************************************';
+ echo '* Running Python Leaderboard with DirectRunner';
+ echo '*****************************************************';
+ python -m apache_beam.examples.complete.game.leader_board \
+ --project=${USER_GCP_PROJECT} \
+ --topic projects/${USER_GCP_PROJECT}/topics/${MOBILE_GAME_PUBSUB_TOPIC} \
+ --dataset ${MOBILE_GAME_DATASET};
+ exec bash"
+
+ echo "***************************************************************"
+ echo "* Please wait for at least 5 mins to let results get populated."
+ echo "* Sleeping for 5 mins"
+ sleep 5m
+ echo "***************************************************************"
+ echo "* How to verify results:"
+ echo "* 1. Check whether there is any error messages in the task running
terminal."
+ echo "* 2. Goto your BigQuery console and check whether your
${MOBILE_GAME_DATASET} has leader_board_users and leader_board_teams table."
+ echo "* 3. Check whether leader_board_users has data, retrieving
BigQuery data as below: "
+ bq head -n 10 ${MOBILE_GAME_DATASET}.leader_board_users
+ echo "* 4. Check whether leader_board_teams has data, retrieving
BigQuery data as below:"
+ bq head -n 10 ${MOBILE_GAME_DATASET}.leader_board_teams
+ echo "***************************************************************"
+
+ echo "If you have verified all items listed above, please terminate the
python job."
+ echo "[Confirmation Required] Please confirm whether you have stopped
this job: [y|N]"
+ read confirmation
+ if [[ $confirmation != "y" ]]; then
+ echo "Current job must be terminated in order to proceed into next
test."
+ clean_up
+ exit
+ fi
fi
- fi
-
- echo "------------------Starting GameStats with
DirectRunner-----------------------"
- echo "[Confirmation Required] Do you want to proceed? [y|N]"
- read confirmation
- if [[ $confirmation = "y" ]]; then
- bq rm -rf --project=${USER_GCP_PROJECT} ${MOBILE_GAME_DATASET}
- bq mk --project=${USER_GCP_PROJECT} ${MOBILE_GAME_DATASET}
- echo "This is a streaming job. This task will be launched in a separate
terminal."
- echo "Streaming job is running with
fixed_window_duration=${FIXED_WINDOW_DURATION}"
- gnome-terminal -x sh -c \
- "echo '*****************************************************';
- echo '* Running GameStats with DirectRunner';
- echo '*****************************************************';
- python -m apache_beam.examples.complete.game.game_stats \
- --project=${USER_GCP_PROJECT} \
- --topic projects/${USER_GCP_PROJECT}/topics/${MOBILE_GAME_PUBSUB_TOPIC} \
- --dataset ${MOBILE_GAME_DATASET} \
- --fixed_window_duration ${FIXED_WINDOW_DURATION}; \
- exec bash"
+ echo "----------------Starting Leaderboard with
DataflowRunner---------------------"
+ echo "[Confirmation Required] Do you want to proceed? [y|N]"
+ read confirmation
+ if [[ $confirmation = "y" ]]; then
+ bq rm -rf --project=${USER_GCP_PROJECT} ${MOBILE_GAME_DATASET}
+ bq mk --project=${USER_GCP_PROJECT} ${MOBILE_GAME_DATASET}
+ echo "This is a streaming job. This task will be launched in a separate
terminal."
+ gnome-terminal -x sh -c \
+ "echo '*****************************************************';
+ echo '* Running Python Leaderboard with DataflowRunner';
+ echo '*****************************************************';
+ python -m apache_beam.examples.complete.game.leader_board \
+ --project=${USER_GCP_PROJECT} \
+ --topic projects/${USER_GCP_PROJECT}/topics/${MOBILE_GAME_PUBSUB_TOPIC} \
+ --dataset ${MOBILE_GAME_DATASET} \
+ --runner DataflowRunner \
+ --temp_location=${MOBILE_GAME_GCS_BUCKET}/temp/ \
+ --sdk_location apache-beam-${RELEASE}.zip; \
+ exec bash"
+
+ echo "***************************************************************"
+ echo "* Please wait for at least 10 mins to let Dataflow job be launched
and results get populated."
+ echo "* Sleeping for 10 mins"
+ sleep 10m
+ echo "* How to verify results:"
+ echo "* 1. Goto your Dataflow job console and check whether there is any
error."
+ echo "* 2. Goto your BigQuery console and check whether your
${MOBILE_GAME_DATASET} has leader_board_users and leader_board_teams table."
+ echo "* 3. Check whether leader_board_users has data, retrieving
BigQuery data as below: "
+ bq head -n 10 ${MOBILE_GAME_DATASET}.leader_board_users
+ echo "* 4. Check whether leader_board_teams has data, retrieving
BigQuery data as below:"
+ bq head -n 10 ${MOBILE_GAME_DATASET}.leader_board_teams
+ echo "***************************************************************"
+
+ echo "If you have verified all items listed above, please terminate this
job in Dataflow Console."
+ echo "[Confirmation Required] Please confirm whether you have stopped
this job: [y|N]"
+ read confirmation
+ if [[ $confirmation != "y" ]]; then
+ echo "Current job must be terminated in order to proceed into next
test."
+ clean_up
+ exit
+ fi
+ fi
- echo "***************************************************************"
- echo "* Please wait for at least 25 mins to let results get populated."
- echo "* Sleeping for 25mins"
- sleep 25m
- echo "* How to verify results:"
- echo "* 1. Check whether there is any error messages in the task running
terminal."
- echo "* 2. Goto your BigQuery console and check whether your
${MOBILE_GAME_DATASET} has game_stats_teams and game_stats_sessions table."
- echo "* 3. Check whether game_stats_teams has data, retrieving BigQuery
data as below: "
- bq head -n 10 ${MOBILE_GAME_DATASET}.game_stats_teams
- echo "* 4. Check whether game_stats_sessions has data, retrieving BigQuery
data as below:"
- bq head -n 10 ${MOBILE_GAME_DATASET}.game_stats_sessions
- echo "***************************************************************"
-
- echo "If you have verified all items listed above, please terminate the
python job."
- echo "[Confirmation Required] Please confirm whether you have stopped this
job: [y|N]"
+ echo "------------------Starting GameStats with
DirectRunner-----------------------"
+ echo "[Confirmation Required] Do you want to proceed? [y|N]"
read confirmation
- if [[ $confirmation != "y" ]]; then
- echo "Current job must be terminated in order to proceed into next test."
- clean_up
- exit
+ if [[ $confirmation = "y" ]]; then
+ bq rm -rf --project=${USER_GCP_PROJECT} ${MOBILE_GAME_DATASET}
+ bq mk --project=${USER_GCP_PROJECT} ${MOBILE_GAME_DATASET}
+
+ echo "This is a streaming job. This task will be launched in a separate
terminal."
+ echo "Streaming job is running with
fixed_window_duration=${FIXED_WINDOW_DURATION}"
+ gnome-terminal -x sh -c \
+ "echo '*****************************************************';
+ echo '* Running GameStats with DirectRunner';
+ echo '*****************************************************';
+ python -m apache_beam.examples.complete.game.game_stats \
+ --project=${USER_GCP_PROJECT} \
+ --topic projects/${USER_GCP_PROJECT}/topics/${MOBILE_GAME_PUBSUB_TOPIC} \
+ --dataset ${MOBILE_GAME_DATASET} \
+ --fixed_window_duration ${FIXED_WINDOW_DURATION}; \
+ exec bash"
+
+ echo "***************************************************************"
+ echo "* Please wait for at least 25 mins to let results get populated."
+ echo "* Sleeping for 25mins"
+ sleep 25m
+ echo "* How to verify results:"
+ echo "* 1. Check whether there is any error messages in the task running
terminal."
+ echo "* 2. Goto your BigQuery console and check whether your
${MOBILE_GAME_DATASET} has game_stats_teams and game_stats_sessions table."
+ echo "* 3. Check whether game_stats_teams has data, retrieving BigQuery
data as below: "
+ bq head -n 10 ${MOBILE_GAME_DATASET}.game_stats_teams
+ echo "* 4. Check whether game_stats_sessions has data, retrieving
BigQuery data as below:"
+ bq head -n 10 ${MOBILE_GAME_DATASET}.game_stats_sessions
+ echo "***************************************************************"
+
+ echo "If you have verified all items listed above, please terminate the
python job."
+ echo "[Confirmation Required] Please confirm whether you have stopped
this job: [y|N]"
+ read confirmation
+ if [[ $confirmation != "y" ]]; then
+ echo "Current job must be terminated in order to proceed into next
test."
+ clean_up
+ exit
+ fi
fi
- fi
- echo "-------------------Starting GameStats with
DataflowRunner--------------------"
- echo "[Confirmation Required] Do you want to proceed? [y|N]"
- read confirmation
- if [[ $confirmation = "y" ]]; then
- bq rm -rf --project=${USER_GCP_PROJECT} ${MOBILE_GAME_DATASET}
- bq mk --project=${USER_GCP_PROJECT} ${MOBILE_GAME_DATASET}
- echo "This is a streaming job. This task will be launched in a separate
terminal."
- echo "Streaming job is running with
fixed_window_duration=${FIXED_WINDOW_DURATION}"
- gnome-terminal -x sh -c \
- "echo '*****************************************************';
- echo '* Running GameStats with DataflowRunner';
- echo '*****************************************************';
- python -m apache_beam.examples.complete.game.game_stats \
- --project=${USER_GCP_PROJECT} \
- --topic projects/${USER_GCP_PROJECT}/topics/${MOBILE_GAME_PUBSUB_TOPIC} \
- --dataset ${MOBILE_GAME_DATASET} \
- --runner DataflowRunner \
- --temp_location=${MOBILE_GAME_GCS_BUCKET}/temp/ \
- --sdk_location apache-beam-${RELEASE}.zip \
- --fixed_window_duration ${FIXED_WINDOW_DURATION}; exec bash"
-
- echo "***************************************************************"
- echo "* Please wait for at least 30 mins to let results get populated."
- echo "* Sleeping for 30 mins"
- sleep 30m
- echo "* How to verify results:"
- echo "* 1. Goto your Dataflow job console and check whether there is any
error."
- echo "* 2. Goto your BigQuery console and check whether your
${MOBILE_GAME_DATASET} has game_stats_teams and game_stats_sessions table."
- echo "* 3. Check whether game_stats_teams has data, retrieving BigQuery
data as below: "
- bq head -n 10 ${MOBILE_GAME_DATASET}.game_stats_teams
- echo "* 4. Check whether game_stats_sessions has data, retrieving BigQuery
data as below:"
- bq head -n 10 ${MOBILE_GAME_DATASET}.game_stats_sessions
- echo "***************************************************************"
-
- echo "If you have verified all items listed above, please terminate the
python job."
- echo "[Confirmation Required] Please confirm whether you have stopped this
job: [y|N]"
+ echo "-------------------Starting GameStats with
DataflowRunner--------------------"
+ echo "[Confirmation Required] Do you want to proceed? [y|N]"
read confirmation
- if [[ $confirmation != "y" ]]; then
- echo "Current job must be terminated in order to proceed into next test."
- clean_up
- exit
+ if [[ $confirmation = "y" ]]; then
+ bq rm -rf --project=${USER_GCP_PROJECT} ${MOBILE_GAME_DATASET}
+ bq mk --project=${USER_GCP_PROJECT} ${MOBILE_GAME_DATASET}
+ echo "This is a streaming job. This task will be launched in a separate
terminal."
+ echo "Streaming job is running with
fixed_window_duration=${FIXED_WINDOW_DURATION}"
+ gnome-terminal -x sh -c \
+ "echo '*****************************************************';
+ echo '* Running GameStats with DataflowRunner';
+ echo '*****************************************************';
+ python -m apache_beam.examples.complete.game.game_stats \
+ --project=${USER_GCP_PROJECT} \
+ --topic projects/${USER_GCP_PROJECT}/topics/${MOBILE_GAME_PUBSUB_TOPIC} \
+ --dataset ${MOBILE_GAME_DATASET} \
+ --runner DataflowRunner \
+ --temp_location=${MOBILE_GAME_GCS_BUCKET}/temp/ \
+ --sdk_location apache-beam-${RELEASE}.zip \
+ --fixed_window_duration ${FIXED_WINDOW_DURATION}; exec bash"
+
+ echo "***************************************************************"
+ echo "* Please wait for at least 30 mins to let results get populated."
+ echo "* Sleeping for 30 mins"
+ sleep 30m
+ echo "* How to verify results:"
+ echo "* 1. Goto your Dataflow job console and check whether there is any
error."
+ echo "* 2. Goto your BigQuery console and check whether your
${MOBILE_GAME_DATASET} has game_stats_teams and game_stats_sessions table."
+ echo "* 3. Check whether game_stats_teams has data, retrieving BigQuery
data as below: "
+ bq head -n 10 ${MOBILE_GAME_DATASET}.game_stats_teams
+ echo "* 4. Check whether game_stats_sessions has data, retrieving
BigQuery data as below:"
+ bq head -n 10 ${MOBILE_GAME_DATASET}.game_stats_sessions
+ echo "***************************************************************"
+
+ echo "If you have verified all items listed above, please terminate the
python job."
+ echo "[Confirmation Required] Please confirm whether you have stopped
this job: [y|N]"
+ read confirmation
+ if [[ $confirmation != "y" ]]; then
+ echo "Current job must be terminated in order to proceed into next
test."
+ clean_up
+ exit
+ fi
fi
- fi
+ done # Loop over Python versions.
fi
clean_up
