This is an automated email from the ASF dual-hosted git repository.
emilyye pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/beam.git
The following commit(s) were added to refs/heads/master by this push:
new 6aafcfd [BEAM-9150] Fix beam_PostRelease_Python_Candidate (python RC
validation scripts) (#16955)
6aafcfd is described below
commit 6aafcfdff3730cad6179ca7266ff8bfc4858dea7
Author: emily <emil...@google.com>
AuthorDate: Mon Feb 28 23:46:44 2022 -0800
[BEAM-9150] Fix beam_PostRelease_Python_Candidate (python RC validation
scripts) (#16955)
---
.../python_release_automation_utils.sh | 35 +++++++++++++++++++---
.../run_release_candidate_python_mobile_gaming.sh | 10 +++++--
.../run_release_candidate_python_quickstart.sh | 4 +--
3 files changed, 41 insertions(+), 8 deletions(-)
diff --git a/release/src/main/python-release/python_release_automation_utils.sh
b/release/src/main/python-release/python_release_automation_utils.sh
index 5d0e1c6..b328c42 100644
--- a/release/src/main/python-release/python_release_automation_utils.sh
+++ b/release/src/main/python-release/python_release_automation_utils.sh
@@ -228,7 +228,7 @@ function cleanup_pubsub() {
# $2 - pid: the pid of running pipeline
# $3 - running_job (DataflowRunner only): the job id of streaming pipeline
running on DataflowRunner
#######################################
-function verify_steaming_result() {
+function verify_streaming_result() {
retry=3
should_see="Python: "
while(( $retry > 0 )); do
@@ -295,9 +295,11 @@ function verify_user_score() {
function verify_hourly_team_score() {
retry=3
should_see='AntiqueBrassPlatypus'
+ runner=$1
+
while(( $retry >= 0 )); do
if [[ $retry > 0 ]]; then
- bq_pull_result=$(bq head -n 500 $DATASET.hourly_team_score_python_$1)
+ bq_pull_result=$(bq head -n 500
${DATASET}.${HOURLY_TEAM_SCORE_TABLE_PREFIX}_${runner})
if [[ $bq_pull_result = *"$should_see"* ]]; then
echo "SUCCEED: hourly_team_score example successful run on $1-runner"
break
@@ -307,14 +309,38 @@ function verify_hourly_team_score() {
sleep 15
fi
else
- echo "FAILED: HourlyTeamScore example failed running on $1-runner. \
- Did not found scores of team $should_see in $DATASET.leader_board"
+ echo "FAILED: HourlyTeamScore example failed running on $runner runner. \
+ Did not found scores of team $should_see in
${DATASET}.${HOURLY_TEAM_SCORE_TABLE_PREFIX}_${runner}"
complete "FAILED"
exit 1
fi
done
}
+function cleanup_hourly_team_score() {
+ retry=3
+ runner=$1
+
+ echo "Removing previously created table
${DATASET}.${HOURLY_TEAM_SCORE_TABLE_PREFIX}_${runner}..."
+ bq rm -q -f -t "${DATASET}.${HOURLY_TEAM_SCORE_TABLE_PREFIX}_${runner}"
+
+ while(( $retry >= 0 )); do
+ if [[ $retry > 0 ]]; then
+ bq_ls_result=$(bq ls $DATASET)
+ if [[ $bq_ls_result = *"${HOURLY_TEAM_SCORE_TABLE_PREFIX}_${runner}"*
]]; then
+ retry=$(($retry-1))
+ echo "${DATASET}.${HOURLY_TEAM_SCORE_TABLE_PREFIX}_${runner} not
cleaned up yet, waiting"
+ sleep 1000
+ else
+ echo "Confirmed ${DATASET}.${HOURLY_TEAM_SCORE_TABLE_PREFIX}_${runner}
removed before running new test."
+ break
+ fi
+ else
+ echo "WARNING: Unable to clean up table
${DATASET}.${HOURLY_TEAM_SCORE_TABLE_PREFIX}_${runner}. \
+ You may need to manually run 'bq rm -r -t
${DATASET}.${HOURLY_TEAM_SCORE_TABLE_PREFIX}_${runner}'."
+ fi
+ done
+}
# Python RC configurations
VERSION=$(get_version)
@@ -336,4 +362,5 @@ PUBSUB_SUBSCRIPTION='wordstream-python-sub2'
# Mobile Gaming Configurations
DATASET='beam_postrelease_mobile_gaming'
USERSCORE_OUTPUT_PREFIX='python-userscore_result'
+HOURLY_TEAM_SCORE_TABLE_PREFIX='hourly_team_score_python'
GAME_INPUT_DATA='gs://dataflow-samples/game/5000_gaming_data.csv'
diff --git
a/release/src/main/python-release/run_release_candidate_python_mobile_gaming.sh
b/release/src/main/python-release/run_release_candidate_python_mobile_gaming.sh
index 30d8cea..efd903d 100755
---
a/release/src/main/python-release/run_release_candidate_python_mobile_gaming.sh
+++
b/release/src/main/python-release/run_release_candidate_python_mobile_gaming.sh
@@ -100,12 +100,15 @@ function verify_userscore_dataflow() {
#######################################
function verify_hourlyteamscore_direct() {
print_separator "Running HourlyTeamScore example with DirectRunner"
+ # Clean up old bq tables
+ cleanup_hourly_team_score "direct"
+
python -m apache_beam.examples.complete.game.hourly_team_score \
--project=$PROJECT_ID \
--dataset=$DATASET \
--input=$GAME_INPUT_DATA \
--temp_location=gs://$BUCKET_NAME/temp/ \
- --table="hourly_team_score_python_direct"
+ --table="${HOURLY_TEAM_SCORE_TABLE_PREFIX}_direct"
verify_hourly_team_score "direct"
}
@@ -121,6 +124,9 @@ function verify_hourlyteamscore_direct() {
#######################################
function verify_hourlyteamscore_dataflow() {
print_separator "Running HourlyTeamScore example with DataflowRunner"
+ # Clean up old bq tables
+ cleanup_hourly_team_score "dataflow"
+
python -m apache_beam.examples.complete.game.hourly_team_score \
--project=$PROJECT_ID \
--region=$REGION_ID \
@@ -129,7 +135,7 @@ function verify_hourlyteamscore_dataflow() {
--temp_location=gs://$BUCKET_NAME/temp/ \
--sdk_location $BEAM_PYTHON_SDK \
--input=$GAME_INPUT_DATA \
- --table="hourly_team_score_python_dataflow"
+ --table="${HOURLY_TEAM_SCORE_TABLE_PREFIX}_dataflow"
verify_hourly_team_score "dataflow"
}
diff --git
a/release/src/main/python-release/run_release_candidate_python_quickstart.sh
b/release/src/main/python-release/run_release_candidate_python_quickstart.sh
index 6f1e10d..3af527a 100755
--- a/release/src/main/python-release/run_release_candidate_python_quickstart.sh
+++ b/release/src/main/python-release/run_release_candidate_python_quickstart.sh
@@ -155,7 +155,7 @@ function verify_streaming_wordcount_direct() {
# verify result
run_pubsub_publish
- verify_steaming_result "DirectRunner" $pid
+ verify_streaming_result "DirectRunner" $pid
kill -9 $pid
sleep 10
@@ -194,7 +194,7 @@ function verify_streaming_wordcount_dataflow() {
# verify result
run_pubsub_publish
sleep 420
- verify_steaming_result "DataflowRunner" $pid $running_job
+ verify_streaming_result "DataflowRunner" $pid $running_job
kill -9 $pid
gcloud dataflow jobs cancel $running_job
