[
https://issues.apache.org/jira/browse/BEAM-5068?focusedWorklogId=131474&page=com.atlassian.jira.plugin.system.issuetabpanels:worklog-tabpanel#worklog-131474
]
ASF GitHub Bot logged work on BEAM-5068:
----------------------------------------
Author: ASF GitHub Bot
Created on: 06/Aug/18 16:44
Start Date: 06/Aug/18 16:44
Worklog Time Spent: 10m
Work Description: pabloem closed pull request #6135: [BEAM-5068] switch
mobile gaming input file to a public storage
URL: https://github.com/apache/beam/pull/6135
This is a PR merged from a forked repository.
As GitHub hides the original diff on merge, it is displayed below for
the sake of provenance:
As this is a foreign pull request (from a fork), the diff is supplied
below (as it won't show otherwise due to GitHub magic):
diff --git a/.test-infra/jenkins/job_PostRelease_NightlySnapshot.groovy
b/.test-infra/jenkins/job_PostRelease_NightlySnapshot.groovy
index 4679229659b..8d3a7fb1279 100644
--- a/.test-infra/jenkins/job_PostRelease_NightlySnapshot.groovy
+++ b/.test-infra/jenkins/job_PostRelease_NightlySnapshot.groovy
@@ -23,9 +23,6 @@ import CommonJobProperties as commonJobProperties
job('beam_PostRelease_NightlySnapshot') {
description('Runs post release verification of the nightly snapshot.')
- // Execute concurrent builds if necessary.
- concurrentBuild()
-
// Set common parameters.
commonJobProperties.setTopLevelMainJobProperties(delegate)
diff --git a/release/src/main/groovy/MobileGamingCommands.groovy
b/release/src/main/groovy/MobileGamingCommands.groovy
index bf8d04982fa..13874d10469 100644
--- a/release/src/main/groovy/MobileGamingCommands.groovy
+++ b/release/src/main/groovy/MobileGamingCommands.groovy
@@ -22,6 +22,8 @@ class MobileGamingCommands {
private TestScripts testScripts
+ private static final INPUT_GAMING_DATA =
"gs://dataflow-samples/game/5000_gaming_data.csv"
+
public static final RUNNERS = [DirectRunner: "direct-runner",
DataflowRunner: "dataflow-runner",
SparkRunner: "spark-runner",
@@ -64,10 +66,10 @@ class MobileGamingCommands {
return "java-hourlyteamscore-result-${RUNNERS[runner]}.txt"
}
- public String createPipelineCommand(String exampleName, String runner){
+ public String createPipelineCommand(String exampleName, String runner,
String jobName=''){
return """mvn compile exec:java -q \
-Dexec.mainClass=org.apache.beam.examples.complete.game.${exampleName} \
- -Dexec.args=\"${getArgs(exampleName, runner)}\" \
+ -Dexec.args=\"${getArgs(exampleName, runner, jobName)}\" \
-P${RUNNERS[runner]}"""
}
@@ -78,7 +80,7 @@ class MobileGamingCommands {
}
- private String getArgs(String exampleName, String runner){
+ private String getArgs(String exampleName, String runner, String jobName){
def args
switch (exampleName) {
case "UserScore":
@@ -88,10 +90,10 @@ class MobileGamingCommands {
args = getHourlyTeamScoreArgs(runner)
break
case "LeaderBoard":
- args = getLeaderBoardArgs(runner)
+ args = getLeaderBoardArgs(runner, jobName)
break
case "GameStats":
- args = getGameStatsArgs(runner)
+ args = getGameStatsArgs(runner, jobName)
break
default:
testScripts.error("Cannot find example ${exampleName} in archetypes.")
@@ -104,39 +106,41 @@ class MobileGamingCommands {
private Map getUserScoreArgs(String runner){
if(runner == "DataflowRunner"){
- return [input: "gs://${testScripts.gcsBucket()}/5000_gaming_data.csv",
+ return [input: INPUT_GAMING_DATA,
project: testScripts.gcpProject(),
output:
"gs://${testScripts.gcsBucket()}/${getUserScoreOutputName(runner)}"]
}
- return [input: "gs://${testScripts.gcsBucket()}/5000_gaming_data.csv",
+ return [input: INPUT_GAMING_DATA,
output: "${getUserScoreOutputName(runner)}"]
}
private Map getHourlyTeamScoreArgs(String runner){
if(runner == "DataflowRunner"){
- return [input: "gs://${testScripts.gcsBucket()}/5000_gaming_data.csv",
+ return [input: INPUT_GAMING_DATA,
project: testScripts.gcpProject(),
output:
"gs://${testScripts.gcsBucket()}/${getHourlyTeamScoreOutputName(runner)}"]
}
- return [input: "gs://${testScripts.gcsBucket()}/5000_gaming_data.csv",
+ return [input: INPUT_GAMING_DATA,
output: "${getHourlyTeamScoreOutputName(runner)}"]
}
- private Map getLeaderBoardArgs(String runner){
+ private Map getLeaderBoardArgs(String runner, String jobName){
return [project: testScripts.gcpProject(),
dataset: testScripts.bqDataset(),
topic:
"projects/${testScripts.gcpProject()}/topics/${testScripts.pubsubTopic()}",
leaderBoardTableName: "leaderboard_${runner}",
- teamWindowDuration: 5]
+ teamWindowDuration: 5,
+ jobName: jobName]
}
- private Map getGameStatsArgs(String runner){
+ private Map getGameStatsArgs(String runner, String jobName){
return [project: testScripts.gcpProject(),
dataset: testScripts.bqDataset(),
topic:
"projects/${testScripts.gcpProject()}/topics/${testScripts.pubsubTopic()}",
fixedWindowDuration: 5,
userActivityWindowDuration: 5,
sessionGap: 1,
- gameStatsTablePrefix: "gamestats_${runner}"]
+ gameStatsTablePrefix: "gamestats_${runner}",
+ jobName: jobName]
}
}
diff --git a/release/src/main/groovy/mobilegaming-java-dataflow.groovy
b/release/src/main/groovy/mobilegaming-java-dataflow.groovy
index 1d721c86ef5..704870deafe 100644
--- a/release/src/main/groovy/mobilegaming-java-dataflow.groovy
+++ b/release/src/main/groovy/mobilegaming-java-dataflow.groovy
@@ -76,8 +76,9 @@ def InjectorThread = Thread.start() {
t.run(mobileGamingCommands.createInjectorCommand())
}
+jobName = "leaderboard-validation-" + new Date().getTime() + "-" + new
Random().nextInt(1000)
def LeaderBoardThread = Thread.start() {
- t.run(mobileGamingCommands.createPipelineCommand("LeaderBoard", runner))
+ t.run(mobileGamingCommands.createPipelineCommand("LeaderBoard", runner,
jobName))
}
t.run("gcloud dataflow jobs list | grep pyflow-wordstream-candidate | grep
Running | cut -d' ' -f1")
@@ -101,7 +102,7 @@ while((System.currentTimeMillis() - startTime)/60000 <
mobileGamingCommands.EXEC
}
InjectorThread.stop()
LeaderBoardThread.stop()
-t.run("gcloud dataflow jobs cancel \$(gcloud dataflow jobs list | grep
leaderboard-jenkins | grep Running | cut -d' ' -f1)")
+t.run("gcloud dataflow jobs cancel \$(gcloud dataflow jobs list | grep
${jobName} | grep Running | cut -d' ' -f1)")
if(!isSuccess){
t.error("FAILED: Failed running LeaderBoard on DataflowRunner")
diff --git a/release/src/main/groovy/mobilegaming-java-direct.groovy
b/release/src/main/groovy/mobilegaming-java-direct.groovy
index b402ae9163a..9ff24cefaf9 100644
--- a/release/src/main/groovy/mobilegaming-java-direct.groovy
+++ b/release/src/main/groovy/mobilegaming-java-direct.groovy
@@ -74,8 +74,9 @@ def InjectorThread = Thread.start() {
t.run(mobileGamingCommands.createInjectorCommand())
}
+jobName = "leaderboard-validation-" + new Date().getTime() + "-" + new
Random().nextInt(1000)
def LeaderBoardThread = Thread.start() {
- t.run(mobileGamingCommands.createPipelineCommand("LeaderBoard", runner))
+ t.run(mobileGamingCommands.createPipelineCommand("LeaderBoard", runner,
jobName))
}
// verify outputs in BQ tables
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
us...@infra.apache.org
Issue Time Tracking
-------------------
Worklog Id: (was: 131474)
Time Spent: 3h 40m (was: 3.5h)
> Get Java mobile-gaming auto validations run on any local environments.
> ----------------------------------------------------------------------
>
> Key: BEAM-5068
> URL: https://issues.apache.org/jira/browse/BEAM-5068
> Project: Beam
> Issue Type: Sub-task
> Components: testing
> Reporter: yifan zou
> Assignee: yifan zou
> Priority: Major
> Time Spent: 3h 40m
> Remaining Estimate: 0h
>
--
This message was sent by Atlassian JIRA
(v7.6.3#76005)
