[
https://issues.apache.org/jira/browse/BEAM-5781?focusedWorklogId=156500&page=com.atlassian.jira.plugin.system.issuetabpanels:worklog-tabpanel#worklog-156500
]
ASF GitHub Bot logged work on BEAM-5781:
----------------------------------------
Author: ASF GitHub Bot
Created on: 19/Oct/18 23:38
Start Date: 19/Oct/18 23:38
Worklog Time Spent: 10m
Work Description: boyuanzz closed pull request #6736: [BEAM-5781] Enable
ValidatesRunner tests for portable Java
URL: https://github.com/apache/beam/pull/6736
This is a PR merged from a forked repository.
As GitHub hides the original diff on merge, it is displayed below for
the sake of provenance:
As this is a foreign pull request (from a fork), the diff is supplied
below (as it won't show otherwise due to GitHub magic):
diff --git a/build.gradle b/build.gradle
index 488ac0ddf62..96d7f641c39 100644
--- a/build.gradle
+++ b/build.gradle
@@ -181,19 +181,20 @@ check.dependsOn rat
// This indirection also makes Jenkins use the branch of the PR
// for the test definitions.
task javaPreCommit() {
- dependsOn ":rat"
+// dependsOn ":rat"
// We need to list the model/* builds since sdks/java/core doesn't
// depend on any of the model.
- dependsOn ":beam-model-pipeline:build"
- dependsOn ":beam-model-job-management:build"
- dependsOn ":beam-model-fn-execution:build"
- dependsOn ":beam-runners-google-cloud-dataflow-java-legacy-worker:build"
- dependsOn ":beam-runners-google-cloud-dataflow-java-fn-api-worker:build"
- dependsOn ":beam-sdks-java-core:buildNeeded"
- dependsOn ":beam-sdks-java-core:buildDependents"
+// dependsOn ":beam-model-pipeline:build"
+// dependsOn ":beam-model-job-management:build"
+// dependsOn ":beam-model-fn-execution:build"
+// dependsOn ":beam-runners-google-cloud-dataflow-java-legacy-worker:build"
+// dependsOn ":beam-runners-google-cloud-dataflow-java-fn-api-worker:build"
+// dependsOn ":beam-sdks-java-core:buildNeeded"
+// dependsOn ":beam-sdks-java-core:buildDependents"
dependsOn ":beam-examples-java:preCommit"
- dependsOn ":beam-sdks-java-extensions-sql-jdbc:preCommit"
- dependsOn ":beam-sdks-java-javadoc:allJavadoc"
+ dependsOn
":beam-runners-google-cloud-dataflow-java:validatesRunnerTestFnApiWorker"
+// dependsOn ":beam-sdks-java-extensions-sql-jdbc:preCommit"
+// dependsOn ":beam-sdks-java-javadoc:allJavadoc"
}
task javaPostCommit() {
diff --git a/runners/google-cloud-dataflow-java/build.gradle
b/runners/google-cloud-dataflow-java/build.gradle
index ebe9dcd1344..2ac2d3f574a 100644
--- a/runners/google-cloud-dataflow-java/build.gradle
+++ b/runners/google-cloud-dataflow-java/build.gradle
@@ -32,6 +32,7 @@
evaluationDependsOn(":beam-sdks-java-io-google-cloud-platform")
evaluationDependsOn(":beam-sdks-java-core")
evaluationDependsOn(":beam-examples-java")
evaluationDependsOn(":beam-runners-google-cloud-dataflow-java-legacy-worker")
+evaluationDependsOn(":beam-runners-google-cloud-dataflow-java-fn-api-worker")
processResources {
filter org.apache.tools.ant.filters.ReplaceTokens, tokens: [
@@ -103,20 +104,40 @@ test {
systemProperties = [ "beamUseDummyRunner" : "true" ]
}
-// For the following test tasks, set workerHarnessContainerImage to empty to
make Dataflow pick up
-// the non-versioned container image, which handles a staged worker jar.
+def dataflowProject = project.findProperty('dataflowProject') ?:
'apache-beam-testing'
+def dataflowTempRoot = project.findProperty('dataflowTempRoot') ?:
'gs://temp-storage-for-validates-runner-tests/'
+def dataflowLegacyWorkerJar = project.findProperty('dataflowWorkerJar') ?:
project(":beam-runners-google-cloud-dataflow-java-legacy-worker").shadowJar.archivePath
+def dataflowFnApiWorkerJar = project.findProperty('dataflowWorkerJar') ?:
project(":beam-runners-google-cloud-dataflow-java-fn-api-worker").shadowJar.archivePath
+
+def dockerImageRoot = "us.gcr.io/${dataflowProject}/java-examples-it"
+def dockerImageContainer = "${dockerImageRoot}/java"
+def dockerTag = new Date().format('yyyyMMddHHmmss')
+
+def commonExcludeCategories = [
+ 'org.apache.beam.sdk.testing.LargeKeys$Above10MB',
+ 'org.apache.beam.sdk.testing.UsesAttemptedMetrics',
+ 'org.apache.beam.sdk.testing.UsesDistributionMetrics',
+ 'org.apache.beam.sdk.testing.UsesGaugeMetrics',
+ 'org.apache.beam.sdk.testing.UsesSetState',
+ 'org.apache.beam.sdk.testing.UsesMapState',
+ 'org.apache.beam.sdk.testing.UsesSplittableParDoWithWindowedSideInputs',
+ 'org.apache.beam.sdk.testing.UsesUnboundedPCollections',
+ 'org.apache.beam.sdk.testing.UsesTestStream',
+ 'org.apache.beam.sdk.testing.UsesParDoLifecycle',
+ 'org.apache.beam.sdk.testing.UsesMetricsPusher',
+]
+
+// For the following test tasks using legacy worker, set
workerHarnessContainerImage to empty to
+// make Dataflow pick up the non-versioned container image, which handles a
staged worker jar.
task validatesRunnerTest(type: Test) {
group = "Verification"
dependsOn ":beam-runners-google-cloud-dataflow-java-legacy-worker:shadowJar"
- def dataflowProject = project.findProperty('dataflowProject') ?:
'apache-beam-testing'
- def dataflowTempRoot = project.findProperty('dataflowTempRoot') ?:
'gs://temp-storage-for-validates-runner-tests/'
- def dataflowWorkerJar = project.findProperty('dataflowWorkerJar') ?:
project(":beam-runners-google-cloud-dataflow-java-legacy-worker").shadowJar.archivePath
systemProperty "beamTestPipelineOptions", JsonOutput.toJson([
"--runner=TestDataflowRunner",
"--project=${dataflowProject}",
"--tempRoot=${dataflowTempRoot}",
- "--dataflowWorkerJar=${dataflowWorkerJar}",
+ "--dataflowWorkerJar=${dataflowLegacyWorkerJar}",
"--workerHarnessContainerImage=",
])
@@ -128,37 +149,64 @@ task validatesRunnerTest(type: Test) {
testClassesDirs =
files(project(":beam-sdks-java-core").sourceSets.test.output.classesDirs)
useJUnit {
includeCategories 'org.apache.beam.sdk.testing.ValidatesRunner'
- excludeCategories 'org.apache.beam.sdk.testing.LargeKeys$Above10MB'
- excludeCategories 'org.apache.beam.sdk.testing.UsesAttemptedMetrics'
- excludeCategories 'org.apache.beam.sdk.testing.UsesDistributionMetrics'
- excludeCategories 'org.apache.beam.sdk.testing.UsesGaugeMetrics'
- excludeCategories 'org.apache.beam.sdk.testing.UsesSetState'
- excludeCategories 'org.apache.beam.sdk.testing.UsesMapState'
- excludeCategories
'org.apache.beam.sdk.testing.UsesSplittableParDoWithWindowedSideInputs'
- excludeCategories 'org.apache.beam.sdk.testing.UsesUnboundedPCollections'
- excludeCategories 'org.apache.beam.sdk.testing.UsesTestStream'
- excludeCategories 'org.apache.beam.sdk.testing.UsesParDoLifecycle'
- excludeCategories 'org.apache.beam.sdk.testing.UsesMetricsPusher'
+ commonExcludeCategories.each {
+ excludeCategories it
+ }
+ }
+}
+
+project.rootProject.ext.set("docker-repository-root", "${dockerImageRoot}")
+project.rootProject.ext.set("docker-tag","${dockerTag}")
+evaluationDependsOn(":beam-sdks-java-container")
+
+task buildAndPushDockerContainer(type:Exec) {
+ dependsOn ":beam-sdks-java-container:docker"
+ commandLine "gcloud", "docker", "--", "push", "${dockerImageContainer}"
+}
+
+
+task validatesRunnerTestFnApiWorker(type: Test) {
+ group = "Verification"
+ dependsOn ":beam-runners-google-cloud-dataflow-java-fn-api-worker:shadowJar"
+ dependsOn buildAndPushDockerContainer
+
+ systemProperty "beamTestPipelineOptions", JsonOutput.toJson([
+ "--runner=TestDataflowRunner",
+ "--project=${dataflowProject}",
+ "--tempRoot=${dataflowTempRoot}",
+ "--dataflowWorkerJar=${dataflowFnApiWorkerJar}",
+ "--workerHarnessContainerImage=${dockerImageContainer}:${dockerTag}",
+ "--experiments=beam_fn_api",
+ ])
+
+ // Increase test parallelism up to the number of Gradle workers. By default
this is equal
+ // to the number of CPU cores, but can be increased by setting
--max-workers=N.
+ maxParallelForks Integer.MAX_VALUE
+ classpath = configurations.validatesRunner
+ testClassesDirs =
files(project(":beam-sdks-java-core").sourceSets.test.output.classesDirs)
+ useJUnit {
+ includeCategories 'org.apache.beam.sdk.testing.ValidatesRunner'
+ commonExcludeCategories.each {
+ excludeCategories it
+ }
}
}
task validatesRunner {
group = "Verification"
description "Validates Dataflow runner"
- dependsOn validatesRunnerTest
+// dependsOn validatesRunnerTest
+ dependsOn validatesRunnerTestFnApiWorker
}
task googleCloudPlatformIntegrationTest(type: Test) {
group = "Verification"
dependsOn ":beam-runners-google-cloud-dataflow-java-legacy-worker:shadowJar"
- def dataflowProject = project.findProperty('dataflowProject') ?:
'apache-beam-testing'
- def dataflowTempRoot = project.findProperty('dataflowTempRoot') ?:
'gs://temp-storage-for-end-to-end-tests'
- def dataflowWorkerJar = project.findProperty('dataflowWorkerJar') ?:
project(":beam-runners-google-cloud-dataflow-java-legacy-worker").shadowJar.archivePath
systemProperty "beamTestPipelineOptions", JsonOutput.toJson([
"--runner=TestDataflowRunner",
"--project=${dataflowProject}",
"--tempRoot=${dataflowTempRoot}",
- "--dataflowWorkerJar=${dataflowWorkerJar}",
+ "--dataflowWorkerJar=${dataflowLegacyWorkerJar}",
"--workerHarnessContainerImage=",
])
@@ -174,15 +222,12 @@ task googleCloudPlatformIntegrationTest(type: Test) {
task examplesJavaIntegrationTest(type: Test) {
group = "Verification"
dependsOn ":beam-runners-google-cloud-dataflow-java-legacy-worker:shadowJar"
- def dataflowProject = project.findProperty('dataflowProject') ?:
'apache-beam-testing'
- def dataflowTempRoot = project.findProperty('dataflowTempRoot') ?:
'gs://temp-storage-for-end-to-end-tests'
- def dataflowWorkerJar = project.findProperty('dataflowWorkerJar') ?:
project(":beam-runners-google-cloud-dataflow-java-legacy-worker").shadowJar.archivePath
systemProperty "beamTestPipelineOptions", JsonOutput.toJson([
"--runner=TestDataflowRunner",
"--project=${dataflowProject}",
"--tempRoot=${dataflowTempRoot}",
- "--dataflowWorkerJar=${dataflowWorkerJar}",
+ "--dataflowWorkerJar=${dataflowLegacyWorkerJar}",
"--workerHarnessContainerImage=",
])
@@ -200,15 +245,12 @@ task examplesJavaIntegrationTest(type: Test) {
task coreSDKJavaIntegrationTest(type: Test) {
group = "Verification"
dependsOn ":beam-runners-google-cloud-dataflow-java-legacy-worker:shadowJar"
- def dataflowProject = project.findProperty('dataflowProject') ?:
'apache-beam-testing'
- def dataflowTempRoot = project.findProperty('dataflowTempRoot') ?:
'gs://temp-storage-for-end-to-end-tests'
- def dataflowWorkerJar = project.findProperty('dataflowWorkerJar') ?:
project(":beam-runners-google-cloud-dataflow-java-legacy-worker").shadowJar.archivePath
systemProperty "beamTestPipelineOptions", JsonOutput.toJson([
"--runner=TestDataflowRunner",
"--project=${dataflowProject}",
"--tempRoot=${dataflowTempRoot}",
- "--dataflowWorkerJar=${dataflowWorkerJar}",
+ "--dataflowWorkerJar=${dataflowLegacyWorkerJar}",
"--workerHarnessContainerImage=",
])
diff --git a/runners/google-cloud-dataflow-java/examples/build.gradle
b/runners/google-cloud-dataflow-java/examples/build.gradle
index 6009ff9f284..18d86b8e555 100644
--- a/runners/google-cloud-dataflow-java/examples/build.gradle
+++ b/runners/google-cloud-dataflow-java/examples/build.gradle
@@ -24,6 +24,7 @@ applyJavaNature(publish: false)
// its sourceSets.test.output directly.
evaluationDependsOn(":beam-examples-java")
evaluationDependsOn(":beam-runners-google-cloud-dataflow-java-legacy-worker")
+evaluationDependsOn(":beam-runners-google-cloud-dataflow-java-fn-api-worker")
configurations { dataflowRunnerPreCommit }
@@ -35,8 +36,11 @@ dependencies {
def gcpProject = project.findProperty('gcpProject') ?: 'apache-beam-testing'
def gcsTempRoot = project.findProperty('gcsTempRoot') ?:
'gs://temp-storage-for-end-to-end-tests/'
+def dockerImageRoot = "us.gcr.io/${gcpProject}/java-examples-it"
+def dockerImageContainer = "${dockerImageRoot}/java"
+def dockerTag = new Date().format('yyyyMMddHHmmss')
-task preCommit(type: Test) {
+task preCommitLegacyWorker(type: Test) {
dependsOn ":beam-runners-google-cloud-dataflow-java-legacy-worker:shadowJar"
def dataflowWorkerJar = project.findProperty('dataflowWorkerJar') ?:
project(":beam-runners-google-cloud-dataflow-java-legacy-worker").shadowJar.archivePath
@@ -56,3 +60,41 @@ task preCommit(type: Test) {
maxParallelForks 4
systemProperty "beamTestPipelineOptions",
JsonOutput.toJson(preCommitBeamTestPipelineOptions)
}
+
+project.rootProject.ext.set("docker-repository-root", "${dockerImageRoot}")
+project.rootProject.ext.set("docker-tag","${dockerTag}")
+evaluationDependsOn(":beam-sdks-java-container")
+
+task buildAndPushDockerContainer(type:Exec) {
+ dependsOn ":beam-sdks-java-container:docker"
+ commandLine "gcloud", "docker", "--", "push", "${dockerImageContainer}"
+}
+
+task preCommitFnApiWorker(type: Test) {
+ dependsOn ":beam-runners-google-cloud-dataflow-java-fn-api-worker:shadowJar"
+ dependsOn buildAndPushDockerContainer
+
+ def dataflowWorkerJar = project.findProperty('dataflowWorkerJar') ?:
project(":beam-runners-google-cloud-dataflow-java-fn-api-worker").shadowJar.archivePath
+ def preCommitBeamTestPipelineOptions = [
+ "--project=${gcpProject}",
+ "--tempRoot=${gcsTempRoot}",
+ "--runner=TestDataflowRunner",
+ "--dataflowWorkerJar=${dataflowWorkerJar}",
+ "--workerHarnessContainerImage=${dockerImageContainer}:${dockerTag}",
+ "--experiments=beam_fn_api",
+ ]
+ testClassesDirs =
files(project(":beam-examples-java").sourceSets.test.output.classesDirs)
+ include "**/WordCountIT.class"
+ include "**/WindowedWordCountIT.class"
+ forkEvery 1
+ maxParallelForks 4
+ systemProperty "beamTestPipelineOptions",
JsonOutput.toJson(preCommitBeamTestPipelineOptions)
+ useJUnit {
+ excludeCategories 'org.apache.beam.sdk.testing.StreamingIT'
+ }
+}
+
+task preCommit() {
+ dependsOn preCommitLegacyWorker
+ dependsOn preCommitFnApiWorker
+}
\ No newline at end of file
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
us...@infra.apache.org
Issue Time Tracking
-------------------
Worklog Id: (was: 156500)
Time Spent: 20m (was: 10m)
> Enable ValidatesRunner tests for portable Java
> ----------------------------------------------
>
> Key: BEAM-5781
> URL: https://issues.apache.org/jira/browse/BEAM-5781
> Project: Beam
> Issue Type: Test
> Components: runner-dataflow
> Reporter: Boyuan Zhang
> Assignee: Boyuan Zhang
> Priority: Major
> Time Spent: 20m
> Remaining Estimate: 0h
>
--
This message was sent by Atlassian JIRA
(v7.6.3#76005)
