This is an automated email from the ASF dual-hosted git repository.
markliu pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/beam.git
The following commit(s) were added to refs/heads/master by this push:
new 0e60dfa Merge pull request #9183: [BEAM-7675] Unify Gradle test
configuration across Py2 and Py3 (#9183)
0e60dfa is described below
commit 0e60dfacacfaf6deb2ea81ee3deb58a96dcbc9e6
Author: Mark Liu <mark...@apache.org>
AuthorDate: Tue Jul 30 09:11:50 2019 -0700
Merge pull request #9183: [BEAM-7675] Unify Gradle test configuration
across Py2 and Py3 (#9183)
---
.../jenkins/job_PerformanceTests_Python.groovy | 2 +-
.../job_PostCommit_Python_MongoDBIO_IT.groovy | 2 +-
...stCommit_Python_ValidatesRunner_Dataflow.groovy | 4 +-
..._PostCommit_Python_ValidatesRunner_Spark.groovy | 2 +-
...b_PreCommit_Python_ValidatesRunner_Flink.groovy | 2 +-
build.gradle | 14 +-
.../apache_beam/io/gcp/gcsio_integration_test.py | 2 +-
sdks/python/build.gradle | 406 +--------------------
sdks/python/scripts/run_integration_test.sh | 8 +-
sdks/python/test-suites/dataflow/build.gradle | 53 ---
sdks/python/test-suites/dataflow/py2/build.gradle | 116 ++++++
sdks/python/test-suites/direct/py2/build.gradle | 114 ++++++
sdks/python/test-suites/portable/py2/build.gradle | 235 ++++++++++++
sdks/python/test-suites/portable/py35/build.gradle | 2 +-
sdks/python/test-suites/tox/py2/build.gradle | 59 +++
settings.gradle | 5 +-
16 files changed, 550 insertions(+), 476 deletions(-)
diff --git a/.test-infra/jenkins/job_PerformanceTests_Python.groovy
b/.test-infra/jenkins/job_PerformanceTests_Python.groovy
index 0b0fb02..686af9b 100644
--- a/.test-infra/jenkins/job_PerformanceTests_Python.groovy
+++ b/.test-infra/jenkins/job_PerformanceTests_Python.groovy
@@ -66,7 +66,7 @@ def testConfigurations = [
jobTriggerPhrase : 'Run Python27 WordCountIT Performance Test',
resultTable : 'beam_performance.wordcount_py27_pkb_results',
itClass :
'apache_beam.examples.wordcount_it_test:WordCountIT.test_wordcount_it',
- itModule : ':sdks:python',
+ itModule : ':sdks:python:test-suites:dataflow:py2',
extraPipelineArgs : dataflowPipelineArgs + [
input:
'gs://apache-beam-samples/input_small_files/ascii_sort_1MB_input.0000*', // 1Gb
output:
'gs://temp-storage-for-end-to-end-tests/py-it-cloud/output',
diff --git a/.test-infra/jenkins/job_PostCommit_Python_MongoDBIO_IT.groovy
b/.test-infra/jenkins/job_PostCommit_Python_MongoDBIO_IT.groovy
index b09acc1..fdf3caa 100644
--- a/.test-infra/jenkins/job_PostCommit_Python_MongoDBIO_IT.groovy
+++ b/.test-infra/jenkins/job_PostCommit_Python_MongoDBIO_IT.groovy
@@ -31,7 +31,7 @@
PostcommitJobBuilder.postCommitJob('beam_PostCommit_Python_MongoDBIO_IT',
steps {
gradle {
rootBuildScriptDir(commonJobProperties.checkoutDir)
- tasks(':sdks:python:mongodbioIT')
+ tasks(':sdks:python:test-suites:direct:py2:mongodbioIT')
commonJobProperties.setGradleSwitches(delegate)
}
}
diff --git
a/.test-infra/jenkins/job_PostCommit_Python_ValidatesRunner_Dataflow.groovy
b/.test-infra/jenkins/job_PostCommit_Python_ValidatesRunner_Dataflow.groovy
index 39045eb..f7fd557 100644
--- a/.test-infra/jenkins/job_PostCommit_Python_ValidatesRunner_Dataflow.groovy
+++ b/.test-infra/jenkins/job_PostCommit_Python_ValidatesRunner_Dataflow.groovy
@@ -36,8 +36,8 @@
PostcommitJobBuilder.postCommitJob('beam_PostCommit_Py_VR_Dataflow', 'Run Python
steps {
gradle {
rootBuildScriptDir(commonJobProperties.checkoutDir)
- tasks(':sdks:python:validatesRunnerBatchTests')
- tasks(':sdks:python:validatesRunnerStreamingTests')
+ tasks(':sdks:python:test-suites:dataflow:py2:validatesRunnerBatchTests')
+
tasks(':sdks:python:test-suites:dataflow:py2:validatesRunnerStreamingTests')
tasks(':sdks:python:test-suites:dataflow:py35:validatesRunnerBatchTests')
tasks(':sdks:python:test-suites:dataflow:py36:validatesRunnerBatchTests')
tasks(':sdks:python:test-suites:dataflow:py37:validatesRunnerBatchTests')
diff --git
a/.test-infra/jenkins/job_PostCommit_Python_ValidatesRunner_Spark.groovy
b/.test-infra/jenkins/job_PostCommit_Python_ValidatesRunner_Spark.groovy
index cd69560..59b45f7 100644
--- a/.test-infra/jenkins/job_PostCommit_Python_ValidatesRunner_Spark.groovy
+++ b/.test-infra/jenkins/job_PostCommit_Python_ValidatesRunner_Spark.groovy
@@ -31,7 +31,7 @@
PostcommitJobBuilder.postCommitJob('beam_PostCommit_Python_VR_Spark',
steps {
gradle {
rootBuildScriptDir(commonJobProperties.checkoutDir)
- tasks(':sdks:python:sparkValidatesRunner')
+ tasks(':sdks:python:test-suites:portable:py2:sparkValidatesRunner')
commonJobProperties.setGradleSwitches(delegate)
}
}
diff --git
a/.test-infra/jenkins/job_PreCommit_Python_ValidatesRunner_Flink.groovy
b/.test-infra/jenkins/job_PreCommit_Python_ValidatesRunner_Flink.groovy
index 3df3c97..eb34f1e 100644
--- a/.test-infra/jenkins/job_PreCommit_Python_ValidatesRunner_Flink.groovy
+++ b/.test-infra/jenkins/job_PreCommit_Python_ValidatesRunner_Flink.groovy
@@ -23,7 +23,7 @@ import PrecommitJobBuilder
PrecommitJobBuilder builder = new PrecommitJobBuilder(
scope: this,
nameBase: 'Python_PVR_Flink',
- gradleTask: ':sdks:python:flinkValidatesRunner',
+ gradleTask: ':sdks:python:test-suites:portable:py2:flinkValidatesRunner',
triggerPathPatterns: [
'^model/.*$',
'^runners/core-construction-java/.*$',
diff --git a/build.gradle b/build.gradle
index 24507fa..7189fc9 100644
--- a/build.gradle
+++ b/build.gradle
@@ -192,11 +192,11 @@ task goIntegrationTests() {
}
task pythonPreCommit() {
- dependsOn ":sdks:python:preCommitPy2"
+ dependsOn ":sdks:python:test-suites:tox:py2:preCommitPy2"
dependsOn ":sdks:python:test-suites:tox:py35:preCommitPy35"
dependsOn ":sdks:python:test-suites:tox:py36:preCommitPy36"
dependsOn ":sdks:python:test-suites:tox:py37:preCommitPy37"
- dependsOn ":sdks:python:test-suites:dataflow:preCommitIT"
+ dependsOn ":sdks:python:test-suites:dataflow:py2:preCommitIT"
dependsOn ":sdks:python:test-suites:dataflow:py37:preCommitIT"
// We don't include Py35, Py36 precommit ITs to reduce quota footprint.
// We can reconsider if we ever see an issue that these suites would
@@ -204,7 +204,11 @@ task pythonPreCommit() {
}
task python2PostCommit() {
- dependsOn ":sdks:python:postCommit"
+ dependsOn ":sdks:python:test-suites:portable:py2:crossLanguageTests"
+ dependsOn ":sdks:python:test-suites:dataflow:py2:postCommitIT"
+ dependsOn ":sdks:python:test-suites:direct:py2:directRunnerIT"
+ dependsOn ":sdks:python:test-suites:direct:py2:hdfsIntegrationTest"
+ dependsOn ":sdks:python:test-suites:direct:py2:mongodbioIT"
}
task python35PostCommit() {
@@ -224,8 +228,8 @@ task python37PostCommit() {
}
task portablePythonPreCommit() {
- dependsOn "sdks:python:portablePreCommitPy2"
- dependsOn "sdks:python:test-suites:portable:py35:portablePreCommitPy35"
+ dependsOn ":sdks:python:test-suites:portable:py2:preCommitPy2"
+ dependsOn ":sdks:python:test-suites:portable:py35:preCommitPy35"
}
task websitePreCommit() {
diff --git a/sdks/python/apache_beam/io/gcp/gcsio_integration_test.py
b/sdks/python/apache_beam/io/gcp/gcsio_integration_test.py
index 26c8655..d4e387b 100644
--- a/sdks/python/apache_beam/io/gcp/gcsio_integration_test.py
+++ b/sdks/python/apache_beam/io/gcp/gcsio_integration_test.py
@@ -31,7 +31,7 @@ The project's Cloud Storage service account requires
Encrypter/Decrypter
permissions for the key specified in --kms_key_name.
To run these tests manually:
- ./gradlew :sdks:python:integrationTest \
+ ./gradlew :sdks:python:test-suites:dataflow:integrationTest \
-Dtests=apache_beam.io.gcp.gcsio_integration_test:GcsIOIntegrationTest \
-DkmsKeyName=KMS_KEY_NAME
"""
diff --git a/sdks/python/build.gradle b/sdks/python/build.gradle
index e0411fe..5fc3740 100644
--- a/sdks/python/build.gradle
+++ b/sdks/python/build.gradle
@@ -16,11 +16,8 @@
* limitations under the License.
*/
-import org.apache.tools.ant.taskdefs.condition.Os
-
plugins { id 'org.apache.beam.module' }
applyPythonNature()
-enablePythonPerformanceTest()
/*************************************************************************************************/
@@ -39,299 +36,7 @@ build.dependsOn buildPython
/*************************************************************************************************/
-// Unit tests for Python 2
-// See Python 3 tests in test-suites/tox
-
-task lint {}
-check.dependsOn lint
-
-toxTask "lintPy27", "py27-lint"
-lint.dependsOn lintPy27
-
-toxTask "lintPy27_3", "py27-lint3"
-lint.dependsOn lintPy27_3
-
-toxTask "testPy2Gcp", "py27-gcp"
-test.dependsOn testPy2Gcp
-
-toxTask "testPython2", "py27"
-test.dependsOn testPython2
-
-toxTask "testPy2Cython", "py27-cython"
-test.dependsOn testPy2Cython
-// Ensure that testPy2Cython runs exclusively to other tests. This line is not
-// actually required, since gradle doesn't do parallel execution within a
-// project.
-testPy2Cython.mustRunAfter testPython2, testPy2Gcp
-
-toxTask "docs", "docs"
-assemble.dependsOn docs
-
-toxTask "cover", "cover"
-
-task preCommitPy2() {
- dependsOn "docs"
- dependsOn "testPy2Cython"
- dependsOn "testPython2"
- dependsOn "testPy2Gcp"
- dependsOn "lint"
-}
-
-addPortableWordCountTasks()
-
-task portablePreCommitPy2() {
- dependsOn ':runners:flink:1.5:job-server-container:docker'
- dependsOn ':sdks:python:container:docker'
- dependsOn portableWordCountBatch
- dependsOn portableWordCountStreaming
-}
-
-
-/*************************************************************************************************/
-// E2E integration testing and validates runner testing
-
-// Basic test options for ITs running on Jenkins.
-def basicTestOpts = [
- "--nocapture", // print stdout instantly
- "--processes=8", // run tests in parallel
- "--process-timeout=4500", // timeout of whole command execution
-]
-
-task directRunnerIT(dependsOn: 'installGcpTest') {
- // Run IT tests with TestDirectRunner in batch.
- doLast {
- def tests = [
- "apache_beam.examples.wordcount_it_test:WordCountIT.test_wordcount_it",
- "apache_beam.io.gcp.pubsub_integration_test:PubSubIntegrationTest",
-
"apache_beam.io.gcp.big_query_query_to_table_it_test:BigQueryQueryToTableIT",
- "apache_beam.io.gcp.bigquery_io_read_it_test",
- "apache_beam.io.gcp.datastore.v1new.datastore_write_it_test",
- ]
- def batchTestOpts = basicTestOpts + ["--tests=${tests.join(',')}"]
- def argMap = ["runner": "TestDirectRunner",
- "test_opts": batchTestOpts,
- "suite": "directRunnerIT-batch"]
- def batchCmdArgs = project.mapToArgString(argMap)
- exec {
- executable 'sh'
- args '-c', ". ${project.ext.envdir}/bin/activate &&
./scripts/run_integration_test.sh $batchCmdArgs"
- }
- }
-
- // Run IT tests with TestDirectRunner in streaming.
- doLast {
- def tests = [
- "apache_beam.examples.wordcount_it_test:WordCountIT.test_wordcount_it",
- "apache_beam.io.gcp.pubsub_integration_test:PubSubIntegrationTest",
-
"apache_beam.io.gcp.bigquery_test:BigQueryStreamingInsertTransformIntegrationTests\
-.test_multiple_destinations_transform",
-
"apache_beam.io.gcp.bigquery_file_loads_test:BigQueryFileLoadsIT.test_bqfl_streaming",
- ]
- def streamingTestOpts = basicTestOpts + ["--tests=${tests.join(',')}"]
- def argMap = ["runner": "TestDirectRunner",
- "streaming": "true",
- "test_opts": streamingTestOpts,
- "suite": "directRunnerIT-streaming"]
- def streamingCmdArgs = project.mapToArgString(argMap)
- exec {
- executable 'sh'
- args '-c', ". ${project.ext.envdir}/bin/activate &&
./scripts/run_integration_test.sh $streamingCmdArgs"
- }
- }
-}
-
-// Before running this, you need to:
-//
-// 1. Build the SDK container:
-//
-// ./gradlew -p sdks/python/container docker
-//
-// 2. Either a) or b)
-// a) If you want the Job Server to run in a Docker container:
-//
-// ./gradlew :runners:flink:1.5:job-server-container:docker
-//
-// b) Otherwise, start a local JobService, for example, the Portable Flink
runner
-// (in a separate shell since it continues to run):
-//
-// ./gradlew :runners:flink:1.5:job-server:runShadow
-//
-// Then you can run this example:
-//
-// Docker (2a):
-//
-// ./gradlew :sdks:python:portableWordCount
-//
-// Local JobService (2b):
-//
-// ./gradlew :sdks:python:portableWordCount -PjobEndpoint=localhost:8099
-//
-
-task portableWordCount {
- dependsOn project.hasProperty("streaming") ? portableWordCountStreaming :
portableWordCountBatch
-}
-
-// Run PostCommit integration tests on default runner (TestDataflowRunner)
-task postCommitIT(dependsOn: ['installGcpTest', 'sdist']) {
- dependsOn ":runners:google-cloud-dataflow-java:worker:shadowJar"
-
- def dataflowWorkerJar =
project(":runners:google-cloud-dataflow-java:worker").shadowJar.archivePath
-
- doLast {
- def testOpts = basicTestOpts + ["--attr=IT"]
- def cmdArgs = project.mapToArgString(["test_opts": testOpts,
- "worker_jar": dataflowWorkerJar,
- "suite": "postCommitIT-df"])
- exec {
- executable 'sh'
- args '-c', ". ${project.ext.envdir}/bin/activate &&
./scripts/run_integration_test.sh $cmdArgs"
- }
- }
-}
-
-task validatesRunnerBatchTests(dependsOn: ['installGcpTest', 'sdist']) {
- dependsOn ":runners:google-cloud-dataflow-java:worker:shadowJar"
-
- def dataflowWorkerJar =
project(":runners:google-cloud-dataflow-java:worker").shadowJar.archivePath
-
- doLast {
- def testOpts = basicTestOpts + ["--attr=ValidatesRunner"]
- def cmdArgs = project.mapToArgString(["test_opts": testOpts,
- "worker_jar": dataflowWorkerJar,
- "suite":
"validatesRunnerBatchTests-df"])
- exec {
- executable 'sh'
- args '-c', ". ${project.ext.envdir}/bin/activate &&
./scripts/run_integration_test.sh $cmdArgs"
- }
- }
-}
-
-task validatesRunnerStreamingTests(dependsOn: ['installGcpTest', 'sdist']) {
- dependsOn ":runners:google-cloud-dataflow-java:worker:shadowJar"
-
- def dataflowWorkerJar =
project(":runners:google-cloud-dataflow-java:worker").shadowJar.archivePath
-
- doLast {
- // TODO(BEAM-3544,BEAM-5025): Disable tests with 'sickbay-streaming' tag.
- def testOpts = basicTestOpts +
["--attr=ValidatesRunner,!sickbay-streaming"]
- def argMap = ["test_opts": testOpts,
- "streaming": "true",
- "worker_jar": dataflowWorkerJar,
- "suite": "validatesRunnerStreamingTests-df"]
- def cmdArgs = project.mapToArgString(argMap)
- exec {
- executable 'sh'
- args '-c', ". ${project.ext.envdir}/bin/activate &&
./scripts/run_integration_test.sh $cmdArgs"
- }
- }
-}
-
-task hdfsIntegrationTest(dependsOn: 'installGcpTest') {
- doLast {
- exec {
- executable 'sh'
- args '-c', ". ${project.ext.envdir}/bin/activate &&
./apache_beam/io/hdfs_integration_test/hdfs_integration_test.sh python:2"
- }
- }
-}
-
-task sparkValidatesRunner() {
- dependsOn 'createProcessWorker'
- dependsOn 'setupVirtualenv'
- dependsOn ':runners:spark:job-server:shadowJar'
- doLast {
- def environment_config = "'{\"command\":
\"${project(":sdks:python:").buildDir.absolutePath}/sdk_worker.sh\"}'"
- def argMap = [
- "environment_type" : "PROCESS",
- "spark_job_server_jar":
project(":runners:spark:job-server:").shadowJar.archivePath,
- "environment_config": environment_config,
- ]
- def argString = project.mapToArgString(argMap)
-
- // Optionally specify test function names separated by space e.g.:
- // ./gradlew :sdks:python:sparkValidatesRunner
-Ptests="test_external_transforms test_read"
- // Otherwise run all test functions under SparkRunnerTest
- def tests = project.hasProperty('tests') ?
- project.property('tests').split().collect{ "SparkRunnerTest.$it"
}.join(' ') : ''
-
- exec {
- executable 'sh'
- args '-c', ". ${project.ext.envdir}/bin/activate && pip install -e
.[test] && python -m apache_beam.runners.portability.spark_runner_test $tests
$argString"
- }
- }
-}
-
-class CompatibilityMatrixConfig {
- // Execute batch or streaming pipelines.
- boolean streaming = false
- // Execute on Docker or Process based environment.
- SDK_WORKER_TYPE workerType = SDK_WORKER_TYPE.DOCKER
-
- enum SDK_WORKER_TYPE {
- DOCKER, PROCESS, LOOPBACK
- }
-
- // Whether to pre-optimize the pipeline with the Python optimizer.
- boolean preOptimize = false
-}
-
-def flinkCompatibilityMatrix = {
- def config = it ? it as CompatibilityMatrixConfig : new
CompatibilityMatrixConfig()
- def workerType = config.workerType.name()
- def streaming = config.streaming
- def environment_config = config.workerType ==
CompatibilityMatrixConfig.SDK_WORKER_TYPE.PROCESS ?
"--environment_config='{\"command\":
\"${project(":sdks:python").buildDir.absolutePath}/sdk_worker.sh\"}'" : ""
- def name = "flinkCompatibilityMatrix${streaming ? 'Streaming' :
'Batch'}${config.preOptimize ? 'PreOptimize' : ''}${workerType}"
- def extra_experiments = []
- if (config.preOptimize)
- extra_experiments.add('pre_optimize=all')
- tasks.create(name: name) {
- dependsOn 'setupVirtualenv'
- dependsOn ':runners:flink:1.5:job-server:shadowJar'
- dependsOn ':sdks:java:container:docker'
- if (workerType.toLowerCase() == 'docker')
- dependsOn ':sdks:python:container:docker'
- else if (workerType.toLowerCase() == 'process')
- dependsOn 'createProcessWorker'
- doLast {
- exec {
- executable 'sh'
- args '-c', ". ${project.ext.envdir}/bin/activate && pip install -e
.[test] && python -m apache_beam.runners.portability.flink_runner_test
--flink_job_server_jar=${project(":runners:flink:1.5:job-server:").shadowJar.archivePath}
--environment_type=${workerType} ${environment_config} ${streaming ?
'--streaming' : ''} ${extra_experiments ? '--extra_experiments=' +
extra_experiments.join(',') : ''}"
- }
- }
- }
-}
-
-task flinkCompatibilityMatrixDocker() {
- dependsOn flinkCompatibilityMatrix(streaming: false)
- dependsOn flinkCompatibilityMatrix(streaming: true)
-}
-
-task flinkCompatibilityMatrixProcess() {
- dependsOn flinkCompatibilityMatrix(streaming: false, workerType:
CompatibilityMatrixConfig.SDK_WORKER_TYPE.PROCESS)
- dependsOn flinkCompatibilityMatrix(streaming: true, workerType:
CompatibilityMatrixConfig.SDK_WORKER_TYPE.PROCESS)
-}
-
-task flinkCompatibilityMatrixLoopback() {
- dependsOn flinkCompatibilityMatrix(streaming: false, workerType:
CompatibilityMatrixConfig.SDK_WORKER_TYPE.LOOPBACK)
- dependsOn flinkCompatibilityMatrix(streaming: true, workerType:
CompatibilityMatrixConfig.SDK_WORKER_TYPE.LOOPBACK)
- dependsOn flinkCompatibilityMatrix(streaming: true, workerType:
CompatibilityMatrixConfig.SDK_WORKER_TYPE.LOOPBACK, preOptimize: true)
-}
-
-task flinkValidatesRunner() {
- dependsOn 'flinkCompatibilityMatrixLoopback'
-}
-
-task postCommit() {
- dependsOn "crossLanguageTests"
- dependsOn "directRunnerIT"
- dependsOn "hdfsIntegrationTest"
- dependsOn "postCommitIT"
- dependsOn "mongodbioIT"
-}
-
-
-/*************************************************************************************************/
-// Other build and analysis tasks
+// Non-testing builds and analysis tasks
// Snapshot of dependency requirements defined in setup.py.
// Results will be stored in files under Gradle build directory.
@@ -359,112 +64,3 @@ task buildSnapshot() {
dependsOn 'sdist'
dependsOn 'depSnapshot'
}
-
-project.task('createProcessWorker') {
- dependsOn ':sdks:python:container:build'
- dependsOn 'setupVirtualenv'
- def sdkWorkerFile = file("${project.buildDir}/sdk_worker.sh")
- def osType = 'linux'
- if (Os.isFamily(Os.FAMILY_MAC))
- osType = 'darwin'
- def workerScript =
"${project(":sdks:python:container:").buildDir.absolutePath}/target/launcher/${osType}_amd64/boot"
- def sdkWorkerFileCode = "sh -c \"pip=`which pip` .
${project.ext.envdir}/bin/activate && ${workerScript} \$* \""
- outputs.file sdkWorkerFile
- doLast {
- sdkWorkerFile.write sdkWorkerFileCode
- exec {
- commandLine('sh', '-c', ". ${project.ext.envdir}/bin/activate && cd
${project.projectDir} && python setup.py install ")
- }
- exec {
- commandLine('chmod', '+x', sdkWorkerFile)
- }
- }
-}
-
-project.task('crossLanguagePythonJavaFlink') {
- dependsOn 'setupVirtualenv'
- dependsOn ':runners:flink:1.5:job-server-container:docker'
- dependsOn ':sdks:python:container:docker'
- dependsOn ':sdks:java:container:docker'
- dependsOn ':runners:core-construction-java:buildTestExpansionServiceJar'
-
- doLast {
- def testServiceExpansionJar =
project(":runners:core-construction-java:").buildTestExpansionServiceJar.archivePath
- def options = [
- "--runner=PortableRunner",
- "--experiments=worker_threads=100",
- "--parallelism=2",
- "--shutdown_sources_on_final_watermark",
- "--environment_cache_millis=10000",
- "--expansion_service_port=8096",
- "--expansion_service_jar=${testServiceExpansionJar}",
- ]
- exec {
- executable 'sh'
- args '-c', ". ${project.ext.envdir}/bin/activate && pip install -e
.[test] && python -m apache_beam.transforms.external_test ${options.join(' ')}"
- }
- }
-}
-
-project.task('crossLanguagePortableWordCount') {
- dependsOn 'setupVirtualenv'
- dependsOn ':runners:flink:1.5:job-server-container:docker'
- dependsOn ':sdks:python:container:docker'
- dependsOn ':sdks:java:container:docker'
- dependsOn ':runners:core-construction-java:buildTestExpansionServiceJar'
-
- doLast {
- def testServiceExpansionJar =
project(":runners:core-construction-java:").buildTestExpansionServiceJar.archivePath
- def options = [
- "--input=/etc/profile",
- "--output=/tmp/py-wordcount-portable",
- "--runner=PortableRunner",
- "--experiments=worker_threads=100",
- "--parallelism=2",
- "--shutdown_sources_on_final_watermark",
- "--environment_cache_millis=10000",
- "--expansion_service_jar=${testServiceExpansionJar}",
- ]
- exec {
- executable 'sh'
- args '-c', ". ${project.ext.envdir}/bin/activate && pip install -e
.[test] && python -m apache_beam.examples.wordcount_xlang ${options.join(' ')}"
- // TODO: Check that the output file is generated and runs.
- }
- }
-}
-
-project.task('crossLanguageTests') {
- dependsOn "crossLanguagePythonJavaFlink"
- dependsOn "crossLanguagePortableWordCount"
-}
-
-project.task('mongodbioIT') {
- dependsOn 'setupVirtualenv'
-
- Random r = new Random()
- def port = r.nextInt(1000) + 27017
- def containerName = "mongoioit" + port
-
- def options = [
- "--mongo_uri=mongodb://localhost:" + port
- ]
-
- // Pull the latest mongodb docker image and run
- doFirst {
- exec {
- executable 'sh'
- args '-c', "docker pull mongo && docker run --name ${containerName} -p
${port}:27017 -d mongo:latest"
- }
- }
-
- doLast {
- exec {
- executable 'sh'
- args '-c', ". ${project.ext.envdir}/bin/activate && pip install -e
.[test] && python -m apache_beam.io.mongodbio_it_test ${options.join(' ')}"
- }
- exec {
- executable 'sh'
- args '-c', "docker stop ${containerName} && docker rm ${containerName}"
- }
- }
-}
diff --git a/sdks/python/scripts/run_integration_test.sh
b/sdks/python/scripts/run_integration_test.sh
index f87b8b8..b5f0c60 100755
--- a/sdks/python/scripts/run_integration_test.sh
+++ b/sdks/python/scripts/run_integration_test.sh
@@ -18,10 +18,10 @@
###########################################################################
#
-# This script is useful to run single or a set of Python integration tests
-# manually or through Gradle. Note, this script doesn't setup python
-# environment which is required before running tests. Use Gradle task
-# `:sdks:python:integrationTests` to do both together.
+# This script is used in Gradle to run single or a set of Python integration
tests
+# locally or on Jenkins. Note, this script doesn't setup python environment
which is
+# required for integration test. In order to do so, run Gradle tasks defined in
+# :sdks:python:test-suites instead.
#
# In order to run test with customer options, use following commandline flags:
#
diff --git a/sdks/python/test-suites/dataflow/build.gradle
b/sdks/python/test-suites/dataflow/build.gradle
deleted file mode 100644
index 8a395a9..0000000
--- a/sdks/python/test-suites/dataflow/build.gradle
+++ /dev/null
@@ -1,53 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * License); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an AS IS BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-plugins { id 'org.apache.beam.module' }
-applyPythonNature()
-
-def runScriptsDir = "${project.rootDir}/sdks/python/scripts"
-
-task preCommitIT(dependsOn: ['sdist', 'installGcpTest']) {
- dependsOn ":runners:google-cloud-dataflow-java:worker:shadowJar"
-
- def dataflowWorkerJar =
project(":runners:google-cloud-dataflow-java:worker").shadowJar.archivePath
-
- doLast {
- // Basic integration tests to run in PreCommit
- def precommitTests = [
- "apache_beam.examples.wordcount_it_test:WordCountIT.test_wordcount_it",
-
"apache_beam.examples.streaming_wordcount_it_test:StreamingWordCountIT.test_streaming_wordcount_it",
- ]
- def testOpts = [
- "--tests=${precommitTests.join(',')}",
- "--nocapture", // Print stdout instantly
- "--processes=2", // Number of tests running in parallel
- "--process-timeout=1800", // Timeout of whole command execution
- ]
- def cmdArgs = project.mapToArgString([
- "test_opts": testOpts,
- "sdk_location": "${project.buildDir}/apache-beam.tar.gz",
- "worker_jar": dataflowWorkerJar,
- "suite": "preCommitIT-df"
- ])
-
- exec {
- executable 'sh'
- args '-c', ". ${project.ext.envdir}/bin/activate &&
${runScriptsDir}/run_integration_test.sh $cmdArgs"
- }
- }
-}
diff --git a/sdks/python/test-suites/dataflow/py2/build.gradle
b/sdks/python/test-suites/dataflow/py2/build.gradle
new file mode 100644
index 0000000..f234497
--- /dev/null
+++ b/sdks/python/test-suites/dataflow/py2/build.gradle
@@ -0,0 +1,116 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * License); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an AS IS BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+plugins { id 'org.apache.beam.module' }
+applyPythonNature()
+enablePythonPerformanceTest()
+
+def runScriptsDir = "${project.rootDir}/sdks/python/scripts"
+
+// Basic test options for ITs running on Jenkins.
+def basicTestOpts = [
+ "--nocapture", // print stdout instantly
+ "--processes=8", // run tests in parallel
+ "--process-timeout=4500", // timeout of whole command execution
+]
+
+task preCommitIT(dependsOn: ['sdist', 'installGcpTest']) {
+ dependsOn ":runners:google-cloud-dataflow-java:worker:shadowJar"
+
+ def dataflowWorkerJar =
project(":runners:google-cloud-dataflow-java:worker").shadowJar.archivePath
+
+ doLast {
+ // Basic integration tests to run in PreCommit
+ def precommitTests = [
+ "apache_beam.examples.wordcount_it_test:WordCountIT.test_wordcount_it",
+
"apache_beam.examples.streaming_wordcount_it_test:StreamingWordCountIT.test_streaming_wordcount_it",
+ ]
+ def testOpts = [
+ "--tests=${precommitTests.join(',')}",
+ "--nocapture", // Print stdout instantly
+ "--processes=2", // Number of tests running in parallel
+ "--process-timeout=1800", // Timeout of whole command execution
+ ]
+ def cmdArgs = project.mapToArgString([
+ "test_opts": testOpts,
+ "sdk_location": "${project.buildDir}/apache-beam.tar.gz",
+ "worker_jar": dataflowWorkerJar,
+ "suite": "preCommitIT-df"
+ ])
+
+ exec {
+ executable 'sh'
+ args '-c', ". ${project.ext.envdir}/bin/activate &&
${runScriptsDir}/run_integration_test.sh $cmdArgs"
+ }
+ }
+}
+
+// Run PostCommit integration tests on default runner (TestDataflowRunner)
+task postCommitIT(dependsOn: ['installGcpTest', 'sdist']) {
+ dependsOn ":runners:google-cloud-dataflow-java:worker:shadowJar"
+
+ def dataflowWorkerJar =
project(":runners:google-cloud-dataflow-java:worker").shadowJar.archivePath
+
+ doLast {
+ def testOpts = basicTestOpts + ["--attr=IT"]
+ def cmdArgs = project.mapToArgString(["test_opts": testOpts,
+ "worker_jar": dataflowWorkerJar,
+ "suite": "postCommitIT-df"])
+ exec {
+ executable 'sh'
+ args '-c', ". ${project.ext.envdir}/bin/activate &&
${runScriptsDir}/run_integration_test.sh $cmdArgs"
+ }
+ }
+}
+
+task validatesRunnerBatchTests(dependsOn: ['installGcpTest', 'sdist']) {
+ dependsOn ":runners:google-cloud-dataflow-java:worker:shadowJar"
+
+ def dataflowWorkerJar =
project(":runners:google-cloud-dataflow-java:worker").shadowJar.archivePath
+
+ doLast {
+ def testOpts = basicTestOpts + ["--attr=ValidatesRunner"]
+ def cmdArgs = project.mapToArgString(["test_opts": testOpts,
+ "worker_jar": dataflowWorkerJar,
+ "suite":
"validatesRunnerBatchTests-df"])
+ exec {
+ executable 'sh'
+ args '-c', ". ${project.ext.envdir}/bin/activate &&
${runScriptsDir}/run_integration_test.sh $cmdArgs"
+ }
+ }
+}
+
+task validatesRunnerStreamingTests(dependsOn: ['installGcpTest', 'sdist']) {
+ dependsOn ":runners:google-cloud-dataflow-java:worker:shadowJar"
+
+ def dataflowWorkerJar =
project(":runners:google-cloud-dataflow-java:worker").shadowJar.archivePath
+
+ doLast {
+ // TODO(BEAM-3544,BEAM-5025): Disable tests with 'sickbay-streaming' tag.
+ def testOpts = basicTestOpts +
["--attr=ValidatesRunner,!sickbay-streaming"]
+ def argMap = ["test_opts": testOpts,
+ "streaming": "true",
+ "worker_jar": dataflowWorkerJar,
+ "suite": "validatesRunnerStreamingTests-df"]
+ def cmdArgs = project.mapToArgString(argMap)
+ exec {
+ executable 'sh'
+ args '-c', ". ${project.ext.envdir}/bin/activate &&
${runScriptsDir}/run_integration_test.sh $cmdArgs"
+ }
+ }
+}
diff --git a/sdks/python/test-suites/direct/py2/build.gradle
b/sdks/python/test-suites/direct/py2/build.gradle
new file mode 100644
index 0000000..543d75e
--- /dev/null
+++ b/sdks/python/test-suites/direct/py2/build.gradle
@@ -0,0 +1,114 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * License); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an AS IS BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+plugins { id 'org.apache.beam.module' }
+applyPythonNature()
+
+def runScriptsDir = "${project.rootDir}/sdks/python/scripts"
+
+// Basic test options for ITs running on Jenkins.
+def basicTestOpts = [
+ "--nocapture", // print stdout instantly
+ "--processes=8", // run tests in parallel
+ "--process-timeout=4500", // timeout of whole command execution
+]
+
+task directRunnerIT(dependsOn: 'installGcpTest') {
+ // Run IT tests with TestDirectRunner in batch.
+ doLast {
+ def tests = [
+ "apache_beam.examples.wordcount_it_test:WordCountIT.test_wordcount_it",
+ "apache_beam.io.gcp.pubsub_integration_test:PubSubIntegrationTest",
+
"apache_beam.io.gcp.big_query_query_to_table_it_test:BigQueryQueryToTableIT",
+ "apache_beam.io.gcp.bigquery_io_read_it_test",
+ "apache_beam.io.gcp.datastore.v1new.datastore_write_it_test",
+ ]
+ def batchTestOpts = basicTestOpts + ["--tests=${tests.join(',')}"]
+ def argMap = ["runner": "TestDirectRunner",
+ "test_opts": batchTestOpts,
+ "suite": "directRunnerIT-batch"]
+ def batchCmdArgs = project.mapToArgString(argMap)
+ exec {
+ executable 'sh'
+ args '-c', ". ${project.ext.envdir}/bin/activate &&
${runScriptsDir}/run_integration_test.sh $batchCmdArgs"
+ }
+ }
+
+ // Run IT tests with TestDirectRunner in streaming.
+ doLast {
+ def tests = [
+ "apache_beam.examples.wordcount_it_test:WordCountIT.test_wordcount_it",
+ "apache_beam.io.gcp.pubsub_integration_test:PubSubIntegrationTest",
+
"apache_beam.io.gcp.bigquery_test:BigQueryStreamingInsertTransformIntegrationTests\
+.test_multiple_destinations_transform",
+
"apache_beam.io.gcp.bigquery_file_loads_test:BigQueryFileLoadsIT.test_bqfl_streaming",
+ ]
+ def streamingTestOpts = basicTestOpts + ["--tests=${tests.join(',')}"]
+ def argMap = ["runner": "TestDirectRunner",
+ "streaming": "true",
+ "test_opts": streamingTestOpts,
+ "suite": "directRunnerIT-streaming"]
+ def streamingCmdArgs = project.mapToArgString(argMap)
+ exec {
+ executable 'sh'
+ args '-c', ". ${project.ext.envdir}/bin/activate &&
${runScriptsDir}/run_integration_test.sh $streamingCmdArgs"
+ }
+ }
+}
+
+task hdfsIntegrationTest(dependsOn: 'installGcpTest') {
+ doLast {
+ exec {
+ executable 'sh'
+ args '-c', ". ${project.ext.envdir}/bin/activate &&
${project.rootDir}/sdks/python/apache_beam/io/hdfs_integration_test/hdfs_integration_test.sh
python:2"
+ }
+ }
+}
+
+project.task('mongodbioIT') {
+ dependsOn 'setupVirtualenv'
+
+ Random r = new Random()
+ def port = r.nextInt(1000) + 27017
+ def containerName = "mongoioit" + port
+
+ def options = [
+ "--mongo_uri=mongodb://localhost:" + port
+ ]
+
+ // Pull the latest mongodb docker image and run
+ doFirst {
+ exec {
+ executable 'sh'
+ args '-c', "docker pull mongo && docker run --name ${containerName} -p
${port}:27017 -d mongo:latest"
+ }
+ }
+
+ doLast {
+ exec {
+ executable 'sh'
+ args '-c', ". ${project.ext.envdir}/bin/activate && pip install -e
${project.rootDir}/sdks/python/[test] && python -m
apache_beam.io.mongodbio_it_test ${options.join(' ')}"
+ }
+ exec {
+ executable 'sh'
+ args '-c', "docker stop ${containerName} && docker rm ${containerName}"
+ }
+ }
+}
+
+
diff --git a/sdks/python/test-suites/portable/py2/build.gradle
b/sdks/python/test-suites/portable/py2/build.gradle
new file mode 100644
index 0000000..6d089d5
--- /dev/null
+++ b/sdks/python/test-suites/portable/py2/build.gradle
@@ -0,0 +1,235 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * License); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an AS IS BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import org.apache.tools.ant.taskdefs.condition.Os
+
+apply plugin: org.apache.beam.gradle.BeamModulePlugin
+applyPythonNature()
+
+def pythonRootDir = "${project.rootDir}/sdks/python"
+
+/*************************************************************************************************/
+
+addPortableWordCountTasks()
+
+task preCommitPy2() {
+ dependsOn ':runners:flink:1.5:job-server-container:docker'
+ dependsOn ':sdks:python:container:docker'
+ dependsOn portableWordCountBatch
+ dependsOn portableWordCountStreaming
+}
+
+// Before running this, you need to:
+//
+// 1. Build the SDK container:
+//
+// ./gradlew -p sdks/python/container docker
+//
+// 2. Either a) or b)
+// a) If you want the Job Server to run in a Docker container:
+//
+// ./gradlew :runners:flink:1.5:job-server-container:docker
+//
+// b) Otherwise, start a local JobService, for example, the Portable Flink
runner
+// (in a separate shell since it continues to run):
+//
+// ./gradlew :runners:flink:1.5:job-server:runShadow
+//
+// Then you can run this example:
+//
+// Docker (2a):
+//
+// ./gradlew :sdks:python:test-suites:portable:py2:portableWordCount
+//
+// Local JobService (2b):
+//
+// ./gradlew :sdks:python:test-suites:portable:py2:portableWordCount
-PjobEndpoint=localhost:8099
+//
+task portableWordCount {
+ dependsOn project.hasProperty("streaming") ? portableWordCountStreaming :
portableWordCountBatch
+}
+
+/*************************************************************************************************/
+
+project.task('crossLanguagePythonJavaFlink') {
+ dependsOn 'setupVirtualenv'
+ dependsOn ':runners:flink:1.5:job-server-container:docker'
+ dependsOn ':sdks:python:container:docker'
+ dependsOn ':sdks:java:container:docker'
+ dependsOn ':runners:core-construction-java:buildTestExpansionServiceJar'
+
+ doLast {
+ def testServiceExpansionJar =
project(":runners:core-construction-java:").buildTestExpansionServiceJar.archivePath
+ def options = [
+ "--runner=PortableRunner",
+ "--experiments=worker_threads=100",
+ "--parallelism=2",
+ "--shutdown_sources_on_final_watermark",
+ "--environment_cache_millis=10000",
+ "--expansion_service_port=8096",
+ "--expansion_service_jar=${testServiceExpansionJar}",
+ ]
+ exec {
+ executable 'sh'
+ args '-c', ". ${project.ext.envdir}/bin/activate && cd ${pythonRootDir}
&& pip install -e .[test] && python -m apache_beam.transforms.external_test
${options.join(' ')}"
+ }
+ }
+}
+
+project.task('crossLanguagePortableWordCount') {
+ dependsOn 'setupVirtualenv'
+ dependsOn ':runners:flink:1.5:job-server-container:docker'
+ dependsOn ':sdks:python:container:docker'
+ dependsOn ':sdks:java:container:docker'
+ dependsOn ':runners:core-construction-java:buildTestExpansionServiceJar'
+
+ doLast {
+ def testServiceExpansionJar =
project(":runners:core-construction-java:").buildTestExpansionServiceJar.archivePath
+ def options = [
+ "--input=/etc/profile",
+ "--output=/tmp/py-wordcount-portable",
+ "--runner=PortableRunner",
+ "--experiments=worker_threads=100",
+ "--parallelism=2",
+ "--shutdown_sources_on_final_watermark",
+ "--environment_cache_millis=10000",
+ "--expansion_service_jar=${testServiceExpansionJar}",
+ ]
+ exec {
+ executable 'sh'
+ args '-c', ". ${project.ext.envdir}/bin/activate && cd ${pythonRootDir}
&& pip install -e .[test] && python -m apache_beam.examples.wordcount_xlang
${options.join(' ')}"
+ // TODO: Check that the output file is generated and runs.
+ }
+ }
+}
+
+project.task('crossLanguageTests') {
+ dependsOn "crossLanguagePythonJavaFlink"
+ dependsOn "crossLanguagePortableWordCount"
+}
+
+/*************************************************************************************************/
+
+project.task('createProcessWorker') {
+ dependsOn ':sdks:python:container:build'
+ dependsOn 'setupVirtualenv'
+ def sdkWorkerFile = file("${project.buildDir}/sdk_worker.sh")
+ def osType = 'linux'
+ if (Os.isFamily(Os.FAMILY_MAC))
+ osType = 'darwin'
+ def workerScript =
"${project(":sdks:python:container:").buildDir.absolutePath}/target/launcher/${osType}_amd64/boot"
+ def sdkWorkerFileCode = "sh -c \"pip=`which pip` .
${project.ext.envdir}/bin/activate && ${workerScript} \$* \""
+ outputs.file sdkWorkerFile
+ doLast {
+ sdkWorkerFile.write sdkWorkerFileCode
+ exec {
+ commandLine('sh', '-c', ". ${project.ext.envdir}/bin/activate && cd
${pythonRootDir} && python setup.py install ")
+ }
+ exec {
+ commandLine('chmod', '+x', sdkWorkerFile)
+ }
+ }
+}
+
+task sparkValidatesRunner() {
+ dependsOn 'createProcessWorker'
+ dependsOn 'setupVirtualenv'
+ dependsOn ':runners:spark:job-server:shadowJar'
+ doLast {
+ def environment_config = "'{\"command\":
\"${project.buildDir.absolutePath}/sdk_worker.sh\"}'"
+ def argMap = [
+ "environment_type" : "PROCESS",
+ "spark_job_server_jar":
project(":runners:spark:job-server:").shadowJar.archivePath,
+ "environment_config": environment_config,
+ ]
+ def argString = project.mapToArgString(argMap)
+
+ // Optionally specify test function names separated by space e.g.:
+ // ./gradlew :sdks:python:test-suites:portable:py2:sparkValidatesRunner
-Ptests="test_external_transforms test_read"
+ // Otherwise run all test functions under SparkRunnerTest
+ def tests = project.hasProperty('tests') ?
+ project.property('tests').split().collect{ "SparkRunnerTest.$it"
}.join(' ') : ''
+
+ exec {
+ executable 'sh'
+ args '-c', ". ${project.ext.envdir}/bin/activate && cd ${pythonRootDir}
&& pip install -e .[test] && python -m
apache_beam.runners.portability.spark_runner_test $tests $argString"
+ }
+ }
+}
+
+/*************************************************************************************************/
+
+class CompatibilityMatrixConfig {
+ // Execute batch or streaming pipelines.
+ boolean streaming = false
+ // Execute on Docker or Process based environment.
+ SDK_WORKER_TYPE workerType = SDK_WORKER_TYPE.DOCKER
+
+ enum SDK_WORKER_TYPE {
+ DOCKER, PROCESS, LOOPBACK
+ }
+
+ // Whether to pre-optimize the pipeline with the Python optimizer.
+ boolean preOptimize = false
+}
+
+def flinkCompatibilityMatrix = {
+ def config = it ? it as CompatibilityMatrixConfig : new
CompatibilityMatrixConfig()
+ def workerType = config.workerType.name()
+ def streaming = config.streaming
+ def environment_config = config.workerType ==
CompatibilityMatrixConfig.SDK_WORKER_TYPE.PROCESS ?
"--environment_config='{\"command\":
\"${project.buildDir.absolutePath}/sdk_worker.sh\"}'" : ""
+ def name = "flinkCompatibilityMatrix${streaming ? 'Streaming' :
'Batch'}${config.preOptimize ? 'PreOptimize' : ''}${workerType}"
+ def extra_experiments = []
+ if (config.preOptimize)
+ extra_experiments.add('pre_optimize=all')
+ tasks.create(name: name) {
+ dependsOn 'setupVirtualenv'
+ dependsOn ':runners:flink:1.5:job-server:shadowJar'
+ dependsOn ':sdks:java:container:docker'
+ if (workerType.toLowerCase() == 'docker')
+ dependsOn ':sdks:python:container:docker'
+ else if (workerType.toLowerCase() == 'process')
+ dependsOn 'createProcessWorker'
+ doLast {
+ exec {
+ executable 'sh'
+ args '-c', ". ${project.ext.envdir}/bin/activate && cd
${pythonRootDir} && pip install -e .[test] && python -m
apache_beam.runners.portability.flink_runner_test
--flink_job_server_jar=${project(":runners:flink:1.5:job-server:").shadowJar.archivePath}
--environment_type=${workerType} ${environment_config} ${streaming ?
'--streaming' : ''} ${extra_experiments ? '--extra_experiments=' +
extra_experiments.join(',') : ''}"
+ }
+ }
+ }
+}
+
+task flinkCompatibilityMatrixDocker() {
+ dependsOn flinkCompatibilityMatrix(streaming: false)
+ dependsOn flinkCompatibilityMatrix(streaming: true)
+}
+
+task flinkCompatibilityMatrixProcess() {
+ dependsOn flinkCompatibilityMatrix(streaming: false, workerType:
CompatibilityMatrixConfig.SDK_WORKER_TYPE.PROCESS)
+ dependsOn flinkCompatibilityMatrix(streaming: true, workerType:
CompatibilityMatrixConfig.SDK_WORKER_TYPE.PROCESS)
+}
+
+task flinkCompatibilityMatrixLoopback() {
+ dependsOn flinkCompatibilityMatrix(streaming: false, workerType:
CompatibilityMatrixConfig.SDK_WORKER_TYPE.LOOPBACK)
+ dependsOn flinkCompatibilityMatrix(streaming: true, workerType:
CompatibilityMatrixConfig.SDK_WORKER_TYPE.LOOPBACK)
+ dependsOn flinkCompatibilityMatrix(streaming: true, workerType:
CompatibilityMatrixConfig.SDK_WORKER_TYPE.LOOPBACK, preOptimize: true)
+}
+
+task flinkValidatesRunner() {
+ dependsOn 'flinkCompatibilityMatrixLoopback'
+}
diff --git a/sdks/python/test-suites/portable/py35/build.gradle
b/sdks/python/test-suites/portable/py35/build.gradle
index e25815f..fc08832 100644
--- a/sdks/python/test-suites/portable/py35/build.gradle
+++ b/sdks/python/test-suites/portable/py35/build.gradle
@@ -23,7 +23,7 @@ pythonVersion = '3.5'
addPortableWordCountTasks()
-task portablePreCommitPy35() {
+task preCommitPy35() {
dependsOn ':runners:flink:1.5:job-server-container:docker'
dependsOn ':sdks:python:container:py3:docker'
dependsOn portableWordCountBatch
diff --git a/sdks/python/test-suites/tox/py2/build.gradle
b/sdks/python/test-suites/tox/py2/build.gradle
new file mode 100644
index 0000000..2cdb5a5
--- /dev/null
+++ b/sdks/python/test-suites/tox/py2/build.gradle
@@ -0,0 +1,59 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * License); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an AS IS BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/**
+ * Unit tests for Python 2
+ */
+
+plugins { id 'org.apache.beam.module' }
+applyPythonNature()
+
+task lint {}
+check.dependsOn lint
+
+toxTask "lintPy27", "py27-lint"
+lint.dependsOn lintPy27
+
+toxTask "lintPy27_3", "py27-lint3"
+lint.dependsOn lintPy27_3
+
+toxTask "testPy2Gcp", "py27-gcp"
+test.dependsOn testPy2Gcp
+
+toxTask "testPython2", "py27"
+test.dependsOn testPython2
+
+toxTask "testPy2Cython", "py27-cython"
+test.dependsOn testPy2Cython
+// Ensure that testPy2Cython runs exclusively to other tests. This line is not
+// actually required, since gradle doesn't do parallel execution within a
+// project.
+testPy2Cython.mustRunAfter testPython2, testPy2Gcp
+
+toxTask "docs", "docs"
+assemble.dependsOn docs
+
+toxTask "cover", "cover"
+
+task preCommitPy2() {
+ dependsOn "docs"
+ dependsOn "testPy2Cython"
+ dependsOn "testPython2"
+ dependsOn "testPy2Gcp"
+ dependsOn "lint"
+}
diff --git a/settings.gradle b/settings.gradle
index 4220945..dd9d189 100644
--- a/settings.gradle
+++ b/settings.gradle
@@ -125,14 +125,17 @@ include ":sdks:python"
include ":sdks:python:apache_beam:testing:load_tests"
include ":sdks:python:container"
include ":sdks:python:container:py3"
-include ":sdks:python:test-suites:dataflow"
+include ":sdks:python:test-suites:dataflow:py2"
include ":sdks:python:test-suites:dataflow:py35"
include ":sdks:python:test-suites:dataflow:py36"
include ":sdks:python:test-suites:dataflow:py37"
+include ":sdks:python:test-suites:direct:py2"
include ":sdks:python:test-suites:direct:py35"
include ":sdks:python:test-suites:direct:py36"
include ":sdks:python:test-suites:direct:py37"
+include ":sdks:python:test-suites:portable:py2"
include ":sdks:python:test-suites:portable:py35"
+include ":sdks:python:test-suites:tox:py2"
include ":sdks:python:test-suites:tox:py35"
include ":sdks:python:test-suites:tox:py36"
include ":sdks:python:test-suites:tox:py37"
