This is an automated email from the ASF dual-hosted git repository. markliu pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/beam.git
The following commit(s) were added to refs/heads/master by this push: new 0e60dfa Merge pull request #9183: [BEAM-7675] Unify Gradle test configuration across Py2 and Py3 (#9183) 0e60dfa is described below commit 0e60dfacacfaf6deb2ea81ee3deb58a96dcbc9e6 Author: Mark Liu <mark...@apache.org> AuthorDate: Tue Jul 30 09:11:50 2019 -0700 Merge pull request #9183: [BEAM-7675] Unify Gradle test configuration across Py2 and Py3 (#9183) --- .../jenkins/job_PerformanceTests_Python.groovy | 2 +- .../job_PostCommit_Python_MongoDBIO_IT.groovy | 2 +- ...stCommit_Python_ValidatesRunner_Dataflow.groovy | 4 +- ..._PostCommit_Python_ValidatesRunner_Spark.groovy | 2 +- ...b_PreCommit_Python_ValidatesRunner_Flink.groovy | 2 +- build.gradle | 14 +- .../apache_beam/io/gcp/gcsio_integration_test.py | 2 +- sdks/python/build.gradle | 406 +-------------------- sdks/python/scripts/run_integration_test.sh | 8 +- sdks/python/test-suites/dataflow/build.gradle | 53 --- sdks/python/test-suites/dataflow/py2/build.gradle | 116 ++++++ sdks/python/test-suites/direct/py2/build.gradle | 114 ++++++ sdks/python/test-suites/portable/py2/build.gradle | 235 ++++++++++++ sdks/python/test-suites/portable/py35/build.gradle | 2 +- sdks/python/test-suites/tox/py2/build.gradle | 59 +++ settings.gradle | 5 +- 16 files changed, 550 insertions(+), 476 deletions(-) diff --git a/.test-infra/jenkins/job_PerformanceTests_Python.groovy b/.test-infra/jenkins/job_PerformanceTests_Python.groovy index 0b0fb02..686af9b 100644 --- a/.test-infra/jenkins/job_PerformanceTests_Python.groovy +++ b/.test-infra/jenkins/job_PerformanceTests_Python.groovy @@ -66,7 +66,7 @@ def testConfigurations = [ jobTriggerPhrase : 'Run Python27 WordCountIT Performance Test', resultTable : 'beam_performance.wordcount_py27_pkb_results', itClass : 'apache_beam.examples.wordcount_it_test:WordCountIT.test_wordcount_it', - itModule : ':sdks:python', + itModule : ':sdks:python:test-suites:dataflow:py2', extraPipelineArgs : dataflowPipelineArgs + [ input: 'gs://apache-beam-samples/input_small_files/ascii_sort_1MB_input.0000*', // 1Gb output: 'gs://temp-storage-for-end-to-end-tests/py-it-cloud/output', diff --git a/.test-infra/jenkins/job_PostCommit_Python_MongoDBIO_IT.groovy b/.test-infra/jenkins/job_PostCommit_Python_MongoDBIO_IT.groovy index b09acc1..fdf3caa 100644 --- a/.test-infra/jenkins/job_PostCommit_Python_MongoDBIO_IT.groovy +++ b/.test-infra/jenkins/job_PostCommit_Python_MongoDBIO_IT.groovy @@ -31,7 +31,7 @@ PostcommitJobBuilder.postCommitJob('beam_PostCommit_Python_MongoDBIO_IT', steps { gradle { rootBuildScriptDir(commonJobProperties.checkoutDir) - tasks(':sdks:python:mongodbioIT') + tasks(':sdks:python:test-suites:direct:py2:mongodbioIT') commonJobProperties.setGradleSwitches(delegate) } } diff --git a/.test-infra/jenkins/job_PostCommit_Python_ValidatesRunner_Dataflow.groovy b/.test-infra/jenkins/job_PostCommit_Python_ValidatesRunner_Dataflow.groovy index 39045eb..f7fd557 100644 --- a/.test-infra/jenkins/job_PostCommit_Python_ValidatesRunner_Dataflow.groovy +++ b/.test-infra/jenkins/job_PostCommit_Python_ValidatesRunner_Dataflow.groovy @@ -36,8 +36,8 @@ PostcommitJobBuilder.postCommitJob('beam_PostCommit_Py_VR_Dataflow', 'Run Python steps { gradle { rootBuildScriptDir(commonJobProperties.checkoutDir) - tasks(':sdks:python:validatesRunnerBatchTests') - tasks(':sdks:python:validatesRunnerStreamingTests') + tasks(':sdks:python:test-suites:dataflow:py2:validatesRunnerBatchTests') + tasks(':sdks:python:test-suites:dataflow:py2:validatesRunnerStreamingTests') tasks(':sdks:python:test-suites:dataflow:py35:validatesRunnerBatchTests') tasks(':sdks:python:test-suites:dataflow:py36:validatesRunnerBatchTests') tasks(':sdks:python:test-suites:dataflow:py37:validatesRunnerBatchTests') diff --git a/.test-infra/jenkins/job_PostCommit_Python_ValidatesRunner_Spark.groovy b/.test-infra/jenkins/job_PostCommit_Python_ValidatesRunner_Spark.groovy index cd69560..59b45f7 100644 --- a/.test-infra/jenkins/job_PostCommit_Python_ValidatesRunner_Spark.groovy +++ b/.test-infra/jenkins/job_PostCommit_Python_ValidatesRunner_Spark.groovy @@ -31,7 +31,7 @@ PostcommitJobBuilder.postCommitJob('beam_PostCommit_Python_VR_Spark', steps { gradle { rootBuildScriptDir(commonJobProperties.checkoutDir) - tasks(':sdks:python:sparkValidatesRunner') + tasks(':sdks:python:test-suites:portable:py2:sparkValidatesRunner') commonJobProperties.setGradleSwitches(delegate) } } diff --git a/.test-infra/jenkins/job_PreCommit_Python_ValidatesRunner_Flink.groovy b/.test-infra/jenkins/job_PreCommit_Python_ValidatesRunner_Flink.groovy index 3df3c97..eb34f1e 100644 --- a/.test-infra/jenkins/job_PreCommit_Python_ValidatesRunner_Flink.groovy +++ b/.test-infra/jenkins/job_PreCommit_Python_ValidatesRunner_Flink.groovy @@ -23,7 +23,7 @@ import PrecommitJobBuilder PrecommitJobBuilder builder = new PrecommitJobBuilder( scope: this, nameBase: 'Python_PVR_Flink', - gradleTask: ':sdks:python:flinkValidatesRunner', + gradleTask: ':sdks:python:test-suites:portable:py2:flinkValidatesRunner', triggerPathPatterns: [ '^model/.*$', '^runners/core-construction-java/.*$', diff --git a/build.gradle b/build.gradle index 24507fa..7189fc9 100644 --- a/build.gradle +++ b/build.gradle @@ -192,11 +192,11 @@ task goIntegrationTests() { } task pythonPreCommit() { - dependsOn ":sdks:python:preCommitPy2" + dependsOn ":sdks:python:test-suites:tox:py2:preCommitPy2" dependsOn ":sdks:python:test-suites:tox:py35:preCommitPy35" dependsOn ":sdks:python:test-suites:tox:py36:preCommitPy36" dependsOn ":sdks:python:test-suites:tox:py37:preCommitPy37" - dependsOn ":sdks:python:test-suites:dataflow:preCommitIT" + dependsOn ":sdks:python:test-suites:dataflow:py2:preCommitIT" dependsOn ":sdks:python:test-suites:dataflow:py37:preCommitIT" // We don't include Py35, Py36 precommit ITs to reduce quota footprint. // We can reconsider if we ever see an issue that these suites would @@ -204,7 +204,11 @@ task pythonPreCommit() { } task python2PostCommit() { - dependsOn ":sdks:python:postCommit" + dependsOn ":sdks:python:test-suites:portable:py2:crossLanguageTests" + dependsOn ":sdks:python:test-suites:dataflow:py2:postCommitIT" + dependsOn ":sdks:python:test-suites:direct:py2:directRunnerIT" + dependsOn ":sdks:python:test-suites:direct:py2:hdfsIntegrationTest" + dependsOn ":sdks:python:test-suites:direct:py2:mongodbioIT" } task python35PostCommit() { @@ -224,8 +228,8 @@ task python37PostCommit() { } task portablePythonPreCommit() { - dependsOn "sdks:python:portablePreCommitPy2" - dependsOn "sdks:python:test-suites:portable:py35:portablePreCommitPy35" + dependsOn ":sdks:python:test-suites:portable:py2:preCommitPy2" + dependsOn ":sdks:python:test-suites:portable:py35:preCommitPy35" } task websitePreCommit() { diff --git a/sdks/python/apache_beam/io/gcp/gcsio_integration_test.py b/sdks/python/apache_beam/io/gcp/gcsio_integration_test.py index 26c8655..d4e387b 100644 --- a/sdks/python/apache_beam/io/gcp/gcsio_integration_test.py +++ b/sdks/python/apache_beam/io/gcp/gcsio_integration_test.py @@ -31,7 +31,7 @@ The project's Cloud Storage service account requires Encrypter/Decrypter permissions for the key specified in --kms_key_name. To run these tests manually: - ./gradlew :sdks:python:integrationTest \ + ./gradlew :sdks:python:test-suites:dataflow:integrationTest \ -Dtests=apache_beam.io.gcp.gcsio_integration_test:GcsIOIntegrationTest \ -DkmsKeyName=KMS_KEY_NAME """ diff --git a/sdks/python/build.gradle b/sdks/python/build.gradle index e0411fe..5fc3740 100644 --- a/sdks/python/build.gradle +++ b/sdks/python/build.gradle @@ -16,11 +16,8 @@ * limitations under the License. */ -import org.apache.tools.ant.taskdefs.condition.Os - plugins { id 'org.apache.beam.module' } applyPythonNature() -enablePythonPerformanceTest() /*************************************************************************************************/ @@ -39,299 +36,7 @@ build.dependsOn buildPython /*************************************************************************************************/ -// Unit tests for Python 2 -// See Python 3 tests in test-suites/tox - -task lint {} -check.dependsOn lint - -toxTask "lintPy27", "py27-lint" -lint.dependsOn lintPy27 - -toxTask "lintPy27_3", "py27-lint3" -lint.dependsOn lintPy27_3 - -toxTask "testPy2Gcp", "py27-gcp" -test.dependsOn testPy2Gcp - -toxTask "testPython2", "py27" -test.dependsOn testPython2 - -toxTask "testPy2Cython", "py27-cython" -test.dependsOn testPy2Cython -// Ensure that testPy2Cython runs exclusively to other tests. This line is not -// actually required, since gradle doesn't do parallel execution within a -// project. -testPy2Cython.mustRunAfter testPython2, testPy2Gcp - -toxTask "docs", "docs" -assemble.dependsOn docs - -toxTask "cover", "cover" - -task preCommitPy2() { - dependsOn "docs" - dependsOn "testPy2Cython" - dependsOn "testPython2" - dependsOn "testPy2Gcp" - dependsOn "lint" -} - -addPortableWordCountTasks() - -task portablePreCommitPy2() { - dependsOn ':runners:flink:1.5:job-server-container:docker' - dependsOn ':sdks:python:container:docker' - dependsOn portableWordCountBatch - dependsOn portableWordCountStreaming -} - - -/*************************************************************************************************/ -// E2E integration testing and validates runner testing - -// Basic test options for ITs running on Jenkins. -def basicTestOpts = [ - "--nocapture", // print stdout instantly - "--processes=8", // run tests in parallel - "--process-timeout=4500", // timeout of whole command execution -] - -task directRunnerIT(dependsOn: 'installGcpTest') { - // Run IT tests with TestDirectRunner in batch. - doLast { - def tests = [ - "apache_beam.examples.wordcount_it_test:WordCountIT.test_wordcount_it", - "apache_beam.io.gcp.pubsub_integration_test:PubSubIntegrationTest", - "apache_beam.io.gcp.big_query_query_to_table_it_test:BigQueryQueryToTableIT", - "apache_beam.io.gcp.bigquery_io_read_it_test", - "apache_beam.io.gcp.datastore.v1new.datastore_write_it_test", - ] - def batchTestOpts = basicTestOpts + ["--tests=${tests.join(',')}"] - def argMap = ["runner": "TestDirectRunner", - "test_opts": batchTestOpts, - "suite": "directRunnerIT-batch"] - def batchCmdArgs = project.mapToArgString(argMap) - exec { - executable 'sh' - args '-c', ". ${project.ext.envdir}/bin/activate && ./scripts/run_integration_test.sh $batchCmdArgs" - } - } - - // Run IT tests with TestDirectRunner in streaming. - doLast { - def tests = [ - "apache_beam.examples.wordcount_it_test:WordCountIT.test_wordcount_it", - "apache_beam.io.gcp.pubsub_integration_test:PubSubIntegrationTest", - "apache_beam.io.gcp.bigquery_test:BigQueryStreamingInsertTransformIntegrationTests\ -.test_multiple_destinations_transform", - "apache_beam.io.gcp.bigquery_file_loads_test:BigQueryFileLoadsIT.test_bqfl_streaming", - ] - def streamingTestOpts = basicTestOpts + ["--tests=${tests.join(',')}"] - def argMap = ["runner": "TestDirectRunner", - "streaming": "true", - "test_opts": streamingTestOpts, - "suite": "directRunnerIT-streaming"] - def streamingCmdArgs = project.mapToArgString(argMap) - exec { - executable 'sh' - args '-c', ". ${project.ext.envdir}/bin/activate && ./scripts/run_integration_test.sh $streamingCmdArgs" - } - } -} - -// Before running this, you need to: -// -// 1. Build the SDK container: -// -// ./gradlew -p sdks/python/container docker -// -// 2. Either a) or b) -// a) If you want the Job Server to run in a Docker container: -// -// ./gradlew :runners:flink:1.5:job-server-container:docker -// -// b) Otherwise, start a local JobService, for example, the Portable Flink runner -// (in a separate shell since it continues to run): -// -// ./gradlew :runners:flink:1.5:job-server:runShadow -// -// Then you can run this example: -// -// Docker (2a): -// -// ./gradlew :sdks:python:portableWordCount -// -// Local JobService (2b): -// -// ./gradlew :sdks:python:portableWordCount -PjobEndpoint=localhost:8099 -// - -task portableWordCount { - dependsOn project.hasProperty("streaming") ? portableWordCountStreaming : portableWordCountBatch -} - -// Run PostCommit integration tests on default runner (TestDataflowRunner) -task postCommitIT(dependsOn: ['installGcpTest', 'sdist']) { - dependsOn ":runners:google-cloud-dataflow-java:worker:shadowJar" - - def dataflowWorkerJar = project(":runners:google-cloud-dataflow-java:worker").shadowJar.archivePath - - doLast { - def testOpts = basicTestOpts + ["--attr=IT"] - def cmdArgs = project.mapToArgString(["test_opts": testOpts, - "worker_jar": dataflowWorkerJar, - "suite": "postCommitIT-df"]) - exec { - executable 'sh' - args '-c', ". ${project.ext.envdir}/bin/activate && ./scripts/run_integration_test.sh $cmdArgs" - } - } -} - -task validatesRunnerBatchTests(dependsOn: ['installGcpTest', 'sdist']) { - dependsOn ":runners:google-cloud-dataflow-java:worker:shadowJar" - - def dataflowWorkerJar = project(":runners:google-cloud-dataflow-java:worker").shadowJar.archivePath - - doLast { - def testOpts = basicTestOpts + ["--attr=ValidatesRunner"] - def cmdArgs = project.mapToArgString(["test_opts": testOpts, - "worker_jar": dataflowWorkerJar, - "suite": "validatesRunnerBatchTests-df"]) - exec { - executable 'sh' - args '-c', ". ${project.ext.envdir}/bin/activate && ./scripts/run_integration_test.sh $cmdArgs" - } - } -} - -task validatesRunnerStreamingTests(dependsOn: ['installGcpTest', 'sdist']) { - dependsOn ":runners:google-cloud-dataflow-java:worker:shadowJar" - - def dataflowWorkerJar = project(":runners:google-cloud-dataflow-java:worker").shadowJar.archivePath - - doLast { - // TODO(BEAM-3544,BEAM-5025): Disable tests with 'sickbay-streaming' tag. - def testOpts = basicTestOpts + ["--attr=ValidatesRunner,!sickbay-streaming"] - def argMap = ["test_opts": testOpts, - "streaming": "true", - "worker_jar": dataflowWorkerJar, - "suite": "validatesRunnerStreamingTests-df"] - def cmdArgs = project.mapToArgString(argMap) - exec { - executable 'sh' - args '-c', ". ${project.ext.envdir}/bin/activate && ./scripts/run_integration_test.sh $cmdArgs" - } - } -} - -task hdfsIntegrationTest(dependsOn: 'installGcpTest') { - doLast { - exec { - executable 'sh' - args '-c', ". ${project.ext.envdir}/bin/activate && ./apache_beam/io/hdfs_integration_test/hdfs_integration_test.sh python:2" - } - } -} - -task sparkValidatesRunner() { - dependsOn 'createProcessWorker' - dependsOn 'setupVirtualenv' - dependsOn ':runners:spark:job-server:shadowJar' - doLast { - def environment_config = "'{\"command\": \"${project(":sdks:python:").buildDir.absolutePath}/sdk_worker.sh\"}'" - def argMap = [ - "environment_type" : "PROCESS", - "spark_job_server_jar": project(":runners:spark:job-server:").shadowJar.archivePath, - "environment_config": environment_config, - ] - def argString = project.mapToArgString(argMap) - - // Optionally specify test function names separated by space e.g.: - // ./gradlew :sdks:python:sparkValidatesRunner -Ptests="test_external_transforms test_read" - // Otherwise run all test functions under SparkRunnerTest - def tests = project.hasProperty('tests') ? - project.property('tests').split().collect{ "SparkRunnerTest.$it" }.join(' ') : '' - - exec { - executable 'sh' - args '-c', ". ${project.ext.envdir}/bin/activate && pip install -e .[test] && python -m apache_beam.runners.portability.spark_runner_test $tests $argString" - } - } -} - -class CompatibilityMatrixConfig { - // Execute batch or streaming pipelines. - boolean streaming = false - // Execute on Docker or Process based environment. - SDK_WORKER_TYPE workerType = SDK_WORKER_TYPE.DOCKER - - enum SDK_WORKER_TYPE { - DOCKER, PROCESS, LOOPBACK - } - - // Whether to pre-optimize the pipeline with the Python optimizer. - boolean preOptimize = false -} - -def flinkCompatibilityMatrix = { - def config = it ? it as CompatibilityMatrixConfig : new CompatibilityMatrixConfig() - def workerType = config.workerType.name() - def streaming = config.streaming - def environment_config = config.workerType == CompatibilityMatrixConfig.SDK_WORKER_TYPE.PROCESS ? "--environment_config='{\"command\": \"${project(":sdks:python").buildDir.absolutePath}/sdk_worker.sh\"}'" : "" - def name = "flinkCompatibilityMatrix${streaming ? 'Streaming' : 'Batch'}${config.preOptimize ? 'PreOptimize' : ''}${workerType}" - def extra_experiments = [] - if (config.preOptimize) - extra_experiments.add('pre_optimize=all') - tasks.create(name: name) { - dependsOn 'setupVirtualenv' - dependsOn ':runners:flink:1.5:job-server:shadowJar' - dependsOn ':sdks:java:container:docker' - if (workerType.toLowerCase() == 'docker') - dependsOn ':sdks:python:container:docker' - else if (workerType.toLowerCase() == 'process') - dependsOn 'createProcessWorker' - doLast { - exec { - executable 'sh' - args '-c', ". ${project.ext.envdir}/bin/activate && pip install -e .[test] && python -m apache_beam.runners.portability.flink_runner_test --flink_job_server_jar=${project(":runners:flink:1.5:job-server:").shadowJar.archivePath} --environment_type=${workerType} ${environment_config} ${streaming ? '--streaming' : ''} ${extra_experiments ? '--extra_experiments=' + extra_experiments.join(',') : ''}" - } - } - } -} - -task flinkCompatibilityMatrixDocker() { - dependsOn flinkCompatibilityMatrix(streaming: false) - dependsOn flinkCompatibilityMatrix(streaming: true) -} - -task flinkCompatibilityMatrixProcess() { - dependsOn flinkCompatibilityMatrix(streaming: false, workerType: CompatibilityMatrixConfig.SDK_WORKER_TYPE.PROCESS) - dependsOn flinkCompatibilityMatrix(streaming: true, workerType: CompatibilityMatrixConfig.SDK_WORKER_TYPE.PROCESS) -} - -task flinkCompatibilityMatrixLoopback() { - dependsOn flinkCompatibilityMatrix(streaming: false, workerType: CompatibilityMatrixConfig.SDK_WORKER_TYPE.LOOPBACK) - dependsOn flinkCompatibilityMatrix(streaming: true, workerType: CompatibilityMatrixConfig.SDK_WORKER_TYPE.LOOPBACK) - dependsOn flinkCompatibilityMatrix(streaming: true, workerType: CompatibilityMatrixConfig.SDK_WORKER_TYPE.LOOPBACK, preOptimize: true) -} - -task flinkValidatesRunner() { - dependsOn 'flinkCompatibilityMatrixLoopback' -} - -task postCommit() { - dependsOn "crossLanguageTests" - dependsOn "directRunnerIT" - dependsOn "hdfsIntegrationTest" - dependsOn "postCommitIT" - dependsOn "mongodbioIT" -} - - -/*************************************************************************************************/ -// Other build and analysis tasks +// Non-testing builds and analysis tasks // Snapshot of dependency requirements defined in setup.py. // Results will be stored in files under Gradle build directory. @@ -359,112 +64,3 @@ task buildSnapshot() { dependsOn 'sdist' dependsOn 'depSnapshot' } - -project.task('createProcessWorker') { - dependsOn ':sdks:python:container:build' - dependsOn 'setupVirtualenv' - def sdkWorkerFile = file("${project.buildDir}/sdk_worker.sh") - def osType = 'linux' - if (Os.isFamily(Os.FAMILY_MAC)) - osType = 'darwin' - def workerScript = "${project(":sdks:python:container:").buildDir.absolutePath}/target/launcher/${osType}_amd64/boot" - def sdkWorkerFileCode = "sh -c \"pip=`which pip` . ${project.ext.envdir}/bin/activate && ${workerScript} \$* \"" - outputs.file sdkWorkerFile - doLast { - sdkWorkerFile.write sdkWorkerFileCode - exec { - commandLine('sh', '-c', ". ${project.ext.envdir}/bin/activate && cd ${project.projectDir} && python setup.py install ") - } - exec { - commandLine('chmod', '+x', sdkWorkerFile) - } - } -} - -project.task('crossLanguagePythonJavaFlink') { - dependsOn 'setupVirtualenv' - dependsOn ':runners:flink:1.5:job-server-container:docker' - dependsOn ':sdks:python:container:docker' - dependsOn ':sdks:java:container:docker' - dependsOn ':runners:core-construction-java:buildTestExpansionServiceJar' - - doLast { - def testServiceExpansionJar = project(":runners:core-construction-java:").buildTestExpansionServiceJar.archivePath - def options = [ - "--runner=PortableRunner", - "--experiments=worker_threads=100", - "--parallelism=2", - "--shutdown_sources_on_final_watermark", - "--environment_cache_millis=10000", - "--expansion_service_port=8096", - "--expansion_service_jar=${testServiceExpansionJar}", - ] - exec { - executable 'sh' - args '-c', ". ${project.ext.envdir}/bin/activate && pip install -e .[test] && python -m apache_beam.transforms.external_test ${options.join(' ')}" - } - } -} - -project.task('crossLanguagePortableWordCount') { - dependsOn 'setupVirtualenv' - dependsOn ':runners:flink:1.5:job-server-container:docker' - dependsOn ':sdks:python:container:docker' - dependsOn ':sdks:java:container:docker' - dependsOn ':runners:core-construction-java:buildTestExpansionServiceJar' - - doLast { - def testServiceExpansionJar = project(":runners:core-construction-java:").buildTestExpansionServiceJar.archivePath - def options = [ - "--input=/etc/profile", - "--output=/tmp/py-wordcount-portable", - "--runner=PortableRunner", - "--experiments=worker_threads=100", - "--parallelism=2", - "--shutdown_sources_on_final_watermark", - "--environment_cache_millis=10000", - "--expansion_service_jar=${testServiceExpansionJar}", - ] - exec { - executable 'sh' - args '-c', ". ${project.ext.envdir}/bin/activate && pip install -e .[test] && python -m apache_beam.examples.wordcount_xlang ${options.join(' ')}" - // TODO: Check that the output file is generated and runs. - } - } -} - -project.task('crossLanguageTests') { - dependsOn "crossLanguagePythonJavaFlink" - dependsOn "crossLanguagePortableWordCount" -} - -project.task('mongodbioIT') { - dependsOn 'setupVirtualenv' - - Random r = new Random() - def port = r.nextInt(1000) + 27017 - def containerName = "mongoioit" + port - - def options = [ - "--mongo_uri=mongodb://localhost:" + port - ] - - // Pull the latest mongodb docker image and run - doFirst { - exec { - executable 'sh' - args '-c', "docker pull mongo && docker run --name ${containerName} -p ${port}:27017 -d mongo:latest" - } - } - - doLast { - exec { - executable 'sh' - args '-c', ". ${project.ext.envdir}/bin/activate && pip install -e .[test] && python -m apache_beam.io.mongodbio_it_test ${options.join(' ')}" - } - exec { - executable 'sh' - args '-c', "docker stop ${containerName} && docker rm ${containerName}" - } - } -} diff --git a/sdks/python/scripts/run_integration_test.sh b/sdks/python/scripts/run_integration_test.sh index f87b8b8..b5f0c60 100755 --- a/sdks/python/scripts/run_integration_test.sh +++ b/sdks/python/scripts/run_integration_test.sh @@ -18,10 +18,10 @@ ########################################################################### # -# This script is useful to run single or a set of Python integration tests -# manually or through Gradle. Note, this script doesn't setup python -# environment which is required before running tests. Use Gradle task -# `:sdks:python:integrationTests` to do both together. +# This script is used in Gradle to run single or a set of Python integration tests +# locally or on Jenkins. Note, this script doesn't setup python environment which is +# required for integration test. In order to do so, run Gradle tasks defined in +# :sdks:python:test-suites instead. # # In order to run test with customer options, use following commandline flags: # diff --git a/sdks/python/test-suites/dataflow/build.gradle b/sdks/python/test-suites/dataflow/build.gradle deleted file mode 100644 index 8a395a9..0000000 --- a/sdks/python/test-suites/dataflow/build.gradle +++ /dev/null @@ -1,53 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * License); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an AS IS BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -plugins { id 'org.apache.beam.module' } -applyPythonNature() - -def runScriptsDir = "${project.rootDir}/sdks/python/scripts" - -task preCommitIT(dependsOn: ['sdist', 'installGcpTest']) { - dependsOn ":runners:google-cloud-dataflow-java:worker:shadowJar" - - def dataflowWorkerJar = project(":runners:google-cloud-dataflow-java:worker").shadowJar.archivePath - - doLast { - // Basic integration tests to run in PreCommit - def precommitTests = [ - "apache_beam.examples.wordcount_it_test:WordCountIT.test_wordcount_it", - "apache_beam.examples.streaming_wordcount_it_test:StreamingWordCountIT.test_streaming_wordcount_it", - ] - def testOpts = [ - "--tests=${precommitTests.join(',')}", - "--nocapture", // Print stdout instantly - "--processes=2", // Number of tests running in parallel - "--process-timeout=1800", // Timeout of whole command execution - ] - def cmdArgs = project.mapToArgString([ - "test_opts": testOpts, - "sdk_location": "${project.buildDir}/apache-beam.tar.gz", - "worker_jar": dataflowWorkerJar, - "suite": "preCommitIT-df" - ]) - - exec { - executable 'sh' - args '-c', ". ${project.ext.envdir}/bin/activate && ${runScriptsDir}/run_integration_test.sh $cmdArgs" - } - } -} diff --git a/sdks/python/test-suites/dataflow/py2/build.gradle b/sdks/python/test-suites/dataflow/py2/build.gradle new file mode 100644 index 0000000..f234497 --- /dev/null +++ b/sdks/python/test-suites/dataflow/py2/build.gradle @@ -0,0 +1,116 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * License); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an AS IS BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +plugins { id 'org.apache.beam.module' } +applyPythonNature() +enablePythonPerformanceTest() + +def runScriptsDir = "${project.rootDir}/sdks/python/scripts" + +// Basic test options for ITs running on Jenkins. +def basicTestOpts = [ + "--nocapture", // print stdout instantly + "--processes=8", // run tests in parallel + "--process-timeout=4500", // timeout of whole command execution +] + +task preCommitIT(dependsOn: ['sdist', 'installGcpTest']) { + dependsOn ":runners:google-cloud-dataflow-java:worker:shadowJar" + + def dataflowWorkerJar = project(":runners:google-cloud-dataflow-java:worker").shadowJar.archivePath + + doLast { + // Basic integration tests to run in PreCommit + def precommitTests = [ + "apache_beam.examples.wordcount_it_test:WordCountIT.test_wordcount_it", + "apache_beam.examples.streaming_wordcount_it_test:StreamingWordCountIT.test_streaming_wordcount_it", + ] + def testOpts = [ + "--tests=${precommitTests.join(',')}", + "--nocapture", // Print stdout instantly + "--processes=2", // Number of tests running in parallel + "--process-timeout=1800", // Timeout of whole command execution + ] + def cmdArgs = project.mapToArgString([ + "test_opts": testOpts, + "sdk_location": "${project.buildDir}/apache-beam.tar.gz", + "worker_jar": dataflowWorkerJar, + "suite": "preCommitIT-df" + ]) + + exec { + executable 'sh' + args '-c', ". ${project.ext.envdir}/bin/activate && ${runScriptsDir}/run_integration_test.sh $cmdArgs" + } + } +} + +// Run PostCommit integration tests on default runner (TestDataflowRunner) +task postCommitIT(dependsOn: ['installGcpTest', 'sdist']) { + dependsOn ":runners:google-cloud-dataflow-java:worker:shadowJar" + + def dataflowWorkerJar = project(":runners:google-cloud-dataflow-java:worker").shadowJar.archivePath + + doLast { + def testOpts = basicTestOpts + ["--attr=IT"] + def cmdArgs = project.mapToArgString(["test_opts": testOpts, + "worker_jar": dataflowWorkerJar, + "suite": "postCommitIT-df"]) + exec { + executable 'sh' + args '-c', ". ${project.ext.envdir}/bin/activate && ${runScriptsDir}/run_integration_test.sh $cmdArgs" + } + } +} + +task validatesRunnerBatchTests(dependsOn: ['installGcpTest', 'sdist']) { + dependsOn ":runners:google-cloud-dataflow-java:worker:shadowJar" + + def dataflowWorkerJar = project(":runners:google-cloud-dataflow-java:worker").shadowJar.archivePath + + doLast { + def testOpts = basicTestOpts + ["--attr=ValidatesRunner"] + def cmdArgs = project.mapToArgString(["test_opts": testOpts, + "worker_jar": dataflowWorkerJar, + "suite": "validatesRunnerBatchTests-df"]) + exec { + executable 'sh' + args '-c', ". ${project.ext.envdir}/bin/activate && ${runScriptsDir}/run_integration_test.sh $cmdArgs" + } + } +} + +task validatesRunnerStreamingTests(dependsOn: ['installGcpTest', 'sdist']) { + dependsOn ":runners:google-cloud-dataflow-java:worker:shadowJar" + + def dataflowWorkerJar = project(":runners:google-cloud-dataflow-java:worker").shadowJar.archivePath + + doLast { + // TODO(BEAM-3544,BEAM-5025): Disable tests with 'sickbay-streaming' tag. + def testOpts = basicTestOpts + ["--attr=ValidatesRunner,!sickbay-streaming"] + def argMap = ["test_opts": testOpts, + "streaming": "true", + "worker_jar": dataflowWorkerJar, + "suite": "validatesRunnerStreamingTests-df"] + def cmdArgs = project.mapToArgString(argMap) + exec { + executable 'sh' + args '-c', ". ${project.ext.envdir}/bin/activate && ${runScriptsDir}/run_integration_test.sh $cmdArgs" + } + } +} diff --git a/sdks/python/test-suites/direct/py2/build.gradle b/sdks/python/test-suites/direct/py2/build.gradle new file mode 100644 index 0000000..543d75e --- /dev/null +++ b/sdks/python/test-suites/direct/py2/build.gradle @@ -0,0 +1,114 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * License); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an AS IS BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +plugins { id 'org.apache.beam.module' } +applyPythonNature() + +def runScriptsDir = "${project.rootDir}/sdks/python/scripts" + +// Basic test options for ITs running on Jenkins. +def basicTestOpts = [ + "--nocapture", // print stdout instantly + "--processes=8", // run tests in parallel + "--process-timeout=4500", // timeout of whole command execution +] + +task directRunnerIT(dependsOn: 'installGcpTest') { + // Run IT tests with TestDirectRunner in batch. + doLast { + def tests = [ + "apache_beam.examples.wordcount_it_test:WordCountIT.test_wordcount_it", + "apache_beam.io.gcp.pubsub_integration_test:PubSubIntegrationTest", + "apache_beam.io.gcp.big_query_query_to_table_it_test:BigQueryQueryToTableIT", + "apache_beam.io.gcp.bigquery_io_read_it_test", + "apache_beam.io.gcp.datastore.v1new.datastore_write_it_test", + ] + def batchTestOpts = basicTestOpts + ["--tests=${tests.join(',')}"] + def argMap = ["runner": "TestDirectRunner", + "test_opts": batchTestOpts, + "suite": "directRunnerIT-batch"] + def batchCmdArgs = project.mapToArgString(argMap) + exec { + executable 'sh' + args '-c', ". ${project.ext.envdir}/bin/activate && ${runScriptsDir}/run_integration_test.sh $batchCmdArgs" + } + } + + // Run IT tests with TestDirectRunner in streaming. + doLast { + def tests = [ + "apache_beam.examples.wordcount_it_test:WordCountIT.test_wordcount_it", + "apache_beam.io.gcp.pubsub_integration_test:PubSubIntegrationTest", + "apache_beam.io.gcp.bigquery_test:BigQueryStreamingInsertTransformIntegrationTests\ +.test_multiple_destinations_transform", + "apache_beam.io.gcp.bigquery_file_loads_test:BigQueryFileLoadsIT.test_bqfl_streaming", + ] + def streamingTestOpts = basicTestOpts + ["--tests=${tests.join(',')}"] + def argMap = ["runner": "TestDirectRunner", + "streaming": "true", + "test_opts": streamingTestOpts, + "suite": "directRunnerIT-streaming"] + def streamingCmdArgs = project.mapToArgString(argMap) + exec { + executable 'sh' + args '-c', ". ${project.ext.envdir}/bin/activate && ${runScriptsDir}/run_integration_test.sh $streamingCmdArgs" + } + } +} + +task hdfsIntegrationTest(dependsOn: 'installGcpTest') { + doLast { + exec { + executable 'sh' + args '-c', ". ${project.ext.envdir}/bin/activate && ${project.rootDir}/sdks/python/apache_beam/io/hdfs_integration_test/hdfs_integration_test.sh python:2" + } + } +} + +project.task('mongodbioIT') { + dependsOn 'setupVirtualenv' + + Random r = new Random() + def port = r.nextInt(1000) + 27017 + def containerName = "mongoioit" + port + + def options = [ + "--mongo_uri=mongodb://localhost:" + port + ] + + // Pull the latest mongodb docker image and run + doFirst { + exec { + executable 'sh' + args '-c', "docker pull mongo && docker run --name ${containerName} -p ${port}:27017 -d mongo:latest" + } + } + + doLast { + exec { + executable 'sh' + args '-c', ". ${project.ext.envdir}/bin/activate && pip install -e ${project.rootDir}/sdks/python/[test] && python -m apache_beam.io.mongodbio_it_test ${options.join(' ')}" + } + exec { + executable 'sh' + args '-c', "docker stop ${containerName} && docker rm ${containerName}" + } + } +} + + diff --git a/sdks/python/test-suites/portable/py2/build.gradle b/sdks/python/test-suites/portable/py2/build.gradle new file mode 100644 index 0000000..6d089d5 --- /dev/null +++ b/sdks/python/test-suites/portable/py2/build.gradle @@ -0,0 +1,235 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * License); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an AS IS BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import org.apache.tools.ant.taskdefs.condition.Os + +apply plugin: org.apache.beam.gradle.BeamModulePlugin +applyPythonNature() + +def pythonRootDir = "${project.rootDir}/sdks/python" + +/*************************************************************************************************/ + +addPortableWordCountTasks() + +task preCommitPy2() { + dependsOn ':runners:flink:1.5:job-server-container:docker' + dependsOn ':sdks:python:container:docker' + dependsOn portableWordCountBatch + dependsOn portableWordCountStreaming +} + +// Before running this, you need to: +// +// 1. Build the SDK container: +// +// ./gradlew -p sdks/python/container docker +// +// 2. Either a) or b) +// a) If you want the Job Server to run in a Docker container: +// +// ./gradlew :runners:flink:1.5:job-server-container:docker +// +// b) Otherwise, start a local JobService, for example, the Portable Flink runner +// (in a separate shell since it continues to run): +// +// ./gradlew :runners:flink:1.5:job-server:runShadow +// +// Then you can run this example: +// +// Docker (2a): +// +// ./gradlew :sdks:python:test-suites:portable:py2:portableWordCount +// +// Local JobService (2b): +// +// ./gradlew :sdks:python:test-suites:portable:py2:portableWordCount -PjobEndpoint=localhost:8099 +// +task portableWordCount { + dependsOn project.hasProperty("streaming") ? portableWordCountStreaming : portableWordCountBatch +} + +/*************************************************************************************************/ + +project.task('crossLanguagePythonJavaFlink') { + dependsOn 'setupVirtualenv' + dependsOn ':runners:flink:1.5:job-server-container:docker' + dependsOn ':sdks:python:container:docker' + dependsOn ':sdks:java:container:docker' + dependsOn ':runners:core-construction-java:buildTestExpansionServiceJar' + + doLast { + def testServiceExpansionJar = project(":runners:core-construction-java:").buildTestExpansionServiceJar.archivePath + def options = [ + "--runner=PortableRunner", + "--experiments=worker_threads=100", + "--parallelism=2", + "--shutdown_sources_on_final_watermark", + "--environment_cache_millis=10000", + "--expansion_service_port=8096", + "--expansion_service_jar=${testServiceExpansionJar}", + ] + exec { + executable 'sh' + args '-c', ". ${project.ext.envdir}/bin/activate && cd ${pythonRootDir} && pip install -e .[test] && python -m apache_beam.transforms.external_test ${options.join(' ')}" + } + } +} + +project.task('crossLanguagePortableWordCount') { + dependsOn 'setupVirtualenv' + dependsOn ':runners:flink:1.5:job-server-container:docker' + dependsOn ':sdks:python:container:docker' + dependsOn ':sdks:java:container:docker' + dependsOn ':runners:core-construction-java:buildTestExpansionServiceJar' + + doLast { + def testServiceExpansionJar = project(":runners:core-construction-java:").buildTestExpansionServiceJar.archivePath + def options = [ + "--input=/etc/profile", + "--output=/tmp/py-wordcount-portable", + "--runner=PortableRunner", + "--experiments=worker_threads=100", + "--parallelism=2", + "--shutdown_sources_on_final_watermark", + "--environment_cache_millis=10000", + "--expansion_service_jar=${testServiceExpansionJar}", + ] + exec { + executable 'sh' + args '-c', ". ${project.ext.envdir}/bin/activate && cd ${pythonRootDir} && pip install -e .[test] && python -m apache_beam.examples.wordcount_xlang ${options.join(' ')}" + // TODO: Check that the output file is generated and runs. + } + } +} + +project.task('crossLanguageTests') { + dependsOn "crossLanguagePythonJavaFlink" + dependsOn "crossLanguagePortableWordCount" +} + +/*************************************************************************************************/ + +project.task('createProcessWorker') { + dependsOn ':sdks:python:container:build' + dependsOn 'setupVirtualenv' + def sdkWorkerFile = file("${project.buildDir}/sdk_worker.sh") + def osType = 'linux' + if (Os.isFamily(Os.FAMILY_MAC)) + osType = 'darwin' + def workerScript = "${project(":sdks:python:container:").buildDir.absolutePath}/target/launcher/${osType}_amd64/boot" + def sdkWorkerFileCode = "sh -c \"pip=`which pip` . ${project.ext.envdir}/bin/activate && ${workerScript} \$* \"" + outputs.file sdkWorkerFile + doLast { + sdkWorkerFile.write sdkWorkerFileCode + exec { + commandLine('sh', '-c', ". ${project.ext.envdir}/bin/activate && cd ${pythonRootDir} && python setup.py install ") + } + exec { + commandLine('chmod', '+x', sdkWorkerFile) + } + } +} + +task sparkValidatesRunner() { + dependsOn 'createProcessWorker' + dependsOn 'setupVirtualenv' + dependsOn ':runners:spark:job-server:shadowJar' + doLast { + def environment_config = "'{\"command\": \"${project.buildDir.absolutePath}/sdk_worker.sh\"}'" + def argMap = [ + "environment_type" : "PROCESS", + "spark_job_server_jar": project(":runners:spark:job-server:").shadowJar.archivePath, + "environment_config": environment_config, + ] + def argString = project.mapToArgString(argMap) + + // Optionally specify test function names separated by space e.g.: + // ./gradlew :sdks:python:test-suites:portable:py2:sparkValidatesRunner -Ptests="test_external_transforms test_read" + // Otherwise run all test functions under SparkRunnerTest + def tests = project.hasProperty('tests') ? + project.property('tests').split().collect{ "SparkRunnerTest.$it" }.join(' ') : '' + + exec { + executable 'sh' + args '-c', ". ${project.ext.envdir}/bin/activate && cd ${pythonRootDir} && pip install -e .[test] && python -m apache_beam.runners.portability.spark_runner_test $tests $argString" + } + } +} + +/*************************************************************************************************/ + +class CompatibilityMatrixConfig { + // Execute batch or streaming pipelines. + boolean streaming = false + // Execute on Docker or Process based environment. + SDK_WORKER_TYPE workerType = SDK_WORKER_TYPE.DOCKER + + enum SDK_WORKER_TYPE { + DOCKER, PROCESS, LOOPBACK + } + + // Whether to pre-optimize the pipeline with the Python optimizer. + boolean preOptimize = false +} + +def flinkCompatibilityMatrix = { + def config = it ? it as CompatibilityMatrixConfig : new CompatibilityMatrixConfig() + def workerType = config.workerType.name() + def streaming = config.streaming + def environment_config = config.workerType == CompatibilityMatrixConfig.SDK_WORKER_TYPE.PROCESS ? "--environment_config='{\"command\": \"${project.buildDir.absolutePath}/sdk_worker.sh\"}'" : "" + def name = "flinkCompatibilityMatrix${streaming ? 'Streaming' : 'Batch'}${config.preOptimize ? 'PreOptimize' : ''}${workerType}" + def extra_experiments = [] + if (config.preOptimize) + extra_experiments.add('pre_optimize=all') + tasks.create(name: name) { + dependsOn 'setupVirtualenv' + dependsOn ':runners:flink:1.5:job-server:shadowJar' + dependsOn ':sdks:java:container:docker' + if (workerType.toLowerCase() == 'docker') + dependsOn ':sdks:python:container:docker' + else if (workerType.toLowerCase() == 'process') + dependsOn 'createProcessWorker' + doLast { + exec { + executable 'sh' + args '-c', ". ${project.ext.envdir}/bin/activate && cd ${pythonRootDir} && pip install -e .[test] && python -m apache_beam.runners.portability.flink_runner_test --flink_job_server_jar=${project(":runners:flink:1.5:job-server:").shadowJar.archivePath} --environment_type=${workerType} ${environment_config} ${streaming ? '--streaming' : ''} ${extra_experiments ? '--extra_experiments=' + extra_experiments.join(',') : ''}" + } + } + } +} + +task flinkCompatibilityMatrixDocker() { + dependsOn flinkCompatibilityMatrix(streaming: false) + dependsOn flinkCompatibilityMatrix(streaming: true) +} + +task flinkCompatibilityMatrixProcess() { + dependsOn flinkCompatibilityMatrix(streaming: false, workerType: CompatibilityMatrixConfig.SDK_WORKER_TYPE.PROCESS) + dependsOn flinkCompatibilityMatrix(streaming: true, workerType: CompatibilityMatrixConfig.SDK_WORKER_TYPE.PROCESS) +} + +task flinkCompatibilityMatrixLoopback() { + dependsOn flinkCompatibilityMatrix(streaming: false, workerType: CompatibilityMatrixConfig.SDK_WORKER_TYPE.LOOPBACK) + dependsOn flinkCompatibilityMatrix(streaming: true, workerType: CompatibilityMatrixConfig.SDK_WORKER_TYPE.LOOPBACK) + dependsOn flinkCompatibilityMatrix(streaming: true, workerType: CompatibilityMatrixConfig.SDK_WORKER_TYPE.LOOPBACK, preOptimize: true) +} + +task flinkValidatesRunner() { + dependsOn 'flinkCompatibilityMatrixLoopback' +} diff --git a/sdks/python/test-suites/portable/py35/build.gradle b/sdks/python/test-suites/portable/py35/build.gradle index e25815f..fc08832 100644 --- a/sdks/python/test-suites/portable/py35/build.gradle +++ b/sdks/python/test-suites/portable/py35/build.gradle @@ -23,7 +23,7 @@ pythonVersion = '3.5' addPortableWordCountTasks() -task portablePreCommitPy35() { +task preCommitPy35() { dependsOn ':runners:flink:1.5:job-server-container:docker' dependsOn ':sdks:python:container:py3:docker' dependsOn portableWordCountBatch diff --git a/sdks/python/test-suites/tox/py2/build.gradle b/sdks/python/test-suites/tox/py2/build.gradle new file mode 100644 index 0000000..2cdb5a5 --- /dev/null +++ b/sdks/python/test-suites/tox/py2/build.gradle @@ -0,0 +1,59 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * License); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an AS IS BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/** + * Unit tests for Python 2 + */ + +plugins { id 'org.apache.beam.module' } +applyPythonNature() + +task lint {} +check.dependsOn lint + +toxTask "lintPy27", "py27-lint" +lint.dependsOn lintPy27 + +toxTask "lintPy27_3", "py27-lint3" +lint.dependsOn lintPy27_3 + +toxTask "testPy2Gcp", "py27-gcp" +test.dependsOn testPy2Gcp + +toxTask "testPython2", "py27" +test.dependsOn testPython2 + +toxTask "testPy2Cython", "py27-cython" +test.dependsOn testPy2Cython +// Ensure that testPy2Cython runs exclusively to other tests. This line is not +// actually required, since gradle doesn't do parallel execution within a +// project. +testPy2Cython.mustRunAfter testPython2, testPy2Gcp + +toxTask "docs", "docs" +assemble.dependsOn docs + +toxTask "cover", "cover" + +task preCommitPy2() { + dependsOn "docs" + dependsOn "testPy2Cython" + dependsOn "testPython2" + dependsOn "testPy2Gcp" + dependsOn "lint" +} diff --git a/settings.gradle b/settings.gradle index 4220945..dd9d189 100644 --- a/settings.gradle +++ b/settings.gradle @@ -125,14 +125,17 @@ include ":sdks:python" include ":sdks:python:apache_beam:testing:load_tests" include ":sdks:python:container" include ":sdks:python:container:py3" -include ":sdks:python:test-suites:dataflow" +include ":sdks:python:test-suites:dataflow:py2" include ":sdks:python:test-suites:dataflow:py35" include ":sdks:python:test-suites:dataflow:py36" include ":sdks:python:test-suites:dataflow:py37" +include ":sdks:python:test-suites:direct:py2" include ":sdks:python:test-suites:direct:py35" include ":sdks:python:test-suites:direct:py36" include ":sdks:python:test-suites:direct:py37" +include ":sdks:python:test-suites:portable:py2" include ":sdks:python:test-suites:portable:py35" +include ":sdks:python:test-suites:tox:py2" include ":sdks:python:test-suites:tox:py35" include ":sdks:python:test-suites:tox:py36" include ":sdks:python:test-suites:tox:py37"