This is an automated email from the ASF dual-hosted git repository. ibzib pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/beam.git
The following commit(s) were added to refs/heads/master by this push: new f8e8daf [BEAM-10278] Move pipeline jar tests from job server projects to python/test-suites/portable. new b002530 Merge pull request #13417 from ibzib/BEAM-10278 f8e8daf is described below commit f8e8dafdf78f29e551baaf64dc612689a31610d5 Author: Kyle Weaver <kcwea...@google.com> AuthorDate: Tue Nov 24 11:45:02 2020 -0800 [BEAM-10278] Move pipeline jar tests from job server projects to python/test-suites/portable. --- .../job_PostCommit_PortableJar_Flink.groovy | 3 +- .../job_PostCommit_PortableJar_Spark.groovy | 2 +- runners/flink/job-server/flink_job_server.gradle | 66 --------------------- .../test_flink_uber_jar.sh | 0 runners/portability/test_pipeline_jar.sh | 3 +- runners/spark/job-server/build.gradle | 32 ---------- sdks/python/test-suites/portable/common.gradle | 69 +++++++++++++++++++++- 7 files changed, 73 insertions(+), 102 deletions(-) diff --git a/.test-infra/jenkins/job_PostCommit_PortableJar_Flink.groovy b/.test-infra/jenkins/job_PostCommit_PortableJar_Flink.groovy index 7572669..856b7bf 100644 --- a/.test-infra/jenkins/job_PostCommit_PortableJar_Flink.groovy +++ b/.test-infra/jenkins/job_PostCommit_PortableJar_Flink.groovy @@ -31,7 +31,8 @@ PostcommitJobBuilder.postCommitJob('beam_PostCommit_PortableJar_Flink', steps { gradle { rootBuildScriptDir(commonJobProperties.checkoutDir) - tasks(':runners:flink:1.10:job-server:testPipelineJar') + tasks(':sdks:python:test-suites:portable:py36:testPipelineJarFlinkRunner') + tasks(':sdks:python:test-suites:portable:py37:testPipelineJarFlinkRunner') commonJobProperties.setGradleSwitches(delegate) } } diff --git a/.test-infra/jenkins/job_PostCommit_PortableJar_Spark.groovy b/.test-infra/jenkins/job_PostCommit_PortableJar_Spark.groovy index 8e4f1c5..93e58af 100644 --- a/.test-infra/jenkins/job_PostCommit_PortableJar_Spark.groovy +++ b/.test-infra/jenkins/job_PostCommit_PortableJar_Spark.groovy @@ -31,7 +31,7 @@ PostcommitJobBuilder.postCommitJob('beam_PostCommit_PortableJar_Spark', steps { gradle { rootBuildScriptDir(commonJobProperties.checkoutDir) - tasks(':runners:spark:job-server:testPipelineJar') + tasks(':sdks:python:test-suites:portable:py37:testPipelineJarSparkRunner') commonJobProperties.setGradleSwitches(delegate) } } diff --git a/runners/flink/job-server/flink_job_server.gradle b/runners/flink/job-server/flink_job_server.gradle index 99cd12d..130dd97 100644 --- a/runners/flink/job-server/flink_job_server.gradle +++ b/runners/flink/job-server/flink_job_server.gradle @@ -239,30 +239,6 @@ createCrossLanguageValidatesRunnerTask( ] ) -def addTestJavaJarCreator(String pyVersion) { - def pyBuildPath = pyVersion.startsWith("2") ? "2" : pyVersion.replaceAll("\\.", "") - project.tasks.create(name: "testJavaJarCreatorPy${pyBuildPath}") { - dependsOn shadowJar - dependsOn ":sdks:python:container:py${pyBuildPath}:docker" - doLast{ - exec { - executable "sh" - def options = [ - "--runner FlinkRunner", - "--job_server_jar ${shadowJar.archivePath}", - "--env_dir ${project.rootProject.buildDir}/gradleenv/${project.path.hashCode()}", - "--python_root_dir ${project.rootDir}/sdks/python", - "--python_version ${pyVersion}", - "--python_container_image ${project.docker_image_default_repo_root}/" - + "${project.docker_image_default_repo_prefix}" - + "python${pyVersion}_sdk:${project.sdk_version}", - ] - args "-c", "../../../portability/test_pipeline_jar.sh ${options.join(' ')}" - } - } - } -} - // miniCluster jar starts an embedded Flink cluster intended for use in testing. task miniCluster(type: Jar, dependsOn: shadowJar) { archiveBaseName = "${project.archivesBaseName}-mini-cluster" @@ -281,45 +257,3 @@ task miniCluster(type: Jar, dependsOn: shadowJar) { } zip64 true // jar needs to contain more than 65535 files } - -def addTestFlinkUberJarPy(String pyVersion, boolean saveMainSession) { - def pyBuildPath = pyVersion.startsWith("2") ? "2" : pyVersion.replaceAll("\\.", "") - project.tasks.create(name: "testFlinkUberJarPy${pyBuildPath}${saveMainSession ? 'saveMainSession' : ''}") { - dependsOn miniCluster - dependsOn shadowJar - dependsOn ":sdks:python:container:py${pyBuildPath}:docker" - doLast{ - exec { - executable "sh" - def options = [ - "--flink_job_server_jar ${shadowJar.archivePath}", - "--flink_mini_cluster_jar ${miniCluster.archivePath}", - "--env_dir ${project.rootProject.buildDir}/gradleenv/${project.path.hashCode()}", - "--python_root_dir ${project.rootDir}/sdks/python", - "--python_version ${pyVersion}", - "--python_container_image ${project.docker_image_default_repo_root}/" - + "${project.docker_image_default_repo_prefix}" - + "python${pyVersion}_sdk:${project.sdk_version}", - ] - if (saveMainSession) { - options.add('--save_main_session') - } - args "-c", "../../job-server/test_flink_uber_jar.sh ${options.join(' ')}" - } - } - } -} - -["2.7", "3.5", "3.6", "3.7"].each{ pyVersion -> - addTestJavaJarCreator(pyVersion) - addTestFlinkUberJarPy(pyVersion, false) - addTestFlinkUberJarPy(pyVersion, true) -} - -task testPipelineJar() { - dependsOn testJavaJarCreatorPy37 - dependsOn testFlinkUberJarPy36 - dependsOn testFlinkUberJarPy36saveMainSession - dependsOn testFlinkUberJarPy37 - dependsOn testFlinkUberJarPy37saveMainSession -} diff --git a/runners/flink/job-server/test_flink_uber_jar.sh b/runners/portability/test_flink_uber_jar.sh similarity index 100% rename from runners/flink/job-server/test_flink_uber_jar.sh rename to runners/portability/test_flink_uber_jar.sh diff --git a/runners/portability/test_pipeline_jar.sh b/runners/portability/test_pipeline_jar.sh index 308c92a..6b11bb1 100755 --- a/runners/portability/test_pipeline_jar.sh +++ b/runners/portability/test_pipeline_jar.sh @@ -68,6 +68,7 @@ command -v docker docker -v # Verify container has already been built +echo "Checking for Docker image ${PYTHON_CONTAINER_IMAGE}" docker images --format "{{.Repository}}:{{.Tag}}" | grep $PYTHON_CONTAINER_IMAGE # Set up Python environment @@ -108,7 +109,7 @@ else fi # Create the jar -OUTPUT_JAR=flink-test-$(date +%Y%m%d-%H%M%S).jar +OUTPUT_JAR="test-pipeline-${RUNNER}-$(date +%Y%m%d-%H%M%S).jar" (python -c "$PIPELINE_PY" \ --runner "$RUNNER" \ --"$INPUT_JAR_ARG" "$JOB_SERVER_JAR" \ diff --git a/runners/spark/job-server/build.gradle b/runners/spark/job-server/build.gradle index a653f3e..e08f8ae 100644 --- a/runners/spark/job-server/build.gradle +++ b/runners/spark/job-server/build.gradle @@ -236,35 +236,3 @@ createCrossLanguageValidatesRunnerTask( "--experiments=beam_fn_api", ] ) - -def addTestJavaJarCreator(String pyVersion) { - def pyBuildPath = pyVersion.startsWith("2") ? "2" : pyVersion.replaceAll("\\.", "") - project.tasks.create(name: "testJavaJarCreatorPy${pyBuildPath}") { - dependsOn shadowJar - dependsOn ":sdks:python:container:py${pyBuildPath}:docker" - doLast{ - exec { - executable "sh" - def options = [ - "--runner SparkRunner", - "--job_server_jar ${shadowJar.archivePath}", - "--env_dir ${project.rootProject.buildDir}/gradleenv/${project.path.hashCode()}", - "--python_root_dir ${project.rootDir}/sdks/python", - "--python_version ${pyVersion}", - "--python_container_image ${project.docker_image_default_repo_root}/" - + "${project.docker_image_default_repo_prefix}" - + "python${pyVersion}_sdk:${project.sdk_version}", - ] - args "-c", "../../portability/test_pipeline_jar.sh ${options.join(' ')}" - } - } - } -} - -["2.7", "3.5", "3.6", "3.7"].each { pyVersion -> - addTestJavaJarCreator(pyVersion) -} - -task testPipelineJar() { - dependsOn testJavaJarCreatorPy37 -} diff --git a/sdks/python/test-suites/portable/common.gradle b/sdks/python/test-suites/portable/common.gradle index 0e3e956..d273996 100644 --- a/sdks/python/test-suites/portable/common.gradle +++ b/sdks/python/test-suites/portable/common.gradle @@ -19,7 +19,7 @@ import org.apache.tools.ant.taskdefs.condition.Os */ def pythonRootDir = "${rootDir}/sdks/python" -def pythonVersionSuffix = project.ext.pythonVersion == '2.7' ? '2' : project.ext.pythonVersion.replace('.', '') +def pythonVersionSuffix = project.ext.pythonVersion.replace('.', '') ext { pythonContainerTask = ":sdks:python:container:py${pythonVersionSuffix}:docker" @@ -193,3 +193,70 @@ project.task("postCommitPy${pythonVersionSuffix}IT") { } } } + +def addTestJavaJarCreator(String runner, Task jobServerJarTask) { + project.tasks.create(name: "testJavaJarCreator${runner}") { + dependsOn jobServerJarTask + dependsOn pythonContainerTask + doLast{ + exec { + executable "sh" + def options = [ + "--runner ${runner}", + "--job_server_jar ${jobServerJarTask.archivePath}", + "--env_dir ${project.rootProject.buildDir}/gradleenv/${project.path.hashCode()}", + "--python_root_dir ${project.rootDir}/sdks/python", + "--python_version ${project.ext.pythonVersion}", + "--python_container_image ${project.docker_image_default_repo_root}/" + + "${project.docker_image_default_repo_prefix}" + + "python${project.ext.pythonVersion}_sdk:${project.sdk_version}", + ] + args "-c", "${project.rootDir}/runners/portability/test_pipeline_jar.sh ${options.join(' ')}" + } + } + } +} + +// TODO(BEAM-11333) Update and test multiple Flink versions. +addTestJavaJarCreator("FlinkRunner", tasks.getByPath(":runners:flink:1.10:job-server:shadowJar")) +addTestJavaJarCreator("SparkRunner", tasks.getByPath(":runners:spark:job-server:shadowJar")) + +def addTestFlinkUberJar(boolean saveMainSession) { + project.tasks.create(name: "testUberJarFlinkRunner${saveMainSession ? 'SaveMainSession' : ''}") { + dependsOn ':runners:flink:1.10:job-server:shadowJar' + dependsOn ':runners:flink:1.10:job-server:miniCluster' + dependsOn pythonContainerTask + doLast{ + exec { + executable "sh" + def options = [ + "--flink_job_server_jar ${tasks.getByPath(':runners:flink:1.10:job-server:shadowJar').archivePath}", + "--flink_mini_cluster_jar ${tasks.getByPath(':runners:flink:1.10:job-server:miniCluster').archivePath}", + "--env_dir ${project.rootProject.buildDir}/gradleenv/${project.path.hashCode()}", + "--python_root_dir ${project.rootDir}/sdks/python", + "--python_version ${project.ext.pythonVersion}", + "--python_container_image ${project.docker_image_default_repo_root}/" + + "${project.docker_image_default_repo_prefix}" + + "python${project.ext.pythonVersion}_sdk:${project.sdk_version}", + ] + if (saveMainSession) { + options.add('--save_main_session') + } + args "-c", "${project.rootDir}/runners/portability/test_flink_uber_jar.sh ${options.join(' ')}" + } + } + } +} + +addTestFlinkUberJar(false) +addTestFlinkUberJar(true) + +task testPipelineJarSparkRunner() { + dependsOn testJavaJarCreatorSparkRunner +} + +task testPipelineJarFlinkRunner() { + dependsOn testJavaJarCreatorFlinkRunner + dependsOn testUberJarFlinkRunner + dependsOn testUberJarFlinkRunnerSaveMainSession +}