This is an automated email from the ASF dual-hosted git repository. heejong pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/beam.git
The following commit(s) were added to refs/heads/master by this push: new f012bf9 [BEAM-11843] Fix empty artifact info generation by copy resource file names from generator new 820f3cd Merge pull request #14029 from y1chi/BEAM-11843 f012bf9 is described below commit f012bf9dd3fcb4480600f5b70413b0227c2a280d Author: Yichi Zhang <zyi...@google.com> AuthorDate: Fri Feb 19 21:01:04 2021 -0800 [BEAM-11843] Fix empty artifact info generation by copy resource file names from generator --- sdks/python/apache_beam/examples/wordcount_it_test.py | 9 ++++++++- .../apache_beam/runners/portability/sdk_container_builder.py | 8 +++++--- 2 files changed, 13 insertions(+), 4 deletions(-) diff --git a/sdks/python/apache_beam/examples/wordcount_it_test.py b/sdks/python/apache_beam/examples/wordcount_it_test.py index 5511ca6..317b5c6 100644 --- a/sdks/python/apache_beam/examples/wordcount_it_test.py +++ b/sdks/python/apache_beam/examples/wordcount_it_test.py @@ -57,12 +57,19 @@ class WordCountIT(unittest.TestCase): self._run_wordcount_it(wordcount.run, experiment='beam_fn_api') @attr('ValidatesContainer') - def test_wordcount_it_with_prebuilt_sdk_container(self): + def test_wordcount_it_with_prebuilt_sdk_container_local_docker(self): self._run_wordcount_it( wordcount.run, experiment='beam_fn_api', prebuild_sdk_container_engine='local_docker') + @attr('ValidatesContainer') + def test_wordcount_it_with_prebuilt_sdk_container_cloud_build(self): + self._run_wordcount_it( + wordcount.run, + experiment='beam_fn_api', + prebuild_sdk_container_engine='cloud_build') + def _run_wordcount_it(self, run_wordcount, **opts): test_pipeline = TestPipeline(is_integration_test=True) extra_opts = {} diff --git a/sdks/python/apache_beam/runners/portability/sdk_container_builder.py b/sdks/python/apache_beam/runners/portability/sdk_container_builder.py index e769d55..cc54416 100644 --- a/sdks/python/apache_beam/runners/portability/sdk_container_builder.py +++ b/sdks/python/apache_beam/runners/portability/sdk_container_builder.py @@ -99,8 +99,10 @@ class SdkContainerImageBuilder(plugin.BeamPlugin): artifacts = Stager.create_job_resources(self._options, tmp) resources = Stager.extract_staging_tuple_iter(artifacts) # make a copy of the staged artifacts into the temp source folder. + file_names = [] for path, name in resources: shutil.copyfile(path, os.path.join(self._temp_src_dir, name)) + file_names.append(name) with open(os.path.join(self._temp_src_dir, 'Dockerfile'), 'w') as file: file.write( DOCKERFILE_TEMPLATE.format( @@ -109,7 +111,7 @@ class SdkContainerImageBuilder(plugin.BeamPlugin): manifest_file=ARTIFACTS_MANIFEST_FILE, entrypoint=SDK_CONTAINER_ENTRYPOINT)) self._generate_artifacts_manifests_json_file( - resources, self._temp_src_dir) + file_names, self._temp_src_dir) def _invoke_docker_build_and_push(self, container_image_name): raise NotImplementedError @@ -119,9 +121,9 @@ class SdkContainerImageBuilder(plugin.BeamPlugin): return f'{cls.__module__}.{cls.__name__}' @staticmethod - def _generate_artifacts_manifests_json_file(resources, temp_dir): + def _generate_artifacts_manifests_json_file(file_names, temp_dir): infos = [] - for _, name in resources: + for name in file_names: info = beam_runner_api_pb2.ArtifactInformation( type_urn=common_urns.StandardArtifacts.Types.FILE.urn, type_payload=beam_runner_api_pb2.ArtifactFilePayload(