See <https://ci-beam.apache.org/job/beam_PostCommit_Py_ValCont/8409/display/redirect?page=changes>
Changes: [noreply] Fixing BigQueryIO request too big corner case for streaming inserts ------------------------------------------ [...truncated 107.57 KB...] output | 'Write' >> WriteToText(known_args.output) apache_beam/pipeline.py:596: in __exit__ self.result = self.run() apache_beam/pipeline.py:549: in run self._options).run(False) apache_beam/pipeline.py:573: in run return self.runner.run_pipeline(self, self._options) apache_beam/runners/dataflow/test_dataflow_runner.py:53: in run_pipeline self.result = super().run_pipeline(pipeline, options) apache_beam/runners/dataflow/dataflow_runner.py:459: in run_pipeline environments.DockerEnvironment.from_options(options)) apache_beam/transforms/environments.py:341: in from_options options) apache_beam/runners/portability/sdk_container_builder.py:139: in build_container_image return builder._build() apache_beam/runners/portability/sdk_container_builder.py:90: in _build self._invoke_docker_build_and_push(container_image_name) _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ self = <apache_beam.runners.portability.sdk_container_builder._SdkContainerImageCloudBuilder object at 0x7fa3a3f06ac8> container_image_name = 'us.gcr.io/apache-beam-testing/jenkins/prebuild_python36_sdk/beam_python_prebuilt_sdk:b609f279-e064-43cb-9250-ef397b61136b' def _invoke_docker_build_and_push(self, container_image_name): project_id = self._google_cloud_options.project temp_location = self._google_cloud_options.temp_location # google cloud build service expects all the build source file to be # compressed into a tarball. tarball_path = os.path.join(self._temp_src_dir, '%s.tgz' % SOURCE_FOLDER) self._make_tarfile(tarball_path, self._temp_src_dir) _LOGGER.info( "Compressed source files for building sdk container at %s" % tarball_path) container_image_tag = container_image_name.split(':')[-1] gcs_location = os.path.join( temp_location, '%s-%s.tgz' % (SOURCE_FOLDER, container_image_tag)) self._upload_to_gcs(tarball_path, gcs_location) build = cloudbuild.Build() build.steps = [] step = cloudbuild.BuildStep() step.name = 'gcr.io/kaniko-project/executor:latest' step.args = ['--destination=' + container_image_name, '--cache=true'] step.dir = SOURCE_FOLDER build.steps.append(step) source = cloudbuild.Source() source.storageSource = cloudbuild.StorageSource() gcs_bucket, gcs_object = self._get_gcs_bucket_and_name(gcs_location) source.storageSource.bucket = os.path.join(gcs_bucket) source.storageSource.object = gcs_object build.source = source # TODO(zyichi): make timeout configurable build.timeout = '7200s' now = time.time() # operation = client.create_build(project_id=project_id, build=build) request = cloudbuild.CloudbuildProjectsBuildsCreateRequest( projectId=project_id, build=build) build = self._cloudbuild_client.projects_builds.Create(request) build_id, log_url = self._get_cloud_build_id_and_log_url(build.metadata) _LOGGER.info( 'Building sdk container with Google Cloud Build, this may ' 'take a few minutes, you may check build log at %s' % log_url) # block until build finish, if build fails exception will be raised and # stops the job submission. response = self._cloudbuild_client.projects_builds.Get( cloudbuild.CloudbuildProjectsBuildsGetRequest( id=build_id, projectId=project_id)) while response.status in [cloudbuild.Build.StatusValueValuesEnum.QUEUED, cloudbuild.Build.StatusValueValuesEnum.WORKING]: time.sleep(10) response = self._cloudbuild_client.projects_builds.Get( cloudbuild.CloudbuildProjectsBuildsGetRequest( id=build_id, projectId=project_id)) if response.status != cloudbuild.Build.StatusValueValuesEnum.SUCCESS: raise RuntimeError( > 'Failed to build python sdk container image on google cloud build, ' 'please check build log for error.') E RuntimeError: Failed to build python sdk container image on google cloud build, please check build log for error. apache_beam/runners/portability/sdk_container_builder.py:287: RuntimeError =============================== warnings summary =============================== apache_beam/io/filesystems_test.py:54 <https://ci-beam.apache.org/job/beam_PostCommit_Py_ValCont/ws/src/sdks/python/apache_beam/io/filesystems_test.py>:54: DeprecationWarning: invalid escape sequence \c self.assertIsNone(FileSystems.get_scheme('c:\\abc\cdf')) # pylint: disable=anomalous-backslash-in-string apache_beam/io/filesystems_test.py:62 <https://ci-beam.apache.org/job/beam_PostCommit_Py_ValCont/ws/src/sdks/python/apache_beam/io/filesystems_test.py>:62: DeprecationWarning: invalid escape sequence \d self.assertTrue(isinstance(FileSystems.get_filesystem('c:\\abc\def'), # pylint: disable=anomalous-backslash-in-string -- Docs: https://docs.pytest.org/en/latest/warnings.html - generated xml file: <https://ci-beam.apache.org/job/beam_PostCommit_Py_ValCont/ws/src/sdks/python/pytest-beam_python3.6_sdk.xml> - ============== 1 failed, 3 passed, 2 warnings in 1289.29 seconds =============== cleanup_container Untagged: us.gcr.io/apache-beam-testing/jenkins/beam_python3.6_sdk:20211025-061538659902022 Untagged: us.gcr.io/apache-beam-testing/jenkins/beam_python3.6_sdk@sha256:3876b41c0cbd55e7b6839209c2ed4b9646a54fa4bee4945de1edcf2f4eef1598 docker images --format '{{.Repository}}:{{.Tag}}' | grep $PREBUILD_SDK_CONTAINER_REGISTRY_PATH Untagged: us.gcr.io/apache-beam-testing/jenkins/prebuild_python36_sdk/beam_python_prebuilt_sdk:b0e5a8d5-1fc6-4cef-b5ef-ac93aa525a26 Untagged: us.gcr.io/apache-beam-testing/jenkins/prebuild_python36_sdk/beam_python_prebuilt_sdk@sha256:de15acafe5181fa7790113ce63593ce8f76adf688a6bb38982af0e4c09b6cbf2 Deleted: sha256:58a933b867c4bc145172b32a115f559fef771de8fd4a0f511b2a023732d217b4 Deleted: sha256:2ae2c82ddd71fcbec3c53ab3fa4d7113313a14a6f6b46c088aaf41075a8e179b Deleted: sha256:ac845b2efdbdb3daa0f1f4b4c60552cf8d50cce17ecf99fbcd52e10628fef9aa Deleted: sha256:e31486d2e4ab63b6edb402393c71e3a3a02f6cf236627fc79a10150b5e6fdb0f Deleted: sha256:eef27c289ab59fbfa6c01fea1e4a61486f4e34e2681c6efd3ef1db41d0492d10 Deleted: sha256:8de510415ba224ee4e3628842d8064ef42b3d88b8c72dcd98e033ffa9d836cc1 Deleted: sha256:fb0f8f90fa58c30a50466ccc101ae24f7b7105a0415b7afd3c5d022e9cb16f21 Deleted: sha256:f4cbe33a20fd0149616cdf6518e2a4774ba115a051c8cc3da77d6b64d5b2e2ac Deleted: sha256:93a04db611c1f5829f61633445ba5eed5942879509d2514576091f4c61b38ccb Deleted: sha256:5ffe31c4f43689f319e6333c0c333dc6145db61ddf9e83a1b4a27d38331510e5 Deleted: sha256:012cce032ec48f10c8e082a196ed976ce1022b4ae5ee075226856c74c1d21f5d Deleted: sha256:eec6fec9974dce6af41fe7c96290cf9088ec36146b7b4ca2e99b188546e6d319 Deleted: sha256:ef2499c318a1f39f2314c80b16aab1123d95060b56271fefc23ccf7f8ac489a6 Deleted: sha256:c689a6059be9a3a948c4ed1c0e11835dbc910ba66e38f06a289bc8bb3b14b922 Deleted: sha256:c1feb7fb6ddd8139c8e81f22c16abb25a205f963097ea23dd64c501e6008b0e7 Deleted: sha256:e556c76801c604985f19b0d8fe35dd6f45c1208912241f6af1bb7aa2ad92cc8e Deleted: sha256:cc12d3e966b6daa5944d6b62a5d048afcbe0f6667e6c994c9551e69bd4b92b39 Deleted: sha256:34d4f957c6d1e046481926beff2f9e6fc4feada0b4245e4e0a98031e4ab9a7e9 Deleted: sha256:264c1df281ecb757a35f637a81cdfbfbcb8fa981ecd19c22da38b24181171aa9 Deleted: sha256:62a41178107bb957aa525c75695125b13b51313a5b9bb03e6126dc07037b616c Deleted: sha256:13fbb4fb296a844e2fed7d88aac5a98b0a544897f873460dfaea499c8eb423c9 Deleted: sha256:32ffd91993c4679c32721590a5159e435e01bab675140ac6b96e286a6eab1889 Deleted: sha256:471cc209d726bc5f4ce9d6d6b5cbdd5c8182cbcc41c49c2eade519ae1839046f Deleted: sha256:67bfd5936f89a4163a96afc0eb16889fd23943251fba7c56951db1a8004eb20e Deleted: sha256:ff81648956dd8cb2719ad258356c729726eb0d0924850a0ef819064262c44f4b Deleted: sha256:52a7446c14d9639982952fb51ead52a6f4f7b15208845d69fe014c62899f26b7 Deleted: sha256:d78812954dbc54d0f3082d91f4aaa3efc59903fdfcd9d8cee546b19dd61798aa Deleted: sha256:bba01d4564aeb007b39661cfce7cdf35dc21c2adb87cdff57ce777d608993721 Deleted: sha256:19633dbbdffea59a5b39061bc358e0b03f1ed2ae3867bc43d85c8a7f18d42188 Deleted: sha256:a9e2617cc6cd62f79819cb30acf0d945512d18e1a7a706205571b094078ac94a Deleted: sha256:66e900f6c20f09aa21e24c2dd33d16a5817fda3c185db88a04f6d936619915f6 Deleted: sha256:c83907b7931a0cd71ca583fc3fb071ef808392f72300f5bb2b5688822a424fcc Deleted: sha256:e5cc5545015eaf3aa35b37381242ecb500d6514b22896946644772aa10f756ee Deleted: sha256:90550931db8dc570b8af8ba48c111839aafc4fd72966bca85788457fd4e9c0d7 Deleted: sha256:c810ddbd23f9e20a159d23fc1dc2467cc1ba913680ef4e53462770eed4b4b8a6 Deleted: sha256:6adfd84cace6c7e7dd77ba0eeff45e9cd9ec884e440f883a3d2f0966967638d1 Deleted: sha256:c1c570ab24ac3e61c553c58cf4a928b450641075a331928885f190c01759b9cf Deleted: sha256:9e346c6cbe103d825180f719fd6cf0d42d737775070d4ef48c14ca982e7598d1 Deleted: sha256:0013106ea2956b2d90ce4cefa8e9c447211b4cc90c663426f5ff0a12dd8c0964 Deleted: sha256:5c6173857940106c1ab44794175a52af096e309fb5dac5c41698e05be1a352ac Deleted: sha256:7d3f42cec70bac9f328b14032fe293769c939148c5f776beffb609aaa4880ca2 Deleted: sha256:b8765545b48b3d3d6dc19de1d17c47977ac3a9ddcf8d40dce830e8b965670688 Deleted: sha256:2039edd92e901e0a91b84435941a54ff51b0ec21586243ee287d0be06d929e87 Deleted: sha256:d84f0fef9a3027f285322b67fc04aee338fb63d502fe9a7936ff3d7721d756dc Deleted: sha256:bb709dd4c8607895a550918c052dfaaf227060bae8dc9edcf03438680a4a14f1 Deleted: sha256:37060df8fd153bdb8061473a5a1a07fa7144889a16497b2c1bff98c1a7fece7d Deleted: sha256:1a9e2de5e13509507800438c244680ba8ab99512c86aa064377926b702e08f23 Deleted: sha256:04c73ac1856ad7a11aab5f93ddb448ab2b9c9a3ebcdc9ce4ada2557d5aa8b13d Deleted: sha256:7bcd503128fbcd517bd0d38437b19495876c7eb381243f745b80a357cc495acc Deleted: sha256:f5db4263ab95397234be1eec802f346e57b703b67b13fc038bdb88f66eaff3bc Deleted: sha256:c24700b71dacece76dbeeb09a89db9f76e4b911abe568708dfbc57e0992cbeea Digests: - us.gcr.io/apache-beam-testing/jenkins/beam_python3.6_sdk@sha256:3876b41c0cbd55e7b6839209c2ed4b9646a54fa4bee4945de1edcf2f4eef1598 Associated tags: - 20211025-061538659902022 Tags: - us.gcr.io/apache-beam-testing/jenkins/beam_python3.6_sdk:20211025-061538659902022 Deleted [us.gcr.io/apache-beam-testing/jenkins/beam_python3.6_sdk:20211025-061538659902022]. Deleted [us.gcr.io/apache-beam-testing/jenkins/beam_python3.6_sdk@sha256:3876b41c0cbd55e7b6839209c2ed4b9646a54fa4bee4945de1edcf2f4eef1598]. gcloud container images list-tags $PREBUILD_SDK_CONTAINER_REGISTRY_PATH/beam_python_prebuilt_sdk --format="get(digest)" Digests: - us.gcr.io/apache-beam-testing/jenkins/prebuild_python36_sdk/beam_python_prebuilt_sdk@sha256:de15acafe5181fa7790113ce63593ce8f76adf688a6bb38982af0e4c09b6cbf2 Associated tags: - b0e5a8d5-1fc6-4cef-b5ef-ac93aa525a26 Deleted [us.gcr.io/apache-beam-testing/jenkins/prebuild_python36_sdk/beam_python_prebuilt_sdk:b0e5a8d5-1fc6-4cef-b5ef-ac93aa525a26]. Deleted [us.gcr.io/apache-beam-testing/jenkins/prebuild_python36_sdk/beam_python_prebuilt_sdk@sha256:de15acafe5181fa7790113ce63593ce8f76adf688a6bb38982af0e4c09b6cbf2]. Removed the container > Task :sdks:python:test-suites:dataflow:py36:validatesContainer FAILED > Task :sdks:python:test-suites:dataflow:py38:validatesContainer cleanup_container docker images --format '{{.Repository}}:{{.Tag}}' | grep $PREBUILD_SDK_CONTAINER_REGISTRY_PATH Digests: - us.gcr.io/apache-beam-testing/jenkins/beam_python3.8_sdk@sha256:994b6641cac1c2ad112c6ee4a9807fc0a3158f6b0a35ac87f178d20e9b3af616 Associated tags: - 20211025-061552046074833 Tags: - us.gcr.io/apache-beam-testing/jenkins/beam_python3.8_sdk:20211025-061552046074833 Deleted [us.gcr.io/apache-beam-testing/jenkins/beam_python3.8_sdk:20211025-061552046074833]. Deleted [us.gcr.io/apache-beam-testing/jenkins/beam_python3.8_sdk@sha256:994b6641cac1c2ad112c6ee4a9807fc0a3158f6b0a35ac87f178d20e9b3af616]. gcloud container images list-tags $PREBUILD_SDK_CONTAINER_REGISTRY_PATH/beam_python_prebuilt_sdk --format="get(digest)" Digests: - us.gcr.io/apache-beam-testing/jenkins/prebuild_python38_sdk/beam_python_prebuilt_sdk@sha256:a9bcb06737376adb88027fca36ee663cf540daee3ef6bbcb4ed54c166ffdb823 Associated tags: - 008fc315-df02-4ae5-9547-6ab7647326f0 Deleted [us.gcr.io/apache-beam-testing/jenkins/prebuild_python38_sdk/beam_python_prebuilt_sdk:008fc315-df02-4ae5-9547-6ab7647326f0]. Deleted [us.gcr.io/apache-beam-testing/jenkins/prebuild_python38_sdk/beam_python_prebuilt_sdk@sha256:a9bcb06737376adb88027fca36ee663cf540daee3ef6bbcb4ed54c166ffdb823]. > Task :sdks:python:test-suites:dataflow:py38:validatesContainer FAILED FAILURE: Build completed with 3 failures. 1: Task failed with an exception. ----------- * Where: Script '<https://ci-beam.apache.org/job/beam_PostCommit_Py_ValCont/ws/src/sdks/python/test-suites/dataflow/common.gradle'> line: 276 * What went wrong: Execution failed for task ':sdks:python:test-suites:dataflow:py37:validatesContainer'. > Process 'command 'sh'' finished with non-zero exit value 1 * Try: Run with --stacktrace option to get the stack trace. Run with --info or --debug option to get more log output. Run with --scan to get full insights. ============================================================================== 2: Task failed with an exception. ----------- * Where: Script '<https://ci-beam.apache.org/job/beam_PostCommit_Py_ValCont/ws/src/sdks/python/test-suites/dataflow/common.gradle'> line: 276 * What went wrong: Execution failed for task ':sdks:python:test-suites:dataflow:py36:validatesContainer'. > Process 'command 'sh'' finished with non-zero exit value 1 * Try: Run with --stacktrace option to get the stack trace. Run with --info or --debug option to get more log output. Run with --scan to get full insights. ============================================================================== 3: Task failed with an exception. ----------- * Where: Script '<https://ci-beam.apache.org/job/beam_PostCommit_Py_ValCont/ws/src/sdks/python/test-suites/dataflow/common.gradle'> line: 276 * What went wrong: Execution failed for task ':sdks:python:test-suites:dataflow:py38:validatesContainer'. > Process 'command 'sh'' finished with non-zero exit value 1 * Try: Run with --stacktrace option to get the stack trace. Run with --info or --debug option to get more log output. Run with --scan to get full insights. ============================================================================== * Get more help at https://help.gradle.org Deprecated Gradle features were used in this build, making it incompatible with Gradle 7.0. Use '--warning-mode all' to show the individual deprecation warnings. See https://docs.gradle.org/6.9.1/userguide/command_line_interface.html#sec:command_line_warnings BUILD FAILED in 40m 17s 44 actionable tasks: 40 executed, 4 up-to-date Publishing build scan... https://gradle.com/s/bpd7s775z7l2e Build step 'Invoke Gradle script' changed build result to FAILURE Build step 'Invoke Gradle script' marked build as failure --------------------------------------------------------------------- To unsubscribe, e-mail: builds-unsubscr...@beam.apache.org For additional commands, e-mail: builds-h...@beam.apache.org