This is an automated email from the ASF dual-hosted git repository. pabloem pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/beam.git
The following commit(s) were added to refs/heads/master by this push: new 68e2a105a3f [BEAM-14146] Python Streaming job failing to drain with BigQueryIO write errors new a0684f502f0 Merge pull request #17566 from [BEAM-14146] Python Streaming job failing to drain with BigQueryIO write errors 68e2a105a3f is described below commit 68e2a105a3f6d552b58eaec2cdc1b7c01296eb0f Author: Heejong Lee <heej...@gmail.com> AuthorDate: Thu May 5 13:30:45 2022 -0700 [BEAM-14146] Python Streaming job failing to drain with BigQueryIO write errors --- sdks/python/apache_beam/io/gcp/bigquery_tools.py | 18 +++--------------- sdks/python/apache_beam/io/gcp/bigquery_tools_test.py | 4 ++-- 2 files changed, 5 insertions(+), 17 deletions(-) diff --git a/sdks/python/apache_beam/io/gcp/bigquery_tools.py b/sdks/python/apache_beam/io/gcp/bigquery_tools.py index 676755d90f9..89efa1ef623 100644 --- a/sdks/python/apache_beam/io/gcp/bigquery_tools.py +++ b/sdks/python/apache_beam/io/gcp/bigquery_tools.py @@ -502,10 +502,9 @@ class BigQueryWrapper(object): job_labels=None): if not source_uris and not source_stream: - raise ValueError( - 'Either a non-empty list of fully-qualified source URIs must be ' - 'provided via the source_uris parameter or an open file object must ' - 'be provided via the source_stream parameter. Got neither.') + _LOGGER.warning( + 'Both source URIs and source stream are not provided. BigQuery load ' + 'job will not load any data.') if source_uris and source_stream: raise ValueError( @@ -1002,17 +1001,6 @@ class BigQueryWrapper(object): Returns: bigquery.JobReference with the information about the job that was started. """ - if not source_uris and not source_stream: - raise ValueError( - 'Either a non-empty list of fully-qualified source URIs must be ' - 'provided via the source_uris parameter or an open file object must ' - 'be provided via the source_stream parameter. Got neither.') - - if source_uris and source_stream: - raise ValueError( - 'Only one of source_uris and source_stream may be specified. ' - 'Got both.') - project_id = ( destination.projectId if load_job_project_id is None else load_job_project_id) diff --git a/sdks/python/apache_beam/io/gcp/bigquery_tools_test.py b/sdks/python/apache_beam/io/gcp/bigquery_tools_test.py index c2d56f56de7..db6abf8ded5 100644 --- a/sdks/python/apache_beam/io/gcp/bigquery_tools_test.py +++ b/sdks/python/apache_beam/io/gcp/bigquery_tools_test.py @@ -443,8 +443,8 @@ class TestBigQueryWrapper(unittest.TestCase): source_stream=io.BytesIO()) # Neither source_uri nor source_stream specified. - with self.assertRaises(ValueError): - wrapper.perform_load_job(destination='P:D.T', job_id='J') + wrapper.perform_load_job( + destination=parse_table_reference('project:dataset.table'), job_id='J') def test_perform_load_job_with_source_stream(self): client = mock.Mock()