BigQuery: limit max job polling time to 1 minute Before the backoff would grow unboundedly, so we could in principle wait 1.5x to 2x the actual job time. For long running jobs this is hours. Now, we just back off at most 1 minute between checking the job state. Note there should be no danger of QPS overload here because we should have very few concurrent outstanding jobs
Project: http://git-wip-us.apache.org/repos/asf/incubator-beam/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-beam/commit/8d306bbf Tree: http://git-wip-us.apache.org/repos/asf/incubator-beam/tree/8d306bbf Diff: http://git-wip-us.apache.org/repos/asf/incubator-beam/diff/8d306bbf Branch: refs/heads/master Commit: 8d306bbf5481ff1a5d27a2ae8d73e710532154a8 Parents: be689df Author: Dan Halperin <dhalp...@google.com> Authored: Sun Sep 4 14:54:42 2016 -0700 Committer: Dan Halperin <dhalp...@google.com> Committed: Tue Sep 6 11:31:38 2016 -0700 ---------------------------------------------------------------------- .../apache/beam/sdk/io/gcp/bigquery/BigQueryServicesImpl.java | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-beam/blob/8d306bbf/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/BigQueryServicesImpl.java ---------------------------------------------------------------------- diff --git a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/BigQueryServicesImpl.java b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/BigQueryServicesImpl.java index 8b5e8c2..20dadff 100644 --- a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/BigQueryServicesImpl.java +++ b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/BigQueryServicesImpl.java @@ -232,11 +232,12 @@ class BigQueryServicesImpl implements BigQueryServices { } @Override - public Job pollJob(JobReference jobRef, int maxAttempts) - throws InterruptedException { + public Job pollJob(JobReference jobRef, int maxAttempts) throws InterruptedException { BackOff backoff = FluentBackoff.DEFAULT - .withMaxRetries(maxAttempts).withInitialBackoff(INITIAL_JOB_STATUS_POLL_BACKOFF) + .withMaxRetries(maxAttempts) + .withInitialBackoff(INITIAL_JOB_STATUS_POLL_BACKOFF) + .withMaxBackoff(Duration.standardMinutes(1)) .backoff(); return pollJob(jobRef, Sleeper.DEFAULT, backoff); }