commit 2fd5ba4bdc8edf0e202cac2a09cc5bbef69c7d7c
author Daniel Halperin <dhalperi@users.noreply.github.com> Tue Sep 06 19:53:06 2016 -0700
committer GitHub <noreply@github.com> Tue Sep 06 19:53:06 2016 -0700
tree b9eb1787f2e8769905e8df41a04d00f6c4fcc120
parent 5b818508dba7abd2ccf8d8aaa24a822baa15e4a4

BigQuery: limit max job polling time to 1 minute (#418)

Before the backoff would grow unboundedly, so we could in principle wait
1.5x to 2x the actual job time. For long running jobs this is hours.
Now, we just back off at most 1 minute between checking the job state.
Note there should be no danger of QPS overload here because we should
have very few concurrent outstanding jobs

sdk/src/main/java/com/google/cloud/dataflow/sdk/util/BigQueryServicesImpl.java

diff --git a/sdk/src/main/java/com/google/cloud/dataflow/sdk/util/BigQueryServicesImpl.java b/sdk/src/main/java/com/google/cloud/dataflow/sdk/util/BigQueryServicesImpl.java
index d1054db..d3b5d44 100644
--- a/sdk/src/main/java/com/google/cloud/dataflow/sdk/util/BigQueryServicesImpl.java
+++ b/sdk/src/main/java/com/google/cloud/dataflow/sdk/util/BigQueryServicesImpl.java
@@ -218,11 +218,12 @@
     }
 
     @Override
-    public Job pollJob(JobReference jobRef, int maxAttempts)
-        throws InterruptedException {
+    public Job pollJob(JobReference jobRef, int maxAttempts) throws InterruptedException {
       BackOff backoff =
           FluentBackoff.DEFAULT
-              .withMaxRetries(maxAttempts).withInitialBackoff(INITIAL_JOB_STATUS_POLL_BACKOFF)
+              .withMaxRetries(maxAttempts)
+              .withInitialBackoff(INITIAL_JOB_STATUS_POLL_BACKOFF)
+              .withMaxBackoff(Duration.standardMinutes(1))
               .backoff();
       return pollJob(jobRef, Sleeper.DEFAULT, backoff);
     }