com.google.api.client.util.BackOff Java Examples

The following examples show how to use com.google.api.client.util.BackOff. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example #1
Source File: GcsUtilTest.java    From beam with Apache License 2.0 6 votes vote down vote up
@Test
public void testGetBucket() throws IOException {
  GcsOptions pipelineOptions = gcsOptionsWithTestCredential();
  GcsUtil gcsUtil = pipelineOptions.getGcsUtil();

  Storage mockStorage = Mockito.mock(Storage.class);
  gcsUtil.setStorageClient(mockStorage);

  Storage.Buckets mockStorageObjects = Mockito.mock(Storage.Buckets.class);
  Storage.Buckets.Get mockStorageGet = Mockito.mock(Storage.Buckets.Get.class);

  BackOff mockBackOff = BackOffAdapter.toGcpBackOff(FluentBackoff.DEFAULT.backoff());

  when(mockStorage.buckets()).thenReturn(mockStorageObjects);
  when(mockStorageObjects.get("testbucket")).thenReturn(mockStorageGet);
  when(mockStorageGet.execute())
      .thenThrow(new SocketTimeoutException("SocketException"))
      .thenReturn(new Bucket());

  assertNotNull(
      gcsUtil.getBucket(
          GcsPath.fromComponents("testbucket", "testobject"),
          mockBackOff,
          new FastNanoClockAndSleeper()));
}
 
Example #2
Source File: BatchRequestService.java    From connector-sdk with Apache License 2.0 6 votes vote down vote up
private boolean backOff(List<AsyncRequest<?>> pendingTasks) {
  long backOffMillis = getBackOffTime();
  if (backOffMillis == BackOff.STOP) {
    pendingTasks.forEach(
        t ->
            onFailure(
                t, new ExecutionException(new Exception("Maximum back-off cycle reached."))));
    return false;
  }
  try {
    batchRequestHelper.sleep(backOffMillis);
  } catch (InterruptedException e) {
    logger.log(Level.WARNING, "Task interrupted while processing batch requests, stopping.", e);
    pendingTasks.forEach(t -> onFailure(t, e));
    Thread.currentThread().interrupt();
    return false;
  }
  return true;
}
 
Example #3
Source File: RetryBoundedBackOffTest.java    From hadoop-connectors with Apache License 2.0 6 votes vote down vote up
@Test
public void resetsCorrectly() throws Exception {
  BackOff backoff = new RetryBoundedBackOff(5, new BackOffTester());
  for (int i = 0; i < 5; i++) {
    assertThat(i + 1).isEqualTo(backoff.nextBackOffMillis());
  }
  assertThat(backoff.nextBackOffMillis()).isEqualTo(BackOff.STOP);
  assertThat(backoff.nextBackOffMillis()).isEqualTo(BackOff.STOP);
  backoff.reset();
  for (int i = 0; i < 3; i++) {
    assertThat(i + 1).isEqualTo(backoff.nextBackOffMillis());
  }
  backoff.reset();
  for (int i = 0; i < 5; i++) {
    assertThat(i + 1).isEqualTo(backoff.nextBackOffMillis());
  }
  assertThat(backoff.nextBackOffMillis()).isEqualTo(BackOff.STOP);
  assertThat(backoff.nextBackOffMillis()).isEqualTo(BackOff.STOP);
}
 
Example #4
Source File: ResilientOperationTest.java    From hadoop-connectors with Apache License 2.0 6 votes vote down vote up
@Test
public void testCallFailsOnBadException() throws Exception {
  MockSleeper sleeper = new MockSleeper();
  ArrayList<Exception> exceptions = new ArrayList<>();
  exceptions.add(new IllegalArgumentException("FakeException"));
  CallableTester<Exception> callTester = new CallableTester<>(exceptions);
  BackOff backoff = new RetryBoundedBackOff(3, new BackOffTester());

  IllegalArgumentException thrown =
      assertThrows(
          IllegalArgumentException.class,
          () ->
              ResilientOperation.retry(
                  callTester, backoff, RetryDeterminer.DEFAULT, Exception.class, sleeper));
  assertThat(thrown).hasMessageThat().contains("FakeException");

  assertThat(callTester.timesCalled()).isEqualTo(1);
  verifySleeper(sleeper, 0);
}
 
Example #5
Source File: BigQueryToTableIT.java    From beam with Apache License 2.0 6 votes vote down vote up
private List<TableRow> getTableRowsFromQuery(String query, int maxRetry) throws Exception {
  FluentBackoff backoffFactory =
      FluentBackoff.DEFAULT
          .withMaxRetries(maxRetry)
          .withInitialBackoff(Duration.standardSeconds(1L));
  Sleeper sleeper = Sleeper.DEFAULT;
  BackOff backoff = BackOffAdapter.toGcpBackOff(backoffFactory.backoff());
  do {
    LOG.info("Starting querying {}", query);
    QueryResponse response = BQ_CLIENT.queryWithRetries(query, project);
    if (response.getRows() != null) {
      LOG.info("Got table content with query {}", query);
      return response.getRows();
    }
  } while (BackOffUtils.next(sleeper, backoff));
  LOG.info("Got empty table for query {} with retry {}", query, maxRetry);
  return Collections.emptyList();
}
 
Example #6
Source File: ResilientOperationTest.java    From hadoop-connectors with Apache License 2.0 6 votes vote down vote up
@Test
public void testCallRetriesAndSucceeds() throws Exception {
  MockSleeper sleeper = new MockSleeper();
  ArrayList<Exception> exceptions = new ArrayList<>();
  exceptions.add(new SocketTimeoutException("socket"));
  exceptions.add(new SocketTimeoutException("socket2"));
  exceptions.add(new SocketTimeoutException("socket3"));
  CallableTester<Exception> callTester = new CallableTester<>(exceptions);
  BackOff backoff = new RetryBoundedBackOff(3, new BackOffTester());
  assertThat(
          ResilientOperation.retry(
              callTester, backoff, RetryDeterminer.DEFAULT, Exception.class, sleeper))
      .isEqualTo(3);
  assertThat(callTester.timesCalled()).isEqualTo(4);
  verifySleeper(sleeper, 3);
}
 
Example #7
Source File: BaseWorkflowSample.java    From googleads-shopping-samples with Apache License 2.0 6 votes vote down vote up
protected <T extends GenericJson> T retryFailures(
    AbstractGoogleClientRequest<T> request, BackOff backOff) throws IOException {
  while (true) {
    try {
      return request.execute();
    } catch (GoogleJsonResponseException e) {
      try {
        long nextPause = backOff.nextBackOffMillis();
        if (nextPause == BackOff.STOP) {
          throw e;
        }
        System.out.printf("Operation failed, retrying in %f seconds.%n", nextPause / 1000.0);
        BackOffUtils.next(Sleeper.DEFAULT, backOff);
      } catch (InterruptedException ie) {
        // Just go straight into retry if interrupted.
      }
    }
  }
}
 
Example #8
Source File: RetryConfigTest.java    From firebase-admin-java with Apache License 2.0 6 votes vote down vote up
@Test
public void testExponentialBackOff() throws IOException {
  RetryConfig config = RetryConfig.builder()
      .setMaxIntervalMillis(12000)
      .build();

  BackOff backOff = config.newBackOff();

  assertEquals(500, backOff.nextBackOffMillis());
  assertEquals(1000, backOff.nextBackOffMillis());
  assertEquals(2000, backOff.nextBackOffMillis());
  assertEquals(4000, backOff.nextBackOffMillis());
  assertEquals(8000, backOff.nextBackOffMillis());
  assertEquals(12000, backOff.nextBackOffMillis());
  assertEquals(12000, backOff.nextBackOffMillis());
}
 
Example #9
Source File: ResilientOperationTest.java    From hadoop-connectors with Apache License 2.0 6 votes vote down vote up
@Test
public void testCallRetriesAndFailsWithSocketErrors() throws Exception {
  MockSleeper sleeper = new MockSleeper();
  ArrayList<IOException> exceptions = new ArrayList<>();
  exceptions.add(new SocketTimeoutException("socket"));
  exceptions.add(new SocketTimeoutException("socket"));
  exceptions.add(new IOException("FakeException"));
  CallableTester<IOException> callTester = new CallableTester<>(exceptions);
  BackOff backoff = new RetryBoundedBackOff(5, new BackOffTester());

  IOException thrown =
      assertThrows(
          IOException.class,
          () ->
              ResilientOperation.retry(
                  callTester,
                  backoff,
                  RetryDeterminer.SOCKET_ERRORS,
                  IOException.class,
                  sleeper));
  assertThat(thrown).hasMessageThat().contains("FakeException");

  assertThat(callTester.timesCalled()).isEqualTo(3);
  verifySleeper(sleeper, 2);
}
 
Example #10
Source File: DataflowPipelineJob.java    From beam with Apache License 2.0 6 votes vote down vote up
/**
 * Attempts to get the underlying {@link Job}. Uses exponential backoff on failure up to the
 * maximum number of passed in attempts.
 *
 * @param backoff the {@link BackOff} used to control retries.
 * @param sleeper Object used to do the sleeps between attempts.
 * @return The underlying {@link Job} object.
 * @throws IOException When the maximum number of retries is exhausted, the last exception is
 *     thrown.
 */
private Job getJobWithRetries(BackOff backoff, Sleeper sleeper) throws IOException {
  // Retry loop ends in return or throw
  while (true) {
    try {
      Job job = dataflowClient.getJob(getJobId());
      State currentState = MonitoringUtil.toState(job.getCurrentState());
      if (currentState.isTerminal()) {
        terminalState = currentState;
        replacedByJob =
            new DataflowPipelineJob(
                dataflowClient, job.getReplacedByJobId(), dataflowOptions, transformStepNames);
      }
      return job;
    } catch (IOException exn) {
      LOG.warn("There were problems getting current job status: {}.", exn.getMessage());
      LOG.debug("Exception information:", exn);

      if (!nextBackOff(sleeper, backoff)) {
        throw exn;
      }
    }
  }
}
 
Example #11
Source File: DataflowPipelineJob.java    From beam with Apache License 2.0 6 votes vote down vote up
/**
 * Reset backoff. If duration is limited, calculate time remaining, otherwise just reset retry
 * count.
 *
 * <p>If a total duration for all backoff has been set, update the new cumulative sleep time to be
 * the remaining total backoff duration, stopping if we have already exceeded the allotted time.
 */
private static BackOff resetBackoff(Duration duration, NanoClock nanoClock, long startNanos) {
  BackOff backoff;
  if (duration.isLongerThan(Duration.ZERO)) {
    long nanosConsumed = nanoClock.nanoTime() - startNanos;
    Duration consumed = Duration.millis((nanosConsumed + 999999) / 1000000);
    Duration remaining = duration.minus(consumed);
    if (remaining.isLongerThan(Duration.ZERO)) {
      backoff = getMessagesBackoff(remaining);
    } else {
      backoff = BackOff.STOP_BACKOFF;
    }
  } else {
    backoff = getMessagesBackoff(duration);
  }
  return backoff;
}
 
Example #12
Source File: GcsUtilTest.java    From beam with Apache License 2.0 6 votes vote down vote up
@Test
public void testCreateBucket() throws IOException {
  GcsOptions pipelineOptions = gcsOptionsWithTestCredential();
  GcsUtil gcsUtil = pipelineOptions.getGcsUtil();

  Storage.Buckets mockStorageObjects = Mockito.mock(Storage.Buckets.class);
  Storage mockStorage = Mockito.mock(Storage.class);
  gcsUtil.setStorageClient(mockStorage);

  Storage.Buckets.Insert mockStorageInsert = Mockito.mock(Storage.Buckets.Insert.class);

  BackOff mockBackOff = BackOffAdapter.toGcpBackOff(FluentBackoff.DEFAULT.backoff());

  when(mockStorage.buckets()).thenReturn(mockStorageObjects);
  when(mockStorageObjects.insert(any(String.class), any(Bucket.class)))
      .thenReturn(mockStorageInsert);
  when(mockStorageInsert.execute())
      .thenThrow(new SocketTimeoutException("SocketException"))
      .thenReturn(new Bucket());

  gcsUtil.createBucket("a", new Bucket(), mockBackOff, new FastNanoClockAndSleeper());
}
 
Example #13
Source File: GcsUtilTest.java    From beam with Apache License 2.0 6 votes vote down vote up
@Test
public void testBucketAccessible() throws IOException {
  GcsOptions pipelineOptions = gcsOptionsWithTestCredential();
  GcsUtil gcsUtil = pipelineOptions.getGcsUtil();

  Storage mockStorage = Mockito.mock(Storage.class);
  gcsUtil.setStorageClient(mockStorage);

  Storage.Buckets mockStorageObjects = Mockito.mock(Storage.Buckets.class);
  Storage.Buckets.Get mockStorageGet = Mockito.mock(Storage.Buckets.Get.class);

  BackOff mockBackOff = BackOffAdapter.toGcpBackOff(FluentBackoff.DEFAULT.backoff());

  when(mockStorage.buckets()).thenReturn(mockStorageObjects);
  when(mockStorageObjects.get("testbucket")).thenReturn(mockStorageGet);
  when(mockStorageGet.execute())
      .thenThrow(new SocketTimeoutException("SocketException"))
      .thenReturn(new Bucket());

  assertTrue(
      gcsUtil.bucketAccessible(
          GcsPath.fromComponents("testbucket", "testobject"),
          mockBackOff,
          new FastNanoClockAndSleeper()));
}
 
Example #14
Source File: BigQueryServicesImplTest.java    From beam with Apache License 2.0 6 votes vote down vote up
@Test
public void testExecuteWithRetries() throws IOException, InterruptedException {
  Table testTable = new Table();

  when(response.getContentType()).thenReturn(Json.MEDIA_TYPE);
  when(response.getStatusCode()).thenReturn(200);
  when(response.getContent()).thenReturn(toStream(testTable));

  Table table =
      BigQueryServicesImpl.executeWithRetries(
          bigquery.tables().get("projectId", "datasetId", "tableId"),
          "Failed to get table.",
          Sleeper.DEFAULT,
          BackOff.STOP_BACKOFF,
          BigQueryServicesImpl.ALWAYS_RETRY);

  assertEquals(testTable, table);
  verify(response, times(1)).getStatusCode();
  verify(response, times(1)).getContent();
  verify(response, times(1)).getContentType();
}
 
Example #15
Source File: ResilientOperation.java    From hadoop-connectors with Apache License 2.0 6 votes vote down vote up
/**
 * Determines the amount to sleep for and sleeps if needed.
 *
 * @param backoff BackOff to determine how long to sleep for
 * @param sleeper Used to sleep
 * @param currentException exception that caused the retry and sleep. For logging.
 * @throws InterruptedException if sleep is interrupted
 */
private static boolean nextSleep(BackOff backoff, Sleeper sleeper, Exception currentException)
    throws InterruptedException {
  long backOffTime;
  try {
    backOffTime = backoff.nextBackOffMillis();
  } catch (IOException e) {
    throw new RuntimeException("Failed to to get next back off time", e);
  }
  if (backOffTime == BackOff.STOP) {
    return false;
  }
  logger.atInfo().withCause(currentException).log(
      "Transient exception caught. Sleeping for %d, then retrying.", backOffTime);
  sleeper.sleep(backOffTime);
  return true;
}
 
Example #16
Source File: BigQueryServicesImplTest.java    From beam with Apache License 2.0 6 votes vote down vote up
@Test
public void testCreateTableSucceeds() throws IOException {
  TableReference ref =
      new TableReference().setProjectId("project").setDatasetId("dataset").setTableId("table");
  Table testTable = new Table().setTableReference(ref);
  when(response.getContentType()).thenReturn(Json.MEDIA_TYPE);
  when(response.getStatusCode()).thenReturn(200);
  when(response.getContent()).thenReturn(toStream(testTable));

  BigQueryServicesImpl.DatasetServiceImpl services =
      new BigQueryServicesImpl.DatasetServiceImpl(bigquery, PipelineOptionsFactory.create());
  Table ret =
      services.tryCreateTable(
          testTable, new RetryBoundedBackOff(0, BackOff.ZERO_BACKOFF), Sleeper.DEFAULT);
  assertEquals(testTable, ret);
  verify(response, times(1)).getStatusCode();
  verify(response, times(1)).getContent();
  verify(response, times(1)).getContentType();
}
 
Example #17
Source File: BigQueryServicesImplTest.java    From beam with Apache License 2.0 6 votes vote down vote up
@Test
public void testIsTableEmptyThrows() throws Exception {
  when(response.getContentType()).thenReturn(Json.MEDIA_TYPE);
  when(response.getStatusCode()).thenReturn(401);

  TableReference tableRef =
      new TableReference()
          .setProjectId("projectId")
          .setDatasetId("datasetId")
          .setTableId("tableId");

  BigQueryServicesImpl.DatasetServiceImpl datasetService =
      new BigQueryServicesImpl.DatasetServiceImpl(bigquery, PipelineOptionsFactory.create());

  thrown.expect(IOException.class);
  thrown.expectMessage(String.format("Unable to list table data: %s", tableRef.getTableId()));

  datasetService.isTableEmpty(tableRef, BackOff.STOP_BACKOFF, Sleeper.DEFAULT);
}
 
Example #18
Source File: BigQueryServicesImplTest.java    From beam with Apache License 2.0 6 votes vote down vote up
@Test
public void testIsTableEmptyNoRetryForNotFound() throws IOException, InterruptedException {
  when(response.getContentType()).thenReturn(Json.MEDIA_TYPE);
  when(response.getStatusCode()).thenReturn(404);

  BigQueryServicesImpl.DatasetServiceImpl datasetService =
      new BigQueryServicesImpl.DatasetServiceImpl(bigquery, PipelineOptionsFactory.create());

  TableReference tableRef =
      new TableReference()
          .setProjectId("projectId")
          .setDatasetId("datasetId")
          .setTableId("tableId");

  thrown.expect(IOException.class);
  thrown.expectMessage(String.format("Unable to list table data: %s", tableRef.getTableId()));

  try {
    datasetService.isTableEmpty(tableRef, BackOff.ZERO_BACKOFF, Sleeper.DEFAULT);
  } finally {
    verify(response, times(1)).getStatusCode();
    verify(response, times(1)).getContent();
    verify(response, times(1)).getContentType();
  }
}
 
Example #19
Source File: BigQueryServicesImplTest.java    From beam with Apache License 2.0 6 votes vote down vote up
@Test
public void testIsTableEmptySucceeds() throws Exception {
  TableReference tableRef =
      new TableReference()
          .setProjectId("projectId")
          .setDatasetId("datasetId")
          .setTableId("tableId");

  TableDataList testDataList = new TableDataList().setRows(ImmutableList.of(new TableRow()));

  // First response is 403 rate limited, second response has valid payload.
  when(response.getContentType()).thenReturn(Json.MEDIA_TYPE);
  when(response.getStatusCode()).thenReturn(403).thenReturn(200);
  when(response.getContent())
      .thenReturn(toStream(errorWithReasonAndStatus("rateLimitExceeded", 403)))
      .thenReturn(toStream(testDataList));

  BigQueryServicesImpl.DatasetServiceImpl datasetService =
      new BigQueryServicesImpl.DatasetServiceImpl(bigquery, PipelineOptionsFactory.create());

  assertFalse(datasetService.isTableEmpty(tableRef, BackOff.ZERO_BACKOFF, Sleeper.DEFAULT));

  verify(response, times(2)).getStatusCode();
  verify(response, times(2)).getContent();
  verify(response, times(2)).getContentType();
}
 
Example #20
Source File: BigQueryServicesImplTest.java    From beam with Apache License 2.0 6 votes vote down vote up
@Test
public void testGetTableThrows() throws Exception {
  when(response.getContentType()).thenReturn(Json.MEDIA_TYPE);
  when(response.getStatusCode()).thenReturn(401);

  TableReference tableRef =
      new TableReference()
          .setProjectId("projectId")
          .setDatasetId("datasetId")
          .setTableId("tableId");

  thrown.expect(IOException.class);
  thrown.expectMessage(String.format("Unable to get table: %s", tableRef.getTableId()));

  BigQueryServicesImpl.DatasetServiceImpl datasetService =
      new BigQueryServicesImpl.DatasetServiceImpl(bigquery, PipelineOptionsFactory.create());
  datasetService.getTable(tableRef, null, BackOff.STOP_BACKOFF, Sleeper.DEFAULT);
}
 
Example #21
Source File: BigQueryServicesImplTest.java    From beam with Apache License 2.0 6 votes vote down vote up
@Test
public void testGetTableNotFound() throws IOException, InterruptedException {
  when(response.getContentType()).thenReturn(Json.MEDIA_TYPE);
  when(response.getStatusCode()).thenReturn(404);

  BigQueryServicesImpl.DatasetServiceImpl datasetService =
      new BigQueryServicesImpl.DatasetServiceImpl(bigquery, PipelineOptionsFactory.create());

  TableReference tableRef =
      new TableReference()
          .setProjectId("projectId")
          .setDatasetId("datasetId")
          .setTableId("tableId");
  Table table = datasetService.getTable(tableRef, null, BackOff.ZERO_BACKOFF, Sleeper.DEFAULT);

  assertNull(table);
  verify(response, times(1)).getStatusCode();
  verify(response, times(1)).getContent();
  verify(response, times(1)).getContentType();
}
 
Example #22
Source File: BigQueryServicesImpl.java    From beam with Apache License 2.0 6 votes vote down vote up
@VisibleForTesting
@Nullable
Table getTable(
    TableReference ref, @Nullable List<String> selectedFields, BackOff backoff, Sleeper sleeper)
    throws IOException, InterruptedException {
  Tables.Get get =
      client.tables().get(ref.getProjectId(), ref.getDatasetId(), ref.getTableId());
  if (selectedFields != null && !selectedFields.isEmpty()) {
    get.setSelectedFields(String.join(",", selectedFields));
  }
  try {
    return executeWithRetries(
        get,
        String.format(
            "Unable to get table: %s, aborting after %d retries.",
            ref.getTableId(), MAX_RPC_RETRIES),
        sleeper,
        backoff,
        DONT_RETRY_NOT_FOUND);
  } catch (IOException e) {
    if (errorExtractor.itemNotFound(e)) {
      return null;
    }
    throw e;
  }
}
 
Example #23
Source File: DeleteVariants.java    From dataflow-java with Apache License 2.0 6 votes vote down vote up
@ProcessElement
public void processElement(DoFn<String, Integer>.ProcessContext context) throws Exception {
  String variantId = context.element();
  // Call the deletion operation via exponential backoff so that "Rate Limit Exceeded"
  // quota issues do not cause the pipeline to fail.
  ExponentialBackOff backoff = new ExponentialBackOff.Builder().build();
  while (true) {
    try {
      genomics.variants().delete(variantId).execute();
      Metrics.counter(DeleteVariantFn.class, "Number of variants deleted").inc();
      context.output(1);
      return;
    } catch (Exception e) {
      if (e.getMessage().startsWith("429 Too Many Requests")) {
        LOG.warn("Backing-off per: ", e);
        long backOffMillis = backoff.nextBackOffMillis();
        if (backOffMillis == BackOff.STOP) {
          throw e;
        }
        Thread.sleep(backOffMillis);
      } else {
        throw e;
      }
    }
  }
}
 
Example #24
Source File: BigQueryServicesImplTest.java    From beam with Apache License 2.0 6 votes vote down vote up
@Test
public void testGetJobSucceeds() throws Exception {
  Job testJob = new Job();
  testJob.setStatus(new JobStatus());

  when(response.getContentType()).thenReturn(Json.MEDIA_TYPE);
  when(response.getStatusCode()).thenReturn(200);
  when(response.getContent()).thenReturn(toStream(testJob));

  BigQueryServicesImpl.JobServiceImpl jobService =
      new BigQueryServicesImpl.JobServiceImpl(bigquery);
  JobReference jobRef = new JobReference().setProjectId("projectId").setJobId("jobId");
  Job job = jobService.getJob(jobRef, Sleeper.DEFAULT, BackOff.ZERO_BACKOFF);

  assertEquals(testJob, job);
  verify(response, times(1)).getStatusCode();
  verify(response, times(1)).getContent();
  verify(response, times(1)).getContentType();
}
 
Example #25
Source File: BigQueryServicesImpl.java    From beam with Apache License 2.0 6 votes vote down vote up
@VisibleForTesting
boolean isTableEmpty(TableReference tableRef, BackOff backoff, Sleeper sleeper)
    throws IOException, InterruptedException {
  TableDataList dataList =
      executeWithRetries(
          client
              .tabledata()
              .list(tableRef.getProjectId(), tableRef.getDatasetId(), tableRef.getTableId()),
          String.format(
              "Unable to list table data: %s, aborting after %d retries.",
              tableRef.getTableId(), MAX_RPC_RETRIES),
          sleeper,
          backoff,
          DONT_RETRY_NOT_FOUND);
  return dataList.getRows() == null || dataList.getRows().isEmpty();
}
 
Example #26
Source File: BigQueryServicesImpl.java    From beam with Apache License 2.0 6 votes vote down vote up
@VisibleForTesting
static <T> T executeWithRetries(
    AbstractGoogleClientRequest<T> request,
    String errorMessage,
    Sleeper sleeper,
    BackOff backoff,
    SerializableFunction<IOException, Boolean> shouldRetry)
    throws IOException, InterruptedException {
  Exception lastException = null;
  do {
    try {
      return request.execute();
    } catch (IOException e) {
      lastException = e;
      if (!shouldRetry.apply(e)) {
        break;
      }
      LOG.info("Ignore the error and retry the request.", e);
    }
  } while (nextBackOff(sleeper, backoff));
  throw new IOException(errorMessage, lastException);
}
 
Example #27
Source File: BigQueryServicesImplTest.java    From beam with Apache License 2.0 6 votes vote down vote up
/** Tests that {@link BigQueryServicesImpl.JobServiceImpl#pollJob} returns UNKNOWN. */
@Test
public void testPollJobUnknown() throws IOException, InterruptedException {
  Job testJob = new Job();
  testJob.setStatus(new JobStatus());

  when(response.getContentType()).thenReturn(Json.MEDIA_TYPE);
  when(response.getStatusCode()).thenReturn(200);
  when(response.getContent()).thenReturn(toStream(testJob));

  BigQueryServicesImpl.JobServiceImpl jobService =
      new BigQueryServicesImpl.JobServiceImpl(bigquery);
  JobReference jobRef = new JobReference().setProjectId("projectId").setJobId("jobId");
  Job job = jobService.pollJob(jobRef, Sleeper.DEFAULT, BackOff.STOP_BACKOFF);

  assertEquals(null, job);
  verify(response, times(1)).getStatusCode();
  verify(response, times(1)).getContent();
  verify(response, times(1)).getContentType();
}
 
Example #28
Source File: BigQueryServicesImplTest.java    From beam with Apache License 2.0 6 votes vote down vote up
/** Tests that {@link BigQueryServicesImpl.JobServiceImpl#pollJob} fails. */
@Test
public void testPollJobFailed() throws IOException, InterruptedException {
  Job testJob = new Job();
  testJob.setStatus(new JobStatus().setState("DONE").setErrorResult(new ErrorProto()));

  when(response.getContentType()).thenReturn(Json.MEDIA_TYPE);
  when(response.getStatusCode()).thenReturn(200);
  when(response.getContent()).thenReturn(toStream(testJob));

  BigQueryServicesImpl.JobServiceImpl jobService =
      new BigQueryServicesImpl.JobServiceImpl(bigquery);
  JobReference jobRef = new JobReference().setProjectId("projectId").setJobId("jobId");
  Job job = jobService.pollJob(jobRef, Sleeper.DEFAULT, BackOff.ZERO_BACKOFF);

  assertEquals(testJob, job);
  verify(response, times(1)).getStatusCode();
  verify(response, times(1)).getContent();
  verify(response, times(1)).getContentType();
}
 
Example #29
Source File: FakeJobService.java    From beam with Apache License 2.0 6 votes vote down vote up
@Override
public Job pollJob(JobReference jobRef, int maxAttempts) throws InterruptedException {
  BackOff backoff =
      BackOffAdapter.toGcpBackOff(
          FluentBackoff.DEFAULT
              .withMaxRetries(maxAttempts)
              .withInitialBackoff(Duration.millis(10))
              .withMaxBackoff(Duration.standardSeconds(1))
              .backoff());
  Sleeper sleeper = Sleeper.DEFAULT;
  try {
    do {
      Job job = getJob(jobRef);
      if (job != null) {
        JobStatus status = job.getStatus();
        if (status != null
            && ("DONE".equals(status.getState()) || "FAILED".equals(status.getState()))) {
          return job;
        }
      }
    } while (BackOffUtils.next(sleeper, backoff));
  } catch (IOException e) {
    return null;
  }
  return null;
}
 
Example #30
Source File: BigQueryServicesImplTest.java    From beam with Apache License 2.0 6 votes vote down vote up
/** Tests that {@link BigQueryServicesImpl.JobServiceImpl#pollJob} succeeds. */
@Test
public void testPollJobSucceeds() throws IOException, InterruptedException {
  Job testJob = new Job();
  testJob.setStatus(new JobStatus().setState("DONE"));

  when(response.getContentType()).thenReturn(Json.MEDIA_TYPE);
  when(response.getStatusCode()).thenReturn(200);
  when(response.getContent()).thenReturn(toStream(testJob));

  BigQueryServicesImpl.JobServiceImpl jobService =
      new BigQueryServicesImpl.JobServiceImpl(bigquery);
  JobReference jobRef = new JobReference().setProjectId("projectId").setJobId("jobId");
  Job job = jobService.pollJob(jobRef, Sleeper.DEFAULT, BackOff.ZERO_BACKOFF);

  assertEquals(testJob, job);
  verify(response, times(1)).getStatusCode();
  verify(response, times(1)).getContent();
  verify(response, times(1)).getContentType();
}