org.apache.beam.sdk.util.BackOffUtils Java Examples

The following examples show how to use org.apache.beam.sdk.util.BackOffUtils. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example #1
Source File: PubsubHelper.java    From beam with Apache License 2.0 6 votes vote down vote up
/**
 * Create a topic from short name. Delete it if it already exists. Ensure the topic will be
 * deleted on cleanup. Return full topic name.
 */
public TopicPath createTopic(String shortTopic) throws IOException {
  TopicPath topic = PubsubClient.topicPathFromName(project, shortTopic);
  while (true) {
    try {
      NexmarkUtils.console("create topic %s", topic);
      pubsubClient.createTopic(topic);
      createdTopics.add(topic);
      return topic;
    } catch (GoogleJsonResponseException ex) {
      NexmarkUtils.console("attempting to cleanup topic %s", topic);
      pubsubClient.deleteTopic(topic);
      try {
        if (!BackOffUtils.next(sleeper, backOff)) {
          NexmarkUtils.console("too many retries for creating topic %s", topic);
          throw ex;
        }
      } catch (InterruptedException in) {
        throw new IOException(in);
      }
    }
  }
}
 
Example #2
Source File: PubsubHelper.java    From beam with Apache License 2.0 6 votes vote down vote up
/**
 * Create a topic from short name if it does not already exist. The topic will not be deleted on
 * cleanup. Return full topic name.
 */
public TopicPath createOrReuseTopic(String shortTopic) throws IOException {
  TopicPath topic = PubsubClient.topicPathFromName(project, shortTopic);
  while (true) {
    try {
      NexmarkUtils.console("create topic %s", topic);
      pubsubClient.createTopic(topic);
      return topic;
    } catch (GoogleJsonResponseException ex) {
      if (topicExists(shortTopic)) {
        NexmarkUtils.console("topic %s already exists", topic);
        return topic;
      }
      try {
        if (!BackOffUtils.next(sleeper, backOff)) {
          NexmarkUtils.console("too many retries for creating/reusing topic %s", topic);
          throw ex;
        }
      } catch (InterruptedException in) {
        throw new IOException(in);
      }
    }
  }
}
 
Example #3
Source File: PubsubHelper.java    From beam with Apache License 2.0 6 votes vote down vote up
/**
 * Create subscription from short name. Ensure the subscription will be deleted on cleanup. Return
 * full subscription name.
 */
public SubscriptionPath createSubscription(String shortTopic, String shortSubscription)
    throws IOException {
  TopicPath topic = PubsubClient.topicPathFromName(project, shortTopic);
  SubscriptionPath subscription =
      PubsubClient.subscriptionPathFromName(project, shortSubscription);
  while (true) {
    try {
      NexmarkUtils.console("create subscription %s", subscription);
      pubsubClient.createSubscription(topic, subscription, 60);
      createdSubscriptions.add(subscription);
      return subscription;
    } catch (GoogleJsonResponseException ex) {
      NexmarkUtils.console("attempting to cleanup subscription %s", subscription);
      pubsubClient.deleteSubscription(subscription);
      try {
        if (!BackOffUtils.next(sleeper, backOff)) {
          NexmarkUtils.console("too many retries for creating subscription %s", subscription);
          throw ex;
        }
      } catch (InterruptedException in) {
        throw new IOException(in);
      }
    }
  }
}
 
Example #4
Source File: DatastoreV1.java    From beam with Apache License 2.0 6 votes vote down vote up
private RunQueryResponse runQueryWithRetries(RunQueryRequest request) throws Exception {
  Sleeper sleeper = Sleeper.DEFAULT;
  BackOff backoff = RUNQUERY_BACKOFF.backoff();
  while (true) {
    try {
      RunQueryResponse response = datastore.runQuery(request);
      rpcSuccesses.inc();
      return response;
    } catch (DatastoreException exception) {
      rpcErrors.inc();

      if (NON_RETRYABLE_ERRORS.contains(exception.getCode())) {
        throw exception;
      }
      if (!BackOffUtils.next(sleeper, backoff)) {
        LOG.error("Aborting after {} retries.", MAX_RETRIES);
        throw exception;
      }
    }
  }
}
 
Example #5
Source File: BackOffExecutor.java    From feast with Apache License 2.0 6 votes vote down vote up
private void execute(Retriable retriable, FluentBackoff backoff) throws Exception {
  Sleeper sleeper = Sleeper.DEFAULT;
  BackOff backOff = backoff.backoff();
  while (true) {
    try {
      retriable.execute();
      break;
    } catch (Exception e) {
      if (retriable.isExceptionRetriable(e) && BackOffUtils.next(sleeper, backOff)) {
        retriable.cleanUpAfterFailure();
      } else {
        throw e;
      }
    }
  }
}
 
Example #6
Source File: GrpcWindmillServer.java    From beam with Apache License 2.0 5 votes vote down vote up
private <ResponseT> ResponseT callWithBackoff(Supplier<ResponseT> function) {
  BackOff backoff = grpcBackoff();
  int rpcErrors = 0;
  while (true) {
    try {
      return function.get();
    } catch (StatusRuntimeException e) {
      try {
        if (++rpcErrors % 20 == 0) {
          LOG.warn(
              "Many exceptions calling gRPC. Last exception: {} with status {}",
              e,
              e.getStatus());
        }
        if (!BackOffUtils.next(Sleeper.DEFAULT, backoff)) {
          throw new WindmillServerStub.RpcException(e);
        }
      } catch (IOException | InterruptedException i) {
        if (i instanceof InterruptedException) {
          Thread.currentThread().interrupt();
        }
        WindmillServerStub.RpcException rpcException = new WindmillServerStub.RpcException(e);
        rpcException.addSuppressed(i);
        throw rpcException;
      }
    }
  }
}
 
Example #7
Source File: StreamingDataflowWorker.java    From beam with Apache License 2.0 5 votes vote down vote up
private void getConfig(String computation) {
  BackOff backoff =
      FluentBackoff.DEFAULT
          .withInitialBackoff(Duration.millis(100))
          .withMaxBackoff(Duration.standardMinutes(1))
          .withMaxCumulativeBackoff(Duration.standardMinutes(5))
          .backoff();
  while (running.get()) {
    try {
      if (windmillServiceEnabled) {
        getConfigFromDataflowService(computation);
      } else {
        getConfigFromWindmill(computation);
      }
      return;
    } catch (IllegalArgumentException | IOException e) {
      LOG.warn("Error fetching config: ", e);
      try {
        if (!BackOffUtils.next(Sleeper.DEFAULT, backoff)) {
          return;
        }
      } catch (IOException ioe) {
        LOG.warn("Error backing off, will not retry: ", ioe);
        return;
      } catch (InterruptedException ie) {
        Thread.currentThread().interrupt();
        return;
      }
    }
  }
}
 
Example #8
Source File: SnsIO.java    From beam with Apache License 2.0 5 votes vote down vote up
@ProcessElement
public void processElement(ProcessContext context) throws Exception {
  PublishRequest request =
      (PublishRequest) spec.getPublishRequestFn().apply(context.element());
  Sleeper sleeper = Sleeper.DEFAULT;
  BackOff backoff = retryBackoff.backoff();
  int attempt = 0;
  while (true) {
    attempt++;
    try {
      PublishResponse pr = producer.publish(request);
      context.output(pr);
      break;
    } catch (Exception ex) {
      // Fail right away if there is no retry configuration
      if (spec.getRetryConfiguration() == null
          || !spec.getRetryConfiguration().getRetryPredicate().test(ex)) {
        SNS_WRITE_FAILURES.inc();
        LOG.info("Unable to publish message {} due to {} ", request.message(), ex);
        throw new IOException("Error writing to SNS (no attempt made to retry)", ex);
      }

      if (!BackOffUtils.next(sleeper, backoff)) {
        throw new IOException(
            String.format(
                "Error writing to SNS after %d attempt(s). No more attempts allowed",
                attempt),
            ex);
      } else {
        // Note: this used in test cases to verify behavior
        LOG.warn(String.format(RETRY_ATTEMPT_LOG, attempt), ex);
      }
    }
  }
}
 
Example #9
Source File: JdbcIO.java    From beam with Apache License 2.0 5 votes vote down vote up
private void executeBatch() throws SQLException, IOException, InterruptedException {
  if (records.isEmpty()) {
    return;
  }
  Sleeper sleeper = Sleeper.DEFAULT;
  BackOff backoff = retryBackOff.backoff();
  while (true) {
    try (PreparedStatement preparedStatement =
        connection.prepareStatement(spec.getStatement().get())) {
      try {
        // add each record in the statement batch
        for (T record : records) {
          processRecord(record, preparedStatement);
        }
        // execute the batch
        preparedStatement.executeBatch();
        // commit the changes
        connection.commit();
        break;
      } catch (SQLException exception) {
        if (!spec.getRetryStrategy().apply(exception)) {
          throw exception;
        }
        LOG.warn("Deadlock detected, retrying", exception);
        // clean up the statement batch and the connection state
        preparedStatement.clearBatch();
        connection.rollback();
        if (!BackOffUtils.next(sleeper, backoff)) {
          // we tried the max number of times
          throw exception;
        }
      }
    }
  }
  records.clear();
}
 
Example #10
Source File: SnsIO.java    From beam with Apache License 2.0 5 votes vote down vote up
@ProcessElement
public void processElement(ProcessContext context) throws Exception {
  PublishRequest request = context.element();
  Sleeper sleeper = Sleeper.DEFAULT;
  BackOff backoff = retryBackoff.backoff();
  int attempt = 0;
  while (true) {
    attempt++;
    try {
      PublishResult pr = producer.publish(request);
      context.output(pr);
      break;
    } catch (Exception ex) {
      // Fail right away if there is no retry configuration
      if (spec.getRetryConfiguration() == null
          || !spec.getRetryConfiguration().getRetryPredicate().test(ex)) {
        SNS_WRITE_FAILURES.inc();
        LOG.info("Unable to publish message {} due to {} ", request.getMessage(), ex);
        throw new IOException("Error writing to SNS (no attempt made to retry)", ex);
      }

      if (!BackOffUtils.next(sleeper, backoff)) {
        throw new IOException(
            String.format(
                "Error writing to SNS after %d attempt(s). No more attempts allowed",
                attempt),
            ex);
      } else {
        // Note: this used in test cases to verify behavior
        LOG.warn(String.format(RETRY_ATTEMPT_LOG, attempt), ex);
      }
    }
  }
}
 
Example #11
Source File: SimplifiedKinesisClient.java    From beam with Apache License 2.0 5 votes vote down vote up
public List<Shard> listShards(final String streamName) throws TransientKinesisException {
  return wrapExceptions(
      () -> {
        List<Shard> shards = Lists.newArrayList();
        String lastShardId = null;

        // DescribeStream has limits that can be hit fairly easily if we are attempting
        // to configure multiple KinesisIO inputs in the same account. Retry up to
        // LIST_SHARDS_DESCRIBE_STREAM_MAX_ATTEMPTS times if we end up hitting that limit.
        //
        // Only pass the wrapped exception up once that limit is reached. Use FluentBackoff
        // to implement the retry policy.
        FluentBackoff retryBackoff =
            FluentBackoff.DEFAULT
                .withMaxRetries(LIST_SHARDS_DESCRIBE_STREAM_MAX_ATTEMPTS)
                .withInitialBackoff(LIST_SHARDS_DESCRIBE_STREAM_INITIAL_BACKOFF);
        StreamDescription description = null;
        do {
          BackOff backoff = retryBackoff.backoff();
          Sleeper sleeper = Sleeper.DEFAULT;
          while (true) {
            try {
              description =
                  kinesis.describeStream(streamName, lastShardId).getStreamDescription();
              break;
            } catch (LimitExceededException exc) {
              if (!BackOffUtils.next(sleeper, backoff)) {
                throw exc;
              }
            }
          }

          shards.addAll(description.getShards());
          lastShardId = shards.get(shards.size() - 1).getShardId();
        } while (description.getHasMoreShards());

        return shards;
      });
}
 
Example #12
Source File: ClickHouseIO.java    From beam with Apache License 2.0 5 votes vote down vote up
private void flush() throws Exception {
  BackOff backOff = retryBackoff.backoff();
  int attempt = 0;

  if (buffer.isEmpty()) {
    return;
  }

  batchSize.update(buffer.size());

  while (true) {
    try (ClickHouseStatement statement = connection.createStatement()) {
      statement.sendRowBinaryStream(
          insertSql(schema(), table()),
          stream -> {
            for (Row row : buffer) {
              ClickHouseWriter.writeRow(stream, schema(), row);
            }
          });
      buffer.clear();
      break;
    } catch (SQLException e) {
      if (!BackOffUtils.next(Sleeper.DEFAULT, backOff)) {
        throw e;
      } else {
        retries.inc();
        LOG.warn(String.format(RETRY_ATTEMPT_LOG, attempt), e);
        attempt++;
      }
    }
  }
}
 
Example #13
Source File: V1TestUtil.java    From beam with Apache License 2.0 5 votes vote down vote up
private void flushBatch() throws DatastoreException, IOException, InterruptedException {
  LOG.info("Writing batch of {} entities", entities.size());
  Sleeper sleeper = Sleeper.DEFAULT;
  BackOff backoff =
      FluentBackoff.DEFAULT
          .withMaxRetries(MAX_RETRIES)
          .withInitialBackoff(INITIAL_BACKOFF)
          .backoff();

  while (true) {
    // Batch mutate entities.
    try {
      CommitRequest.Builder commitRequest = CommitRequest.newBuilder();
      for (Entity entity : entities) {
        commitRequest.addMutations(mutationBuilder.apply(entity));
      }
      commitRequest.setMode(CommitRequest.Mode.NON_TRANSACTIONAL);
      datastore.commit(commitRequest.build());
      // Break if the commit threw no exception.
      break;
    } catch (DatastoreException exception) {
      LOG.error(
          "Error writing to the Datastore ({}): {}",
          exception.getCode(),
          exception.getMessage());
      if (!BackOffUtils.next(sleeper, backoff)) {
        LOG.error("Aborting after {} retries.", MAX_RETRIES);
        throw exception;
      }
    }
  }
  LOG.info("Successfully wrote {} entities", entities.size());
  entities.clear();
}
 
Example #14
Source File: DynamoDBIO.java    From beam with Apache License 2.0 4 votes vote down vote up
private void flushBatch() throws IOException, InterruptedException {
  if (batch.isEmpty()) {
    return;
  }

  try {
    // Since each element is a KV<tableName, writeRequest> in the batch, we need to group them
    // by tableName
    Map<String, List<WriteRequest>> mapTableRequest =
        batch.stream()
            .collect(
                Collectors.groupingBy(
                    KV::getKey, Collectors.mapping(KV::getValue, Collectors.toList())));

    BatchWriteItemRequest batchRequest = new BatchWriteItemRequest();
    mapTableRequest
        .entrySet()
        .forEach(
            entry -> batchRequest.addRequestItemsEntry(entry.getKey(), entry.getValue()));

    Sleeper sleeper = Sleeper.DEFAULT;
    BackOff backoff = retryBackoff.backoff();
    int attempt = 0;
    while (true) {
      attempt++;
      try {
        client.batchWriteItem(batchRequest);
        break;
      } catch (Exception ex) {
        // Fail right away if there is no retry configuration
        if (spec.getRetryConfiguration() == null
            || !spec.getRetryConfiguration().getRetryPredicate().test(ex)) {
          DYNAMO_DB_WRITE_FAILURES.inc();
          LOG.info(
              "Unable to write batch items {} due to {} ",
              batchRequest.getRequestItems().entrySet(),
              ex);
          throw new IOException("Error writing to DynamoDB (no attempt made to retry)", ex);
        }

        if (!BackOffUtils.next(sleeper, backoff)) {
          throw new IOException(
              String.format(
                  "Error writing to DynamoDB after %d attempt(s). No more attempts allowed",
                  attempt),
              ex);
        } else {
          // Note: this used in test cases to verify behavior
          LOG.warn(String.format(RETRY_ATTEMPT_LOG, attempt), ex);
        }
      }
    }
  } finally {
    batch.clear();
  }
}
 
Example #15
Source File: BaseClickHouseTest.java    From beam with Apache License 2.0 4 votes vote down vote up
@BeforeClass
public static void setup() throws IOException, InterruptedException {
  // network sharing doesn't work with ClassRule
  network = Network.newNetwork();

  zookeeper =
      new GenericContainer<>("zookeeper:3.4.13")
          .withStartupAttempts(10)
          .withExposedPorts(2181)
          .withNetwork(network)
          .withNetworkAliases("zookeeper");

  // so far zookeeper container always starts successfully, so no extra retries
  zookeeper.start();

  clickHouse =
      (ClickHouseContainer)
          new ClickHouseContainer(CLICKHOUSE_IMAGE)
              .withStartupAttempts(10)
              .withCreateContainerCmdModifier(
                  // type inference for `(CreateContainerCmd) -> cmd.` doesn't work
                  cmd ->
                      ((CreateContainerCmd) cmd)
                          .withMemory(256 * 1024 * 1024L)
                          .withMemorySwap(4L * 1024 * 1024 * 1024L))
              .withNetwork(network)
              .withClasspathResourceMapping(
                  "config.d/zookeeper_default.xml",
                  "/etc/clickhouse-server/config.d/zookeeper_default.xml",
                  BindMode.READ_ONLY);

  BackOff backOff =
      FluentBackoff.DEFAULT
          .withMaxRetries(3)
          .withInitialBackoff(Duration.standardSeconds(15))
          .backoff();

  // try to start clickhouse-server a couple of times, see BEAM-6639
  while (true) {
    try {
      Unreliables.retryUntilSuccess(
          10,
          () -> {
            DockerClientFactory.instance()
                .checkAndPullImage(DockerClientFactory.instance().client(), CLICKHOUSE_IMAGE);

            return null;
          });

      clickHouse.start();
      break;
    } catch (Exception e) {
      if (!BackOffUtils.next(Sleeper.DEFAULT, backOff)) {
        throw e;
      } else {
        List<Image> images =
            DockerClientFactory.instance().client().listImagesCmd().withShowAll(true).exec();
        String listImagesOutput = "listImagesCmd:\n" + Joiner.on('\n').join(images) + "\n";

        LOG.warn("failed to start clickhouse-server\n\n" + listImagesOutput, e);
      }
    }
  }
}
 
Example #16
Source File: DynamoDBIO.java    From beam with Apache License 2.0 4 votes vote down vote up
private void flushBatch() throws IOException, InterruptedException {
  if (batch.isEmpty()) {
    return;
  }

  try {
    // Since each element is a KV<tableName, writeRequest> in the batch, we need to group them
    // by tableName
    Map<String, List<WriteRequest>> mapTableRequest =
        batch.stream()
            .collect(
                Collectors.groupingBy(
                    KV::getKey, Collectors.mapping(KV::getValue, Collectors.toList())));

    BatchWriteItemRequest batchRequest =
        BatchWriteItemRequest.builder().requestItems(mapTableRequest).build();

    Sleeper sleeper = Sleeper.DEFAULT;
    BackOff backoff = retryBackoff.backoff();
    int attempt = 0;
    while (true) {
      attempt++;
      try {
        client.batchWriteItem(batchRequest);
        break;
      } catch (Exception ex) {
        // Fail right away if there is no retry configuration
        if (spec.getRetryConfiguration() == null
            || !spec.getRetryConfiguration().getRetryPredicate().test(ex)) {
          DYNAMO_DB_WRITE_FAILURES.inc();
          LOG.info(
              "Unable to write batch items {} due to {} ",
              batchRequest.requestItems().entrySet(),
              ex);
          throw new IOException("Error writing to DynamoDB (no attempt made to retry)", ex);
        }

        if (!BackOffUtils.next(sleeper, backoff)) {
          throw new IOException(
              String.format(
                  "Error writing to DynamoDB after %d attempt(s). No more attempts allowed",
                  attempt),
              ex);
        } else {
          // Note: this used in test cases to verify behavior
          LOG.warn(String.format(RETRY_ATTEMPT_LOG, attempt), ex);
        }
      }
    }
  } finally {
    batch.clear();
  }
}
 
Example #17
Source File: SolrIO.java    From beam with Apache License 2.0 4 votes vote down vote up
private void flushBatch() throws IOException, InterruptedException {
  if (batch.isEmpty()) {
    return;
  }
  try {
    UpdateRequest updateRequest = new UpdateRequest();
    updateRequest.add(batch);

    Sleeper sleeper = Sleeper.DEFAULT;
    BackOff backoff = retryBackoff.backoff();
    int attempt = 0;
    while (true) {
      attempt++;
      try {
        solrClient.process(spec.getCollection(), updateRequest);
        break;
      } catch (Exception exception) {

        // fail immediately if no retry configuration doesn't handle this
        if (spec.getRetryConfiguration() == null
            || !spec.getRetryConfiguration().getRetryPredicate().test(exception)) {
          throw new IOException(
              "Error writing to Solr (no attempt made to retry)", exception);
        }

        // see if we can pause and try again
        if (!BackOffUtils.next(sleeper, backoff)) {
          throw new IOException(
              String.format(
                  "Error writing to Solr after %d attempt(s). No more attempts allowed",
                  attempt),
              exception);

        } else {
          // Note: this used in test cases to verify behavior
          LOG.warn(String.format(RETRY_ATTEMPT_LOG, attempt), exception);
        }
      }
    }
  } finally {
    batch.clear();
  }
}
 
Example #18
Source File: DatastoreV1.java    From beam with Apache License 2.0 4 votes vote down vote up
/**
 * Writes a batch of mutations to Cloud Datastore.
 *
 * <p>If a commit fails, it will be retried up to {@link #MAX_RETRIES} times. All mutations in
 * the batch will be committed again, even if the commit was partially successful. If the retry
 * limit is exceeded, the last exception from Cloud Datastore will be thrown.
 *
 * @throws DatastoreException if the commit fails or IOException or InterruptedException if
 *     backing off between retries fails.
 */
private void flushBatch() throws DatastoreException, IOException, InterruptedException {
  LOG.debug("Writing batch of {} mutations", mutations.size());
  Sleeper sleeper = Sleeper.DEFAULT;
  BackOff backoff = BUNDLE_WRITE_BACKOFF.backoff();

  while (true) {
    // Batch upsert entities.
    CommitRequest.Builder commitRequest = CommitRequest.newBuilder();
    commitRequest.addAllMutations(mutations);
    commitRequest.setMode(CommitRequest.Mode.NON_TRANSACTIONAL);
    long startTime = System.currentTimeMillis(), endTime;

    if (throttler.throttleRequest(startTime)) {
      LOG.info("Delaying request due to previous failures");
      throttledSeconds.inc(WriteBatcherImpl.DATASTORE_BATCH_TARGET_LATENCY_MS / 1000);
      sleeper.sleep(WriteBatcherImpl.DATASTORE_BATCH_TARGET_LATENCY_MS);
      continue;
    }

    try {
      datastore.commit(commitRequest.build());
      endTime = System.currentTimeMillis();

      writeBatcher.addRequestLatency(endTime, endTime - startTime, mutations.size());
      throttler.successfulRequest(startTime);
      rpcSuccesses.inc();

      // Break if the commit threw no exception.
      break;
    } catch (DatastoreException exception) {
      if (exception.getCode() == Code.DEADLINE_EXCEEDED) {
        /* Most errors are not related to request size, and should not change our expectation of
         * the latency of successful requests. DEADLINE_EXCEEDED can be taken into
         * consideration, though. */
        endTime = System.currentTimeMillis();
        writeBatcher.addRequestLatency(endTime, endTime - startTime, mutations.size());
      }
      // Only log the code and message for potentially-transient errors. The entire exception
      // will be propagated upon the last retry.
      LOG.error(
          "Error writing batch of {} mutations to Datastore ({}): {}",
          mutations.size(),
          exception.getCode(),
          exception.getMessage());
      rpcErrors.inc();

      if (NON_RETRYABLE_ERRORS.contains(exception.getCode())) {
        throw exception;
      }
      if (!BackOffUtils.next(sleeper, backoff)) {
        LOG.error("Aborting after {} retries.", MAX_RETRIES);
        throw exception;
      }
    }
  }
  LOG.debug("Successfully wrote {} mutations", mutations.size());
  mutations.clear();
  mutationsSize = 0;
}