Java Code Examples for com.amazonaws.ClientConfiguration#setRetryPolicy()

The following examples show how to use com.amazonaws.ClientConfiguration#setRetryPolicy() . You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source File: GlueTestClientFactory.java    From aws-glue-data-catalog-client-for-apache-hive-metastore with Apache License 2.0 7 votes vote down vote up 
private static ClientConfiguration createGatewayTimeoutRetryableConfiguration() {
  ClientConfiguration retryableConfig = new ClientConfiguration();
  RetryPolicy.RetryCondition retryCondition = new PredefinedRetryPolicies.SDKDefaultRetryCondition() {
    @Override
    public boolean shouldRetry(AmazonWebServiceRequest originalRequest, AmazonClientException exception,
                               int retriesAttempted) {
      if (super.shouldRetry(originalRequest, exception, retriesAttempted)) {
        return true;
      }
      if (exception != null && exception instanceof AmazonServiceException) {
        AmazonServiceException ase = (AmazonServiceException) exception;
        if (ase.getStatusCode() == SC_GATEWAY_TIMEOUT) {
          return true;
        }
      }
      return false;
    }
  };
  RetryPolicy retryPolicy = new RetryPolicy(retryCondition, PredefinedRetryPolicies.DEFAULT_BACKOFF_STRATEGY,
                                                   PredefinedRetryPolicies.DEFAULT_MAX_ERROR_RETRY, true);
  retryableConfig.setRetryPolicy(retryPolicy);
  return retryableConfig;
}
Example 2
Source File: AwsS3ClientFactory.java    From circus-train with Apache License 2.0 5 votes vote down vote up 
public AmazonS3 newInstance(Configuration conf) {
  int maxErrorRetry = conf.getInt(ConfigurationVariable.UPLOAD_RETRY_COUNT.getName(),
      ConfigurationVariable.UPLOAD_RETRY_COUNT.defaultIntValue());
  long errorRetryDelay = conf.getLong(ConfigurationVariable.UPLOAD_RETRY_DELAY_MS.getName(),
      ConfigurationVariable.UPLOAD_RETRY_DELAY_MS.defaultLongValue());

  LOG.info("Creating AWS S3 client with a retry policy of {} retries and {} ms of exponential backoff delay",
      maxErrorRetry, errorRetryDelay);

  RetryPolicy retryPolicy = new RetryPolicy(new CounterBasedRetryCondition(maxErrorRetry),
      new ExponentialBackoffStrategy(errorRetryDelay), maxErrorRetry, true);
  ClientConfiguration clientConfiguration = new ClientConfiguration();
  clientConfiguration.setRetryPolicy(retryPolicy);
  clientConfiguration.setMaxErrorRetry(maxErrorRetry);

  AmazonS3ClientBuilder builder = AmazonS3ClientBuilder
      .standard()
      .withCredentials(new HadoopAWSCredentialProviderChain(conf))
      .withClientConfiguration(clientConfiguration);

  EndpointConfiguration endpointConfiguration = getEndpointConfiguration(conf);
  if (endpointConfiguration != null) {
    builder.withEndpointConfiguration(endpointConfiguration);
  } else {
    builder.withRegion(getRegion(conf));
  }

  return builder.build();
}
Example 3
Source File: AWSClientFactory.java    From pipeline-aws-plugin with Apache License 2.0 5 votes vote down vote up 
private static ClientConfiguration getClientConfiguration(EnvVars vars) {
	ClientConfiguration clientConfiguration = new ClientConfiguration();
	//the default max retry is 3. Increasing this to be more resilient to upstream errors
	clientConfiguration.setRetryPolicy(new RetryPolicy(null, null, 10, false));
	ProxyConfiguration.configure(vars, clientConfiguration);
	return clientConfiguration;
}
Example 4
Source File: AWSAutoConfiguration.java    From genie with Apache License 2.0 5 votes vote down vote up 
/**
 * Create a named {@link ClientConfiguration} to be used by the {@link AmazonSNS} client, unless a bean by that
 * name already exists in context.
 *
 * @param retryPolicy The retry policy
 * @return a named {@link ClientConfiguration}
 */
@Bean(name = SNS_CLIENT_CONFIGURATION_BEAN_NAME)
@ConditionalOnMissingBean(name = SNS_CLIENT_CONFIGURATION_BEAN_NAME)
public ClientConfiguration jobNotificationsSNSClientConfiguration(
    @Qualifier(SNS_CLIENT_RETRY_POLICY_BEAN_NAME) final RetryPolicy retryPolicy
) {
    final ClientConfiguration configuration = new ClientConfigurationFactory().getConfig();
    configuration.setRetryPolicy(retryPolicy);
    return configuration;
}
Example 5
Source File: AwsEc2ServiceImpl.java    From crate with Apache License 2.0 5 votes vote down vote up 
static ClientConfiguration buildConfiguration(Logger logger, Ec2ClientSettings clientSettings) {
    final ClientConfiguration clientConfiguration = new ClientConfiguration();
    // the response metadata cache is only there for diagnostics purposes,
    // but can force objects from every response to the old generation.
    clientConfiguration.setResponseMetadataCacheSize(0);
    clientConfiguration.setProtocol(clientSettings.protocol);
    if (Strings.hasText(clientSettings.proxyHost)) {
        // TODO: remove this leniency, these settings should exist together and be validated
        clientConfiguration.setProxyHost(clientSettings.proxyHost);
        clientConfiguration.setProxyPort(clientSettings.proxyPort);
        clientConfiguration.setProxyUsername(clientSettings.proxyUsername);
        clientConfiguration.setProxyPassword(clientSettings.proxyPassword);
    }
    // Increase the number of retries in case of 5xx API responses
    final Random rand = Randomness.get();
    final RetryPolicy retryPolicy = new RetryPolicy(
        RetryPolicy.RetryCondition.NO_RETRY_CONDITION,
        (originalRequest, exception, retriesAttempted) -> {
            // with 10 retries the max delay time is 320s/320000ms (10 * 2^5 * 1 * 1000)
            logger.warn("EC2 API request failed, retry again. Reason was:", exception);
            return 1000L * (long) (10d * Math.pow(2, retriesAttempted / 2.0d) * (1.0d + rand.nextDouble()));
        },
        10,
        false);
    clientConfiguration.setRetryPolicy(retryPolicy);
    clientConfiguration.setSocketTimeout(clientSettings.readTimeoutMillis);
    return clientConfiguration;
}
Example 6
Source File: AwsS3ObjectStorage.java    From james-project with Apache License 2.0 4 votes vote down vote up 
private static ClientConfiguration getClientConfiguration() {
    ClientConfiguration clientConfiguration = new ClientConfiguration();
    clientConfiguration.setRetryPolicy(PredefinedRetryPolicies.getDefaultRetryPolicyWithCustomMaxRetries(MAX_ERROR_RETRY));
    return clientConfiguration;
}
Example 7
Source File: KinesisAppender.java    From kinesis-log4j-appender with Apache License 2.0 4 votes vote down vote up 
/**
  * Configures this appender instance and makes it ready for use by the
  * consumers. It validates mandatory parameters and confirms if the configured
  * stream is ready for publishing data yet.
  * 
  * Error details are made available through the fallback handler for this
  * appender
  * 
  * @throws IllegalStateException
  *           if we encounter issues configuring this appender instance
  */
 @Override
 public void activateOptions() {
   if (streamName == null) {
     initializationFailed = true;
     error("Invalid configuration - streamName cannot be null for appender: " + name);
   }

   if (layout == null) {
     initializationFailed = true;
     error("Invalid configuration - No layout for appender: " + name);
   }

   ClientConfiguration clientConfiguration = new ClientConfiguration();
   clientConfiguration = setProxySettingsFromSystemProperties(clientConfiguration);

   clientConfiguration.setMaxErrorRetry(maxRetries);
   clientConfiguration.setRetryPolicy(new RetryPolicy(PredefinedRetryPolicies.DEFAULT_RETRY_CONDITION,
       PredefinedRetryPolicies.DEFAULT_BACKOFF_STRATEGY, maxRetries, true));
   clientConfiguration.setUserAgent(AppenderConstants.USER_AGENT_STRING);

   BlockingQueue<Runnable> taskBuffer = new LinkedBlockingDeque<Runnable>(bufferSize);
   ThreadPoolExecutor threadPoolExecutor = new ThreadPoolExecutor(threadCount, threadCount,
       AppenderConstants.DEFAULT_THREAD_KEEP_ALIVE_SEC, TimeUnit.SECONDS, taskBuffer, new BlockFastProducerPolicy());
   threadPoolExecutor.prestartAllCoreThreads();
   kinesisClient = new AmazonKinesisAsyncClient(new CustomCredentialsProviderChain(), clientConfiguration,
       threadPoolExecutor);

   boolean regionProvided = !Validator.isBlank(region);
   if (!regionProvided) {
     region = AppenderConstants.DEFAULT_REGION;
   }
   if (!Validator.isBlank(endpoint)) {
     if (regionProvided) {
LOGGER
    .warn("Received configuration for both region as well as Amazon Kinesis endpoint. ("
	+ endpoint
	+ ") will be used as endpoint instead of default endpoint for region ("
	+ region + ")");
     }
     kinesisClient.setEndpoint(endpoint,
  AppenderConstants.DEFAULT_SERVICE_NAME, region);
   } else {
     kinesisClient.setRegion(Region.getRegion(Regions.fromName(region)));
   }

   DescribeStreamResult describeResult = null;
   try {
     describeResult = kinesisClient.describeStream(streamName);
     String streamStatus = describeResult.getStreamDescription().getStreamStatus();
     if (!StreamStatus.ACTIVE.name().equals(streamStatus) && !StreamStatus.UPDATING.name().equals(streamStatus)) {
       initializationFailed = true;
       error("Stream " + streamName + " is not ready (in active/updating status) for appender: " + name);
     }
   } catch (ResourceNotFoundException rnfe) {
     initializationFailed = true;
     error("Stream " + streamName + " doesn't exist for appender: " + name, rnfe);
   }

   asyncCallHander = new AsyncPutCallStatsReporter(name);
 }
Example 8
Source File: AWSDatabaseHolder.java    From billow with Apache License 2.0 4 votes vote down vote up 
public AWSDatabaseHolder(Config config) {
    maxAgeInMs = config.getDuration("maxAge", TimeUnit.MILLISECONDS);

    final DefaultAWSCredentialsProviderChain awsCredentialsProviderChain = new DefaultAWSCredentialsProviderChain();

    final ClientConfiguration clientConfig = new ClientConfiguration();
    clientConfig.setRetryPolicy(new RetryPolicy(null, null, config.getInt("maxErrorRetry"), true));
    clientConfig.setSocketTimeout(config.getInt("socketTimeout") * 1000);

    final AmazonEC2 bootstrapEC2Client = AmazonEC2ClientBuilder.standard().withCredentials(awsCredentialsProviderChain).build();

    ec2Clients = Maps.newHashMap();
    rdsClients = Maps.newHashMap();
    sqsClients = Maps.newHashMap();
    dynamoDBClients = Maps.newHashMap();
    elasticacheClients = Maps.newHashMap();
    elasticsearchClients = Maps.newHashMap();

    final List<Region> ec2Regions = bootstrapEC2Client.describeRegions().getRegions();
    for (Region region : ec2Regions) {
        final String regionName = region.getRegionName();
        final String endpoint = region.getEndpoint();
        log.debug("Adding ec2 region {}", region);

        if (config.getBoolean("ec2Enabled")) {
            final AmazonEC2Client ec2Client = new AmazonEC2Client(awsCredentialsProviderChain, clientConfig);
            ec2Client.setEndpoint(endpoint);
            ec2Clients.put(regionName, ec2Client);
        }

        if (config.getBoolean("rdsEnabled")) {
            final AmazonRDSClient rdsClient = new AmazonRDSClient(awsCredentialsProviderChain, clientConfig);
            rdsClient.setEndpoint(endpoint.replaceFirst("ec2\\.", "rds."));
            rdsClients.put(regionName, rdsClient);
        }

        if (config.getBoolean("dynamodbEnabled")) {
            final AmazonDynamoDBClient dynamoDBClient =
                new AmazonDynamoDBClient(awsCredentialsProviderChain, clientConfig);
            dynamoDBClient.setEndpoint(endpoint.replaceFirst("ec2\\.", "dynamodb."));
            dynamoDBClients.put(regionName, dynamoDBClient);
        }

        if (config.getBoolean("sqsEnabled")) {
            final AmazonSQSClient sqsClient = new AmazonSQSClient(awsCredentialsProviderChain, clientConfig);
            sqsClient.setEndpoint(endpoint.replaceFirst("ec2\\.", "sqs."));
            sqsClients.put(regionName, sqsClient);
        }

        if (config.getBoolean("elasticacheEnabled")) {
            final AmazonElastiCacheClient elastiCacheClient = new AmazonElastiCacheClient
                (awsCredentialsProviderChain, clientConfig);
            elastiCacheClient.setEndpoint(endpoint.replaceFirst("ec2\\.", "elasticache."));
            elasticacheClients.put(regionName, elastiCacheClient);
        }

        if (config.getBoolean("elasticsearchEnabled")) {
            final AWSElasticsearchClient elasticsearchClient = new AWSElasticsearchClient
                (awsCredentialsProviderChain, clientConfig);
            elasticsearchClient.setEndpoint(endpoint.replaceFirst("ec2\\.", "es."));
            elasticsearchClients.put(regionName, elasticsearchClient);
        }
    }

    this.iamClient = AmazonIdentityManagementClientBuilder.standard()
        .withCredentials(awsCredentialsProviderChain)
        .withClientConfiguration(clientConfig)
        .build();

    if (config.hasPath("accountNumber")) {
        this.awsAccountNumber = config.getString("accountNumber");
    } else {
        this.awsAccountNumber = null;
    }

    if (config.hasPath("arnPartition")) {
        this.awsARNPartition = config.getString("arnPartition");
    } else {
        this.awsARNPartition = "aws";
    }

    rebuild();
}