Java Code Examples for com.amazonaws.services.s3.AmazonS3ClientBuilder#withEndpointConfiguration()

private AmazonS3 newGlobalInstance(
    S3S3CopierOptions s3s3CopierOptions,
    HadoopAWSCredentialProviderChain credentialsChain) {
  AmazonS3ClientBuilder builder = AmazonS3ClientBuilder
      .standard()
      .withForceGlobalBucketAccessEnabled(Boolean.TRUE)
      .withCredentials(credentialsChain);

  applyClientConfigurations(builder, s3s3CopierOptions);

  URI s3Endpoint = s3s3CopierOptions.getS3Endpoint();
  if (s3Endpoint != null) {
    EndpointConfiguration endpointConfiguration = new EndpointConfiguration(s3Endpoint.toString(),
        Region.US_Standard.getFirstRegionId());
    builder.withEndpointConfiguration(endpointConfiguration);
  }
  return builder.build();
}
 

private AmazonS3 newInstance(
    String region,
    S3S3CopierOptions s3s3CopierOptions,
    HadoopAWSCredentialProviderChain credentialsChain) {
  AmazonS3ClientBuilder builder = AmazonS3ClientBuilder
      .standard()
      .withCredentials(credentialsChain);

  applyClientConfigurations(builder, s3s3CopierOptions);

  URI s3Endpoint = s3s3CopierOptions.getS3Endpoint(region);
  if (s3Endpoint != null) {
    EndpointConfiguration endpointConfiguration = new EndpointConfiguration(s3Endpoint.toString(), region);
    builder.withEndpointConfiguration(endpointConfiguration);
  } else {
    builder.withRegion(region);
  }

  return builder.build();
}
 

private static AmazonS3 getAmazonS3(final S3BucketConfiguration s3BucketConfiguration, Clock clock) {
    AmazonS3ClientBuilder amazonS3ClientBuilder = AmazonS3ClientBuilder.standard()
            .withCredentials(getAwsCredentialsProvider(s3BucketConfiguration))
            .withAccelerateModeEnabled(s3BucketConfiguration.getAccelerateModeEnabled());
    S3ClientConfiguration.RateLimitConfiguration rateLimitConfiguration = new S3ClientConfiguration.RateLimitConfiguration();
    if (null != s3BucketConfiguration.getRegion()) {
        amazonS3ClientBuilder
                .withRegion(Regions.fromName(s3BucketConfiguration.getRegion()));
    } else if (null != s3BucketConfiguration.getS3ClientConfiguration()) {
        S3ClientConfiguration.EndpointConfiguration endpointConfiguration = s3BucketConfiguration.getS3ClientConfiguration().getEndpointConfiguration();
        amazonS3ClientBuilder
                .withEndpointConfiguration(new AwsClientBuilder.EndpointConfiguration(endpointConfiguration.getServiceEndpoint(), endpointConfiguration.getSigningRegion()));
        rateLimitConfiguration = s3BucketConfiguration.getS3ClientConfiguration().getRateLimitConfiguration();
    }
    AmazonS3 amazonS3 = amazonS3ClientBuilder
            .build();

    return new S3RateLimiter(clock, rateLimitConfiguration)
            .rateLimit(amazonS3);
}
 

public static AmazonS3 buildS3(LocalServerConfig configuration) {
    String region = configuration.getProperty(pRegion);
    String endpoint = configuration.getProperty(pEndpoint);
    String credentialsFile =  configuration.getProperty(pCredentialFile);
    String credentialsProfile =  configuration.getProperty(pCredentialProfile);

    AmazonS3ClientBuilder builder = AmazonS3ClientBuilder.standard();
    if ( endpoint == null )
        builder.withRegion(region);
    else  {
        // Needed for S3mock
        builder.withPathStyleAccessEnabled(true);
        builder.withEndpointConfiguration(new EndpointConfiguration(endpoint, region));
        builder.withCredentials(new AWSStaticCredentialsProvider(new AnonymousAWSCredentials()));
    }
    if ( credentialsFile != null )
        builder.withCredentials(new ProfileCredentialsProvider(credentialsFile, credentialsProfile));
    return builder.build();
}
 

@Override
public AmazonS3ClientBuilder createBuilder(S3Options s3Options) {
  AmazonS3ClientBuilder builder =
      AmazonS3ClientBuilder.standard().withCredentials(s3Options.getAwsCredentialsProvider());

  if (s3Options.getClientConfiguration() != null) {
    builder = builder.withClientConfiguration(s3Options.getClientConfiguration());
  }

  if (!Strings.isNullOrEmpty(s3Options.getAwsServiceEndpoint())) {
    builder =
        builder.withEndpointConfiguration(
            new AwsClientBuilder.EndpointConfiguration(
                s3Options.getAwsServiceEndpoint(), s3Options.getAwsRegion()));
  } else if (!Strings.isNullOrEmpty(s3Options.getAwsRegion())) {
    builder = builder.withRegion(s3Options.getAwsRegion());
  } else {
    LOG.info(
        "The AWS S3 Beam extension was included in this build, but the awsRegion flag "
            + "was not specified. If you don't plan to use S3, then ignore this message.");
  }
  return builder;
}
 

private AmazonS3 buildClient(final S3ClientSettings clientSettings) {
    final AmazonS3ClientBuilder builder = AmazonS3ClientBuilder.standard();
    builder.withCredentials(buildCredentials(LOGGER, clientSettings));
    builder.withClientConfiguration(buildConfiguration(clientSettings));

    final String endpoint = Strings.hasLength(clientSettings.endpoint)
        ? clientSettings.endpoint
        : Constants.S3_HOSTNAME;
    LOGGER.debug("using endpoint [{}]", endpoint);

    // If the endpoint configuration isn't set on the builder then the default behaviour is to try
    // and work out what region we are in and use an appropriate endpoint - see AwsClientBuilder#setRegion.
    // In contrast, directly-constructed clients use s3.amazonaws.com unless otherwise instructed. We currently
    // use a directly-constructed client, and need to keep the existing behaviour to avoid a breaking change,
    // so to move to using the builder we must set it explicitly to keep the existing behaviour.
    //
    // We do this because directly constructing the client is deprecated (was already deprecated in 1.1.223 too)
    // so this change removes that usage of a deprecated API.
    builder.withEndpointConfiguration(new AwsClientBuilder.EndpointConfiguration(endpoint, null));

    return builder.build();
}
 

private AmazonS3 createS3Client(AwsS3BuildCache config) {
  AmazonS3 s3;
  try {
    AmazonS3ClientBuilder s3Builder = AmazonS3ClientBuilder.standard();
    if (!isNullOrEmpty(config.getAwsAccessKeyId()) && !isNullOrEmpty(config.getAwsSecretKey()) &&
              !isNullOrEmpty(config.getSessionToken())) {
          s3Builder.withCredentials(new AWSStaticCredentialsProvider(
                  new BasicSessionCredentials(config.getAwsAccessKeyId(), config.getAwsSecretKey(),
                          config.getSessionToken())));
    } else if (!isNullOrEmpty(config.getAwsAccessKeyId()) && !isNullOrEmpty(config.getAwsSecretKey())) {
      s3Builder.withCredentials(new AWSStaticCredentialsProvider(
          new BasicAWSCredentials(config.getAwsAccessKeyId(), config.getAwsSecretKey())));
    }

    addHttpHeaders(s3Builder, config);

    if (isNullOrEmpty(config.getEndpoint())) {
      s3Builder.withRegion(config.getRegion());
    } else {
      s3Builder.withEndpointConfiguration(
          new AwsClientBuilder.EndpointConfiguration(config.getEndpoint(), config.getRegion()));
    }
    s3 = s3Builder.build();
  } catch (SdkClientException e) {
    logger.debug("Error while building AWS S3 client: {}", e.getMessage());
    throw new GradleException("Creation of S3 build cache failed; cannot create S3 client", e);
  }
  return s3;
}
 

public AmazonS3 newInstance(Configuration conf) {
  int maxErrorRetry = conf.getInt(ConfigurationVariable.UPLOAD_RETRY_COUNT.getName(),
      ConfigurationVariable.UPLOAD_RETRY_COUNT.defaultIntValue());
  long errorRetryDelay = conf.getLong(ConfigurationVariable.UPLOAD_RETRY_DELAY_MS.getName(),
      ConfigurationVariable.UPLOAD_RETRY_DELAY_MS.defaultLongValue());

  LOG.info("Creating AWS S3 client with a retry policy of {} retries and {} ms of exponential backoff delay",
      maxErrorRetry, errorRetryDelay);

  RetryPolicy retryPolicy = new RetryPolicy(new CounterBasedRetryCondition(maxErrorRetry),
      new ExponentialBackoffStrategy(errorRetryDelay), maxErrorRetry, true);
  ClientConfiguration clientConfiguration = new ClientConfiguration();
  clientConfiguration.setRetryPolicy(retryPolicy);
  clientConfiguration.setMaxErrorRetry(maxErrorRetry);

  AmazonS3ClientBuilder builder = AmazonS3ClientBuilder
      .standard()
      .withCredentials(new HadoopAWSCredentialProviderChain(conf))
      .withClientConfiguration(clientConfiguration);

  EndpointConfiguration endpointConfiguration = getEndpointConfiguration(conf);
  if (endpointConfiguration != null) {
    builder.withEndpointConfiguration(endpointConfiguration);
  } else {
    builder.withRegion(getRegion(conf));
  }

  return builder.build();
}
 

/**
 * @return the {@link AmazonS3Client} if no other {@link AmazonS3} bean is
 *         registered.
 */
@Bean
@ConditionalOnMissingBean
public AmazonS3 amazonS3() {
    final AmazonS3ClientBuilder s3ClientBuilder = AmazonS3ClientBuilder.standard()
            .withCredentials(awsCredentialsProvider()).withClientConfiguration(awsClientConfiguration());
    if (!StringUtils.isEmpty(endpoint)) {
        final String signingRegion = StringUtils.isEmpty(region) ? "" : region;
        s3ClientBuilder.withEndpointConfiguration(new EndpointConfiguration(endpoint, signingRegion));
    } else if (!StringUtils.isEmpty(region)) {
        s3ClientBuilder.withRegion(region);
    }
    return s3ClientBuilder.build();
}
 

AmazonS3Client createS3Client() throws StageException {

    AmazonS3ClientBuilder builder = createAmazonS3ClientBuilder().withClientConfiguration(createClientConfiguration())
                                                                 .withChunkedEncodingDisabled(connectionConfigs
                                                                     .isChunkedEncodingEnabled())
                                                                 .withPathStyleAccessEnabled(true);
    AWSCredentialsProvider awsCredentialsProvider = getCredentialsProvider();
    // If we don't call build.withCredentials(...) then we will not overwrite the default credentials provider
    // already set in the builder when doing AmazonS3ClientBuilder.standard() so only calling build.withCredentials(...)
    // if our own provider exists
    if (awsCredentialsProvider != null) {
      builder.withCredentials(awsCredentialsProvider);
    }

    String region = (connectionConfigs.getRegion() == null || connectionConfigs.getRegion().isEmpty())
                    ? null
                    : connectionConfigs.getRegion();

    if (connectionConfigs.isUseEndpoint()) {
      builder.withEndpointConfiguration(new AwsClientBuilder.EndpointConfiguration(connectionConfigs.getEndpoint(),
          region
      ));
    } else if (region != null) {
      builder.withRegion(connectionConfigs.getRegion());
    } else {
      builder.withRegion(AwsRegion.US_WEST_1.getId());
      builder.withForceGlobalBucketAccessEnabled(true);
    }
    return (AmazonS3Client) builder.build();
  }
 

private void createConnection(
    Stage.Context context,
    String configPrefix,
    ProxyConfig proxyConfig,
    List<Stage.ConfigIssue> issues,
    int maxErrorRetries
) throws StageException {
  AWSCredentialsProvider credentials = AWSUtil.getCredentialsProvider(awsConfig);
  ClientConfiguration clientConfig = AWSUtil.getClientConfiguration(proxyConfig);

  if (maxErrorRetries >= 0) {
    clientConfig.setMaxErrorRetry(maxErrorRetries);
  }

  AmazonS3ClientBuilder builder = AmazonS3ClientBuilder
      .standard()
      .withCredentials(credentials)
      .withClientConfiguration(clientConfig)
      .withChunkedEncodingDisabled(awsConfig.disableChunkedEncoding)
      .withPathStyleAccessEnabled(usePathAddressModel);

  if (region == AwsRegion.OTHER) {
    if (endpoint == null || endpoint.isEmpty()) {
      issues.add(context.createConfigIssue(Groups.S3.name(), configPrefix + "endpoint", Errors.S3_SPOOLDIR_10));
      return;
    }
    builder.withEndpointConfiguration(new AwsClientBuilder.EndpointConfiguration(endpoint, null));
  } else {
    builder.withRegion(region.getId());
  }
  s3Client = builder.build();
}
 

private AmazonS3 provideAmazonS3(final Provider<CoreV1Api> coreV1ApiProvider, final AbstractOperationRequest operationRequest) {

            final S3Configuration s3Conf = resolveS3Configuration(coreV1ApiProvider, operationRequest);

            final AmazonS3ClientBuilder builder = AmazonS3ClientBuilder.standard();

            if (s3Conf.awsEndpoint != null) {
                // AWS_REGION must be set if AWS_ENDPOINT is set
                if (s3Conf.awsRegion == null) {
                    throw new IllegalArgumentException("AWS_REGION must be set if AWS_ENDPOINT is set.");
                }

                builder.withEndpointConfiguration(new EndpointConfiguration(s3Conf.awsEndpoint, s3Conf.awsRegion.toLowerCase()));
            } else if (s3Conf.awsRegion != null) {
                builder.withRegion(Regions.fromName(s3Conf.awsRegion.toLowerCase()));
            }

            // if we are not running against Kubernetes, credentials should be fetched from ~/.aws/...
            if (isRunningInKubernetes()) {
                // it is possible that we have not set any secrets for s3 so the last
                // resort is to fallback to AWS instance credentials.
                if (s3Conf.awsAccessKeyId != null && s3Conf.awsSecretKey != null) {
                    builder.setCredentials(new AWSCredentialsProvider() {
                        @Override
                        public AWSCredentials getCredentials() {
                            return new AWSCredentials() {
                                @Override
                                public String getAWSAccessKeyId() {
                                    return s3Conf.awsAccessKeyId;
                                }

                                @Override
                                public String getAWSSecretKey() {
                                    return s3Conf.awsSecretKey;
                                }
                            };
                        }

                        @Override
                        public void refresh() {
                        }
                    });
                }
            }

            return builder.build();
        }
 

@Test
public void testS3() throws Exception {
    Properties syncProperties = com.emc.ecs.sync.test.TestConfig.getProperties();
    final String bucket = "ecs-sync-s3-test-bucket";
    final String endpoint = syncProperties.getProperty(com.emc.ecs.sync.test.TestConfig.PROP_S3_ENDPOINT);
    final String accessKey = syncProperties.getProperty(com.emc.ecs.sync.test.TestConfig.PROP_S3_ACCESS_KEY_ID);
    final String secretKey = syncProperties.getProperty(com.emc.ecs.sync.test.TestConfig.PROP_S3_SECRET_KEY);
    final String region = syncProperties.getProperty(com.emc.ecs.sync.test.TestConfig.PROP_S3_REGION);
    Assume.assumeNotNull(endpoint, accessKey, secretKey);
    URI endpointUri = new URI(endpoint);

    ClientConfiguration config = new ClientConfiguration().withSignerOverride("S3SignerType");
    AmazonS3ClientBuilder builder = AmazonS3ClientBuilder.standard();
    builder.withCredentials(new AWSStaticCredentialsProvider(new BasicAWSCredentials(accessKey, secretKey)));
    builder.withClientConfiguration(config);
    builder.withEndpointConfiguration(new AwsClientBuilder.EndpointConfiguration(endpoint, region));

    AmazonS3 s3 = builder.build();
    try {
        s3.createBucket(bucket);
    } catch (AmazonServiceException e) {
        if (!e.getErrorCode().equals("BucketAlreadyExists")) throw e;
    }

    // for testing ACLs
    String authUsers = "http://acs.amazonaws.com/groups/global/AuthenticatedUsers";
    String everyone = "http://acs.amazonaws.com/groups/global/AllUsers";
    String[] validGroups = {authUsers, everyone};
    String[] validPermissions = {"READ", "WRITE", "FULL_CONTROL"};

    AwsS3Config awsS3Config = new AwsS3Config();
    if (endpointUri.getScheme() != null)
        awsS3Config.setProtocol(Protocol.valueOf(endpointUri.getScheme().toLowerCase()));
    awsS3Config.setHost(endpointUri.getHost());
    awsS3Config.setPort(endpointUri.getPort());
    awsS3Config.setAccessKey(accessKey);
    awsS3Config.setSecretKey(secretKey);
    awsS3Config.setRegion(region);
    awsS3Config.setLegacySignatures(true);
    awsS3Config.setDisableVHosts(true);
    awsS3Config.setBucketName(bucket);
    awsS3Config.setPreserveDirectories(true);

    TestConfig testConfig = new TestConfig();
    testConfig.setObjectOwner(accessKey);
    testConfig.setValidGroups(validGroups);
    testConfig.setValidPermissions(validPermissions);

    try {
        multiEndToEndTest(awsS3Config, testConfig, true);
    } finally {
        try {
            s3.deleteBucket(bucket);
        } catch (Throwable t) {
            log.warn("could not delete bucket", t);
        }
    }
}