Java Code Examples for com.amazonaws.services.s3.AmazonS3ClientBuilder#withRegion()

private AmazonS3 newInstance(
    String region,
    S3S3CopierOptions s3s3CopierOptions,
    HadoopAWSCredentialProviderChain credentialsChain) {
  AmazonS3ClientBuilder builder = AmazonS3ClientBuilder
      .standard()
      .withCredentials(credentialsChain);

  applyClientConfigurations(builder, s3s3CopierOptions);

  URI s3Endpoint = s3s3CopierOptions.getS3Endpoint(region);
  if (s3Endpoint != null) {
    EndpointConfiguration endpointConfiguration = new EndpointConfiguration(s3Endpoint.toString(), region);
    builder.withEndpointConfiguration(endpointConfiguration);
  } else {
    builder.withRegion(region);
  }

  return builder.build();
}
 

private static AmazonS3 getAmazonS3(final S3BucketConfiguration s3BucketConfiguration, Clock clock) {
    AmazonS3ClientBuilder amazonS3ClientBuilder = AmazonS3ClientBuilder.standard()
            .withCredentials(getAwsCredentialsProvider(s3BucketConfiguration))
            .withAccelerateModeEnabled(s3BucketConfiguration.getAccelerateModeEnabled());
    S3ClientConfiguration.RateLimitConfiguration rateLimitConfiguration = new S3ClientConfiguration.RateLimitConfiguration();
    if (null != s3BucketConfiguration.getRegion()) {
        amazonS3ClientBuilder
                .withRegion(Regions.fromName(s3BucketConfiguration.getRegion()));
    } else if (null != s3BucketConfiguration.getS3ClientConfiguration()) {
        S3ClientConfiguration.EndpointConfiguration endpointConfiguration = s3BucketConfiguration.getS3ClientConfiguration().getEndpointConfiguration();
        amazonS3ClientBuilder
                .withEndpointConfiguration(new AwsClientBuilder.EndpointConfiguration(endpointConfiguration.getServiceEndpoint(), endpointConfiguration.getSigningRegion()));
        rateLimitConfiguration = s3BucketConfiguration.getS3ClientConfiguration().getRateLimitConfiguration();
    }
    AmazonS3 amazonS3 = amazonS3ClientBuilder
            .build();

    return new S3RateLimiter(clock, rateLimitConfiguration)
            .rateLimit(amazonS3);
}
 

public static AmazonS3 buildS3(LocalServerConfig configuration) {
    String region = configuration.getProperty(pRegion);
    String endpoint = configuration.getProperty(pEndpoint);
    String credentialsFile =  configuration.getProperty(pCredentialFile);
    String credentialsProfile =  configuration.getProperty(pCredentialProfile);

    AmazonS3ClientBuilder builder = AmazonS3ClientBuilder.standard();
    if ( endpoint == null )
        builder.withRegion(region);
    else  {
        // Needed for S3mock
        builder.withPathStyleAccessEnabled(true);
        builder.withEndpointConfiguration(new EndpointConfiguration(endpoint, region));
        builder.withCredentials(new AWSStaticCredentialsProvider(new AnonymousAWSCredentials()));
    }
    if ( credentialsFile != null )
        builder.withCredentials(new ProfileCredentialsProvider(credentialsFile, credentialsProfile));
    return builder.build();
}
 

@Override
public AmazonS3ClientBuilder createBuilder(S3Options s3Options) {
  AmazonS3ClientBuilder builder =
      AmazonS3ClientBuilder.standard().withCredentials(s3Options.getAwsCredentialsProvider());

  if (s3Options.getClientConfiguration() != null) {
    builder = builder.withClientConfiguration(s3Options.getClientConfiguration());
  }

  if (!Strings.isNullOrEmpty(s3Options.getAwsServiceEndpoint())) {
    builder =
        builder.withEndpointConfiguration(
            new AwsClientBuilder.EndpointConfiguration(
                s3Options.getAwsServiceEndpoint(), s3Options.getAwsRegion()));
  } else if (!Strings.isNullOrEmpty(s3Options.getAwsRegion())) {
    builder = builder.withRegion(s3Options.getAwsRegion());
  } else {
    LOG.info(
        "The AWS S3 Beam extension was included in this build, but the awsRegion flag "
            + "was not specified. If you don't plan to use S3, then ignore this message.");
  }
  return builder;
}
 

@Override
public Void call() throws Exception {

  if (multiPart && fileSize < OM_MULTIPART_MIN_SIZE) {
    throw new IllegalArgumentException(
        "Size of multipart upload parts should be at least 5MB (5242880)");
  }
  init();

  AmazonS3ClientBuilder amazonS3ClientBuilder =
      AmazonS3ClientBuilder.standard()
          .withCredentials(new EnvironmentVariableCredentialsProvider());

  if (endpoint.length() > 0) {
    amazonS3ClientBuilder
        .withPathStyleAccessEnabled(true)
        .withEndpointConfiguration(
            new EndpointConfiguration(endpoint, "us-east-1"));

  } else {
    amazonS3ClientBuilder.withRegion(Regions.DEFAULT_REGION);
  }

  s3 = amazonS3ClientBuilder.build();

  content = RandomStringUtils.randomAscii(fileSize);

  timer = getMetrics().timer("key-create");

  System.setProperty(DISABLE_PUT_OBJECT_MD5_VALIDATION_PROPERTY, "true");
  runTests(this::createKey);

  return null;
}
 

private AmazonS3 createS3Client(AwsS3BuildCache config) {
  AmazonS3 s3;
  try {
    AmazonS3ClientBuilder s3Builder = AmazonS3ClientBuilder.standard();
    if (!isNullOrEmpty(config.getAwsAccessKeyId()) && !isNullOrEmpty(config.getAwsSecretKey()) &&
              !isNullOrEmpty(config.getSessionToken())) {
          s3Builder.withCredentials(new AWSStaticCredentialsProvider(
                  new BasicSessionCredentials(config.getAwsAccessKeyId(), config.getAwsSecretKey(),
                          config.getSessionToken())));
    } else if (!isNullOrEmpty(config.getAwsAccessKeyId()) && !isNullOrEmpty(config.getAwsSecretKey())) {
      s3Builder.withCredentials(new AWSStaticCredentialsProvider(
          new BasicAWSCredentials(config.getAwsAccessKeyId(), config.getAwsSecretKey())));
    }

    addHttpHeaders(s3Builder, config);

    if (isNullOrEmpty(config.getEndpoint())) {
      s3Builder.withRegion(config.getRegion());
    } else {
      s3Builder.withEndpointConfiguration(
          new AwsClientBuilder.EndpointConfiguration(config.getEndpoint(), config.getRegion()));
    }
    s3 = s3Builder.build();
  } catch (SdkClientException e) {
    logger.debug("Error while building AWS S3 client: {}", e.getMessage());
    throw new GradleException("Creation of S3 build cache failed; cannot create S3 client", e);
  }
  return s3;
}
 

public AmazonS3 newInstance(Configuration conf) {
  int maxErrorRetry = conf.getInt(ConfigurationVariable.UPLOAD_RETRY_COUNT.getName(),
      ConfigurationVariable.UPLOAD_RETRY_COUNT.defaultIntValue());
  long errorRetryDelay = conf.getLong(ConfigurationVariable.UPLOAD_RETRY_DELAY_MS.getName(),
      ConfigurationVariable.UPLOAD_RETRY_DELAY_MS.defaultLongValue());

  LOG.info("Creating AWS S3 client with a retry policy of {} retries and {} ms of exponential backoff delay",
      maxErrorRetry, errorRetryDelay);

  RetryPolicy retryPolicy = new RetryPolicy(new CounterBasedRetryCondition(maxErrorRetry),
      new ExponentialBackoffStrategy(errorRetryDelay), maxErrorRetry, true);
  ClientConfiguration clientConfiguration = new ClientConfiguration();
  clientConfiguration.setRetryPolicy(retryPolicy);
  clientConfiguration.setMaxErrorRetry(maxErrorRetry);

  AmazonS3ClientBuilder builder = AmazonS3ClientBuilder
      .standard()
      .withCredentials(new HadoopAWSCredentialProviderChain(conf))
      .withClientConfiguration(clientConfiguration);

  EndpointConfiguration endpointConfiguration = getEndpointConfiguration(conf);
  if (endpointConfiguration != null) {
    builder.withEndpointConfiguration(endpointConfiguration);
  } else {
    builder.withRegion(getRegion(conf));
  }

  return builder.build();
}
 

/**
 * @return the {@link AmazonS3Client} if no other {@link AmazonS3} bean is
 *         registered.
 */
@Bean
@ConditionalOnMissingBean
public AmazonS3 amazonS3() {
    final AmazonS3ClientBuilder s3ClientBuilder = AmazonS3ClientBuilder.standard()
            .withCredentials(awsCredentialsProvider()).withClientConfiguration(awsClientConfiguration());
    if (!StringUtils.isEmpty(endpoint)) {
        final String signingRegion = StringUtils.isEmpty(region) ? "" : region;
        s3ClientBuilder.withEndpointConfiguration(new EndpointConfiguration(endpoint, signingRegion));
    } else if (!StringUtils.isEmpty(region)) {
        s3ClientBuilder.withRegion(region);
    }
    return s3ClientBuilder.build();
}
 

AmazonS3Client createS3Client() throws StageException {

    AmazonS3ClientBuilder builder = createAmazonS3ClientBuilder().withClientConfiguration(createClientConfiguration())
                                                                 .withChunkedEncodingDisabled(connectionConfigs
                                                                     .isChunkedEncodingEnabled())
                                                                 .withPathStyleAccessEnabled(true);
    AWSCredentialsProvider awsCredentialsProvider = getCredentialsProvider();
    // If we don't call build.withCredentials(...) then we will not overwrite the default credentials provider
    // already set in the builder when doing AmazonS3ClientBuilder.standard() so only calling build.withCredentials(...)
    // if our own provider exists
    if (awsCredentialsProvider != null) {
      builder.withCredentials(awsCredentialsProvider);
    }

    String region = (connectionConfigs.getRegion() == null || connectionConfigs.getRegion().isEmpty())
                    ? null
                    : connectionConfigs.getRegion();

    if (connectionConfigs.isUseEndpoint()) {
      builder.withEndpointConfiguration(new AwsClientBuilder.EndpointConfiguration(connectionConfigs.getEndpoint(),
          region
      ));
    } else if (region != null) {
      builder.withRegion(connectionConfigs.getRegion());
    } else {
      builder.withRegion(AwsRegion.US_WEST_1.getId());
      builder.withForceGlobalBucketAccessEnabled(true);
    }
    return (AmazonS3Client) builder.build();
  }
 

private void createConnection(
    Stage.Context context,
    String configPrefix,
    ProxyConfig proxyConfig,
    List<Stage.ConfigIssue> issues,
    int maxErrorRetries
) throws StageException {
  AWSCredentialsProvider credentials = AWSUtil.getCredentialsProvider(awsConfig);
  ClientConfiguration clientConfig = AWSUtil.getClientConfiguration(proxyConfig);

  if (maxErrorRetries >= 0) {
    clientConfig.setMaxErrorRetry(maxErrorRetries);
  }

  AmazonS3ClientBuilder builder = AmazonS3ClientBuilder
      .standard()
      .withCredentials(credentials)
      .withClientConfiguration(clientConfig)
      .withChunkedEncodingDisabled(awsConfig.disableChunkedEncoding)
      .withPathStyleAccessEnabled(usePathAddressModel);

  if (region == AwsRegion.OTHER) {
    if (endpoint == null || endpoint.isEmpty()) {
      issues.add(context.createConfigIssue(Groups.S3.name(), configPrefix + "endpoint", Errors.S3_SPOOLDIR_10));
      return;
    }
    builder.withEndpointConfiguration(new AwsClientBuilder.EndpointConfiguration(endpoint, null));
  } else {
    builder.withRegion(region.getId());
  }
  s3Client = builder.build();
}
 

public static AmazonS3 getS3client(GoEnvironment env) {
    AmazonS3ClientBuilder amazonS3ClientBuilder = AmazonS3ClientBuilder.standard();

    if (env.has(AWS_REGION)) {
        amazonS3ClientBuilder.withRegion(env.get(AWS_REGION));
    }
    if (env.hasAWSUseIamRole()) {
        amazonS3ClientBuilder.withCredentials(new InstanceProfileCredentialsProvider(false));
    } else if (env.has(AWS_ACCESS_KEY_ID) && env.has(AWS_SECRET_ACCESS_KEY)) {
        BasicAWSCredentials basicCreds = new BasicAWSCredentials(env.get(AWS_ACCESS_KEY_ID), env.get(AWS_SECRET_ACCESS_KEY));
        amazonS3ClientBuilder.withCredentials(new AWSStaticCredentialsProvider(basicCreds));
    }

    return amazonS3ClientBuilder.build();
}
 

@Override
public AwsS3EnvironmentRepository build(
		AwsS3EnvironmentProperties environmentProperties) {
	final AmazonS3ClientBuilder clientBuilder = AmazonS3ClientBuilder.standard();
	if (environmentProperties.getRegion() != null) {
		clientBuilder.withRegion(environmentProperties.getRegion());
	}
	final AmazonS3 client = clientBuilder.build();
	if (environmentProperties.getEndpoint() != null) {
		client.setEndpoint(environmentProperties.getEndpoint());
	}
	AwsS3EnvironmentRepository repository = new AwsS3EnvironmentRepository(client,
			environmentProperties.getBucket(), server);
	return repository;
}
 

private AmazonS3ClientBuilder buildAmazonS3ForRegion(AmazonS3 prototype,
		String region) {
	AmazonS3ClientBuilder clientBuilder = AmazonS3ClientBuilder.standard();

	AmazonS3Client target = getAmazonS3ClientFromProxy(prototype);
	if (target != null) {
		AWSCredentialsProvider awsCredentialsProvider = (AWSCredentialsProvider) ReflectionUtils
				.getField(this.credentialsProviderField, target);
		clientBuilder.withCredentials(awsCredentialsProvider);
	}

	return clientBuilder.withRegion(region);
}
 

public static AmazonS3 getS3client(GoEnvironment env) {
    AmazonS3ClientBuilder amazonS3ClientBuilder = AmazonS3ClientBuilder.standard();

    if (env.has(AWS_REGION)) {
        amazonS3ClientBuilder.withRegion(env.get(AWS_REGION));
    }
    if (env.hasAWSUseIamRole()) {
        amazonS3ClientBuilder.withCredentials(new InstanceProfileCredentialsProvider(false));
    } else if (env.has(AWS_ACCESS_KEY_ID) && env.has(AWS_SECRET_ACCESS_KEY)) {
        BasicAWSCredentials basicCreds = new BasicAWSCredentials(env.get(AWS_ACCESS_KEY_ID), env.get(AWS_SECRET_ACCESS_KEY));
        amazonS3ClientBuilder.withCredentials(new AWSStaticCredentialsProvider(basicCreds));
    }

    return amazonS3ClientBuilder.build();
}
 

private AmazonS3 provideAmazonS3(final Provider<CoreV1Api> coreV1ApiProvider, final AbstractOperationRequest operationRequest) {

            final S3Configuration s3Conf = resolveS3Configuration(coreV1ApiProvider, operationRequest);

            final AmazonS3ClientBuilder builder = AmazonS3ClientBuilder.standard();

            if (s3Conf.awsEndpoint != null) {
                // AWS_REGION must be set if AWS_ENDPOINT is set
                if (s3Conf.awsRegion == null) {
                    throw new IllegalArgumentException("AWS_REGION must be set if AWS_ENDPOINT is set.");
                }

                builder.withEndpointConfiguration(new EndpointConfiguration(s3Conf.awsEndpoint, s3Conf.awsRegion.toLowerCase()));
            } else if (s3Conf.awsRegion != null) {
                builder.withRegion(Regions.fromName(s3Conf.awsRegion.toLowerCase()));
            }

            // if we are not running against Kubernetes, credentials should be fetched from ~/.aws/...
            if (isRunningInKubernetes()) {
                // it is possible that we have not set any secrets for s3 so the last
                // resort is to fallback to AWS instance credentials.
                if (s3Conf.awsAccessKeyId != null && s3Conf.awsSecretKey != null) {
                    builder.setCredentials(new AWSCredentialsProvider() {
                        @Override
                        public AWSCredentials getCredentials() {
                            return new AWSCredentials() {
                                @Override
                                public String getAWSAccessKeyId() {
                                    return s3Conf.awsAccessKeyId;
                                }

                                @Override
                                public String getAWSSecretKey() {
                                    return s3Conf.awsSecretKey;
                                }
                            };
                        }

                        @Override
                        public void refresh() {
                        }
                    });
                }
            }

            return builder.build();
        }
 

@Override
public void configure(SyncStorage source, Iterator<SyncFilter> filters, SyncStorage target) {
    super.configure(source, filters, target);

    Assert.hasText(config.getAccessKey(), "accessKey is required");
    Assert.hasText(config.getSecretKey(), "secretKey is required");
    Assert.hasText(config.getBucketName(), "bucketName is required");
    Assert.isTrue(config.getBucketName().matches("[A-Za-z0-9._-]+"), config.getBucketName() + " is not a valid bucket name");

    AWSCredentials creds = new BasicAWSCredentials(config.getAccessKey(), config.getSecretKey());
    ClientConfiguration cc = new ClientConfiguration();

    if (config.getProtocol() != null)
        cc.setProtocol(Protocol.valueOf(config.getProtocol().toString().toUpperCase()));

    if (config.isLegacySignatures()) cc.setSignerOverride("S3SignerType");

    if (config.getSocketTimeoutMs() >= 0) cc.setSocketTimeout(config.getSocketTimeoutMs());

    AmazonS3ClientBuilder builder = AmazonS3ClientBuilder.standard()
            .withCredentials(new AWSStaticCredentialsProvider(creds))
            .withClientConfiguration(cc);

    if (config.getHost() != null) {
        String endpoint = "";
        if (config.getProtocol() != null) endpoint += config.getProtocol() + "://";
        endpoint += config.getHost();
        if (config.getPort() > 0) endpoint += ":" + config.getPort();
        builder.setEndpointConfiguration(new AwsClientBuilder.EndpointConfiguration(endpoint, config.getRegion()));
    } else if (config.getRegion() != null) {
        builder.withRegion(config.getRegion());
    }

    if (config.isDisableVHosts()) {
        log.info("The use of virtual hosted buckets has been DISABLED.  Path style buckets will be used.");
        builder.withPathStyleAccessEnabled(true);
    }

    s3 = builder.build();

    boolean bucketExists = s3.doesBucketExistV2(config.getBucketName());

    boolean bucketHasVersions = false;
    if (bucketExists && config.isIncludeVersions()) {
        // check if versioning has ever been enabled on the bucket (versions will not be collected unless required)
        BucketVersioningConfiguration versioningConfig = s3.getBucketVersioningConfiguration(config.getBucketName());
        List<String> versionedStates = Arrays.asList(BucketVersioningConfiguration.ENABLED, BucketVersioningConfiguration.SUSPENDED);
        bucketHasVersions = versionedStates.contains(versioningConfig.getStatus());
    }

    if (config.getKeyPrefix() == null) config.setKeyPrefix(""); // make sure keyPrefix isn't null

    if (source == this) {
        if (config.getExcludedKeys() != null) {
            excludedKeyPatterns = new ArrayList<>();
            for (String pattern : config.getExcludedKeys()) {
                excludedKeyPatterns.add(Pattern.compile(pattern));
            }
        }
    }

    if (target == this) {
        // create bucket if it doesn't exist
        if (!bucketExists && config.isCreateBucket()) {
            s3.createBucket(config.getBucketName());
            bucketExists = true;
            if (config.isIncludeVersions()) {
                s3.setBucketVersioningConfiguration(new SetBucketVersioningConfigurationRequest(config.getBucketName(),
                        new BucketVersioningConfiguration(BucketVersioningConfiguration.ENABLED)));
                bucketHasVersions = true;
            }
        }

        // make sure MPU settings are valid
        if (config.getMpuThresholdMb() > MAX_PUT_SIZE_MB) {
            log.warn("{}MB is above the maximum PUT size of {}MB. the maximum will be used instead",
                    config.getMpuThresholdMb(), MAX_PUT_SIZE_MB);
            config.setMpuThresholdMb(MAX_PUT_SIZE_MB);
        }
        if (config.getMpuPartSizeMb() < MIN_PART_SIZE_MB) {
            log.warn("{}MB is below the minimum MPU part size of {}MB. the minimum will be used instead",
                    config.getMpuPartSizeMb(), MIN_PART_SIZE_MB);
            config.setMpuPartSizeMb(MIN_PART_SIZE_MB);
        }

        if (source != null) sourceReadWindow = source.getReadWindow();
    }

    // make sure bucket exists
    if (!bucketExists)
        throw new ConfigurationException("The bucket " + config.getBucketName() + " does not exist.");

    // if syncing versions, make sure plugins support it and bucket has versioning enabled
    if (config.isIncludeVersions()) {
        if (!(source instanceof AbstractS3Storage && target instanceof AbstractS3Storage))
            throw new ConfigurationException("Version migration is only supported between two S3 plugins");

        if (!bucketHasVersions)
            throw new ConfigurationException("The specified bucket does not have versioning enabled.");
    }
}