software.amazon.awssdk.services.s3.S3Configuration Java Examples

@Test
public void regionalSettingEnabled_usesRegionalIadEndpoint() throws UnsupportedEncodingException {
    EnvironmentVariableHelper environmentVariableHelper = new EnvironmentVariableHelper();
    environmentVariableHelper.set(SdkSystemSetting.AWS_S3_US_EAST_1_REGIONAL_ENDPOINT.environmentVariable(), "regional");

    mockHttpClient.stubNextResponse(mockListObjectsResponse());

    S3Client s3Client = S3Client.builder()
            .credentialsProvider(StaticCredentialsProvider.create(AwsBasicCredentials.create("akid", "skid")))
            .httpClient(mockHttpClient)
            .region(Region.US_EAST_1)
            .serviceConfiguration(S3Configuration.builder()
                    .pathStyleAccessEnabled(true)
                    .build())
            .build();
    try {
        s3Client.listObjects(ListObjectsRequest.builder().bucket(BUCKET).build());
        assertThat(mockHttpClient.getLastRequest().getUri().getHost()).isEqualTo("s3.us-east-1.amazonaws.com");
    } finally {
        environmentVariableHelper.reset();
    }
}
 

@Bean
public S3AsyncClient s3client(S3ClientConfigurarionProperties s3props, AwsCredentialsProvider credentialsProvider) {

    SdkAsyncHttpClient httpClient = NettyNioAsyncHttpClient.builder()
        .writeTimeout(Duration.ZERO)
        .maxConcurrency(64)
        .build();

    S3Configuration serviceConfiguration = S3Configuration.builder()
        .checksumValidationEnabled(false)
        .chunkedEncodingEnabled(true)
        .build();

    S3AsyncClientBuilder b = S3AsyncClient.builder()
        .httpClient(httpClient)
        .region(s3props.getRegion())
        .credentialsProvider(credentialsProvider)
        .serviceConfiguration(serviceConfiguration);

    if (s3props.getEndpoint() != null) {
        b = b.endpointOverride(s3props.getEndpoint());
    }

    return b.build();
}
 

@Test
public void modifyRequest_existingValueInExecutionAttributes_TakesPrecedenceOverClientConfig() {

    boolean configValue = false;
    S3Configuration config = S3Configuration.builder()
            .chunkedEncodingEnabled(configValue)
            .build();

    ExecutionAttributes executionAttributes = new ExecutionAttributes()
            .putAttribute(SERVICE_CONFIG, config)
            .putAttribute(ENABLE_CHUNKED_ENCODING, !configValue);

    interceptor.modifyRequest(context(PutObjectRequest.builder().build()), executionAttributes);

    assertThat(executionAttributes.getAttribute(ENABLE_CHUNKED_ENCODING)).isEqualTo(!configValue);
}
 

@Test
public void accesspointArn_withFipsRegionPrefix_useArnRegionEnabled_shouldThrowIllegalArgumentException() {
    assertThatThrownBy(() -> verifyAccesspointArn("http",
                         "arn:aws:s3:us-east-1:12345678910:accesspoint/foobar",
                         "http://foobar-12345678910.s3-accesspoint.us-east-1.amazonaws.com",
                         Region.of("us-east-1"),
                         S3Configuration.builder().useArnRegionEnabled(true),
                         Region.of("fips-us-east-1")))
        .isInstanceOf(IllegalArgumentException.class)
        .hasMessageContaining("FIPS");
    assertThatThrownBy(() -> verifyAccesspointArn("https",
                         "arn:aws:s3:us-east-1:12345678910:accesspoint/foobar",
                         "https://foobar-12345678910.s3-accesspoint.us-east-1.amazonaws.com",
                         Region.of("us-east-1"),
                         S3Configuration.builder().useArnRegionEnabled(true),
                         Region.of("fips-us-east-1")))
        .isInstanceOf(IllegalArgumentException.class)
        .hasMessageContaining("FIPS");
}
 

@Test
public void accesspointArn_withFipsRegionPrefix_shouldThrowIllegalArgumentException() {
    assertThatThrownBy(() -> verifyAccesspointArn("http",
                                                  "arn:aws:s3:us-east-1:12345678910:accesspoint/foobar",
                                                  "http://foobar-12345678910.s3-accesspoint.us-east-1.amazonaws.com",
                                                  Region.of("us-east-1"),
                                                  S3Configuration.builder(),
                                                  Region.of("fips-us-east-1")))
        .isInstanceOf(IllegalArgumentException.class)
        .hasMessageContaining("FIPS");
    assertThatThrownBy(() -> verifyAccesspointArn("https",
                                                  "arn:aws:s3:us-east-1:12345678910:accesspoint/foobar",
                                                  "https://foobar-12345678910.s3-accesspoint.us-east-1.amazonaws.com",
                                                  Region.of("us-east-1"),
                                                  S3Configuration.builder(),
                                                  Region.of("fips-us-east-1")))
        .isInstanceOf(IllegalArgumentException.class)
        .hasMessageContaining("FIPS");
}
 

@Test
public void accesspointArn_withFipsRegionSuffix_useArnRegionEnabled_shouldThrowIllegalArgumentException() {
    assertThatThrownBy(() -> verifyAccesspointArn("http",
                                                  "arn:aws:s3:us-east-1:12345678910:accesspoint/foobar",
                                                  "http://foobar-12345678910.s3-accesspoint.us-east-1.amazonaws.com",
                                                  Region.of("us-east-1"),
                                                  S3Configuration.builder().useArnRegionEnabled(true),
                                                  Region.of("us-east-1-fips")))
        .isInstanceOf(IllegalArgumentException.class)
        .hasMessageContaining("FIPS");
    assertThatThrownBy(() -> verifyAccesspointArn("https",
                                                  "arn:aws:s3:us-east-1:12345678910:accesspoint/foobar",
                                                  "https://foobar-12345678910.s3-accesspoint.us-east-1.amazonaws.com",
                                                  Region.of("us-east-1"),
                                                  S3Configuration.builder().useArnRegionEnabled(true),
                                                  Region.of("us-east-1-fips")))
        .isInstanceOf(IllegalArgumentException.class)
        .hasMessageContaining("FIPS");
}
 

@Test
public void accesspointArn_withFipsRegionSuffix_shouldThrowIllegalArgumentException() {
    assertThatThrownBy(() -> verifyAccesspointArn("http",
                                                  "arn:aws:s3:us-east-1:12345678910:accesspoint/foobar",
                                                  "http://foobar-12345678910.s3-accesspoint.us-east-1.amazonaws.com",
                                                  Region.of("us-east-1"),
                                                  S3Configuration.builder(),
                                                  Region.of("us-east-1-fips")))
        .isInstanceOf(IllegalArgumentException.class)
        .hasMessageContaining("FIPS");
    assertThatThrownBy(() -> verifyAccesspointArn("https",
                                                  "arn:aws:s3:us-east-1:12345678910:accesspoint/foobar",
                                                  "https://foobar-12345678910.s3-accesspoint.us-east-1.amazonaws.com",
                                                  Region.of("us-east-1"),
                                                  S3Configuration.builder(),
                                                  Region.of("us-east-1-fips")))
        .isInstanceOf(IllegalArgumentException.class)
        .hasMessageContaining("FIPS");
}
 

@Test
public void regionalSettingUnset_usesGlobalEndpoint() throws UnsupportedEncodingException {
    mockHttpClient.stubNextResponse(mockListObjectsResponse());

    S3Client s3Client = S3Client.builder()
            .credentialsProvider(StaticCredentialsProvider.create(AwsBasicCredentials.create("akid", "skid")))
            .httpClient(mockHttpClient)
            .region(Region.US_EAST_1)
            .serviceConfiguration(S3Configuration.builder()
                    .pathStyleAccessEnabled(true)
                    .build())
            .build();

    s3Client.listObjects(ListObjectsRequest.builder().bucket(BUCKET).build());
    assertThat(mockHttpClient.getLastRequest().getUri().getHost()).isEqualTo("s3.amazonaws.com");
}
 

@Test
public void regionalSettingDisabled_usesGlobalEndpoint() throws UnsupportedEncodingException {
    EnvironmentVariableHelper environmentVariableHelper = new EnvironmentVariableHelper();
    environmentVariableHelper.set(SdkSystemSetting.AWS_S3_US_EAST_1_REGIONAL_ENDPOINT.environmentVariable(), "nonregional");

    mockHttpClient.stubNextResponse(mockListObjectsResponse());

    S3Client s3Client = S3Client.builder()
            .credentialsProvider(StaticCredentialsProvider.create(AwsBasicCredentials.create("akid", "skid")))
            .httpClient(mockHttpClient)
            .region(Region.US_EAST_1)
            .serviceConfiguration(S3Configuration.builder()
                    .pathStyleAccessEnabled(true)
                    .build())
            .build();
    try {
        s3Client.listObjects(ListObjectsRequest.builder().bucket(BUCKET).build());
        assertThat(mockHttpClient.getLastRequest().getUri().getHost()).isEqualTo("s3.amazonaws.com");
    } finally {
        environmentVariableHelper.reset();
    }
}
 

private void verifyVirtualStyleConvertDnsEndpoint(String protocol) {
    String bucketName = "test-bucket";
    String key = "test-key";
    URI customUri = URI.create(String.format("%s://s3-test.com/%s/%s", protocol, bucketName, key));
    URI expectedUri = URI.create(String.format("%s://%s.s3.dualstack.us-east-1.amazonaws.com/%s", protocol,
                                               bucketName, key));

    Context.ModifyHttpRequest ctx = context(ListObjectsV2Request.builder().bucket(bucketName).build(),
                                            sdkHttpRequest(customUri));
    ExecutionAttributes executionAttributes = new ExecutionAttributes();
    S3Configuration s3Configuration = S3Configuration.builder().dualstackEnabled(true).build();

    executionAttributes.putAttribute(SERVICE_CONFIG, s3Configuration);
    executionAttributes.putAttribute(AWS_REGION, Region.US_EAST_1);

    SdkHttpRequest sdkHttpFullRequest = interceptor.modifyHttpRequest(ctx, executionAttributes);

    assertThat(sdkHttpFullRequest.getUri()).isEqualTo(expectedUri);
}
 

@Override
public SdkRequest modifyRequest(Context.ModifyRequest context, ExecutionAttributes executionAttributes) {
    SdkRequest sdkRequest = context.request();

    if (sdkRequest instanceof PutObjectRequest || sdkRequest instanceof UploadPartRequest) {
        S3Configuration serviceConfiguration =
                (S3Configuration) executionAttributes.getAttribute(AwsSignerExecutionAttribute.SERVICE_CONFIG);

        boolean enableChunkedEncoding;
        if (serviceConfiguration != null) {
            enableChunkedEncoding = serviceConfiguration.chunkedEncodingEnabled();
        } else {
            enableChunkedEncoding = true;
        }

        executionAttributes.putAttributeIfAbsent(S3SignerExecutionAttribute.ENABLE_CHUNKED_ENCODING, enableChunkedEncoding);
    }

    return sdkRequest;
}
 

@Override
public SdkHttpRequest modifyHttpRequest(Context.ModifyHttpRequest context,
                                        ExecutionAttributes executionAttributes) {
    ConfiguredS3SdkHttpRequest configuredRequest =
        S3EndpointUtils.applyEndpointConfiguration(
                context.httpRequest(),
                context.request(),
                executionAttributes.getAttribute(AwsExecutionAttribute.AWS_REGION),
                (S3Configuration) executionAttributes.getAttribute(AwsSignerExecutionAttribute.SERVICE_CONFIG),
                Boolean.TRUE.equals(executionAttributes.getAttribute(SdkExecutionAttribute.ENDPOINT_OVERRIDDEN)));

    configuredRequest.signingRegionModification().ifPresent(
        region -> executionAttributes.putAttribute(AwsSignerExecutionAttribute.SIGNING_REGION, region));

    return configuredRequest.sdkHttpRequest();
}
 

private void verifyEndpoint(String protocol, String expectedEndpoint,
                            S3Configuration.Builder builder) {
    String bucket = "test-bucket";
    String key = "test-key";
    URI customUri = URI.create(String.format("%s://s3-test.com/%s/%s", protocol, bucket, key));
    URI expectedUri = URI.create(String.format("%s/%s/%s", expectedEndpoint, bucket, key));
    Context.ModifyHttpRequest ctx = context(PutObjectRequest.builder().build(), sdkHttpRequest(customUri));
    ExecutionAttributes executionAttributes = new ExecutionAttributes();
    S3Configuration s3Configuration = builder.build();

    executionAttributes.putAttribute(SERVICE_CONFIG, s3Configuration);
    executionAttributes.putAttribute(AWS_REGION, Region.US_EAST_1);

    SdkHttpRequest sdkHttpFullRequest = interceptor.modifyHttpRequest(ctx, executionAttributes);

    assertThat(sdkHttpFullRequest.getUri()).isEqualTo(expectedUri);
}
 

/**
 * Determine which endpoint to use based on region and {@link S3Configuration}. Will either be a traditional
 * S3 endpoint (i.e. s3.us-east-1.amazonaws.com), the global S3 accelerate endpoint (i.e. s3-accelerate.amazonaws.com) or
 * a regional dualstack endpoint for IPV6 (i.e. s3.dualstack.us-east-1.amazonaws.com).
 */
private static URI resolveEndpoint(SdkHttpRequest request,
                                   SdkRequest originalRequest,
                                   Region region,
                                   S3Configuration serviceConfiguration) {

    String protocol = request.protocol();

    RegionMetadata regionMetadata = RegionMetadata.of(region);

    if (isAccelerateEnabled(serviceConfiguration) && isAccelerateSupported(originalRequest)) {
        return accelerateEndpoint(serviceConfiguration, regionMetadata, protocol);
    }

    if (serviceConfiguration != null && serviceConfiguration.dualstackEnabled()) {
        return dualstackEndpoint(regionMetadata, protocol);
    }

    return invokeSafely(() -> new URI(request.protocol(), null, request.host(), request.port(), null, null, null));
}
 

private void verifyAccesspointArn(String protocol, String accessPointArn, String expectedEndpoint,
                                  Region expectedSigningRegion,
                                  S3Configuration.Builder builder, Region region) {
    String key = "test-key";

    URI customUri = URI.create(String.format("%s://s3-test.com/%s/%s", protocol, urlEncode(accessPointArn), key));
    URI expectedUri = URI.create(String.format("%s/%s", expectedEndpoint, key));
    PutObjectRequest putObjectRequest = PutObjectRequest.builder()
                                                        .bucket(accessPointArn)
                                                        .key(key)
                                                        .build();
    Context.ModifyHttpRequest ctx = context(putObjectRequest, sdkHttpRequest(customUri));
    ExecutionAttributes executionAttributes = new ExecutionAttributes();
    S3Configuration s3Configuration = builder.build();

    executionAttributes.putAttribute(SERVICE_CONFIG, s3Configuration);
    executionAttributes.putAttribute(AWS_REGION, region);
    executionAttributes.putAttribute(SIGNING_REGION, region);

    SdkHttpRequest sdkHttpFullRequest = interceptor.modifyHttpRequest(ctx, executionAttributes);

    assertThat(executionAttributes.getAttribute(SIGNING_REGION))
        .isEqualTo(expectedSigningRegion);
    assertThat(sdkHttpFullRequest.getUri()).isEqualTo(expectedUri);
}
 

public static S3Client createS3Client(final Integer mappedPort) {
    final AwsBasicCredentials credentials = AwsBasicCredentials.create("test", "test");
    final StaticCredentialsProvider credentialsProvider = StaticCredentialsProvider.create(credentials);

    return S3Client.builder()
            .credentialsProvider(credentialsProvider)
            .endpointOverride(URI.create(String.format("http://localhost:%d", mappedPort)))
            .region(Region.EU_CENTRAL_1)
            .serviceConfiguration(S3Configuration.builder()
                    .pathStyleAccessEnabled(true)
                    .build())
            .overrideConfiguration(ClientOverrideConfiguration
                    .builder()
                    .putAdvancedOption(SdkAdvancedClientOption.SIGNER, new NoOpSigner())
                    .build())
            .build();
}
 

@Test
public void accesspointArn_futureUnknownRegion_crossRegion_correctlyInfersPartition() {
    verifyAccesspointArn("http",
                         "arn:aws:s3:us-future-2:12345678910:accesspoint:foobar",
                         "http://foobar-12345678910.s3-accesspoint.us-future-2.amazonaws.com",
                         Region.of("us-future-2"),
                         S3Configuration.builder().useArnRegionEnabled(true),
                         Region.of("us-future-1"));
}
 

@Test
public void accesspointArn_futureUnknownRegion_CN_correctlyInfersPartition() {
    verifyAccesspointArn("http",
                         "arn:aws-cn:s3:cn-future-1:12345678910:accesspoint:foobar",
                         "http://foobar-12345678910.s3-accesspoint.cn-future-1.amazonaws.com.cn",
                         Region.of("cn-future-1"),
                         S3Configuration.builder(),
                         Region.of("cn-future-1"));
}
 

@Test
public void accesspointArn_futureUnknownRegionAndPartition_defaultsToAws() {
    verifyAccesspointArn("http",
                         "arn:aws:s3:unknown:12345678910:accesspoint:foobar",
                         "http://foobar-12345678910.s3-accesspoint.unknown.amazonaws.com",
                         Region.of("unknown"),
                         S3Configuration.builder(),
                         Region.of("unknown"));
}
 

@Test
public void malformedArn_throwsIllegalArgumentException() {
    assertThatThrownBy(() -> verifyAccesspointArn("http",
                         "arn:foobar",
                         null,
                         S3Configuration.builder()))
        .isInstanceOf(IllegalArgumentException.class)
        .hasMessageContaining("ARN");
}
 

@Test
public void unsupportedArn_throwsIllegalArgumentException() {
    assertThatThrownBy(() -> verifyAccesspointArn("http",
                                                  "arn:aws:s3:us-east-1:12345678910:unsupported:foobar",
                                                  null,
                                                  S3Configuration.builder()))
        .isInstanceOf(IllegalArgumentException.class)
        .hasMessageContaining("ARN");
}
 

@Test
public void accesspointArn_invalidPartition_throwsIllegalArgumentException() {
    assertThatThrownBy(() -> verifyAccesspointArn("http",
                                                  "arn:bar:s3:us-east-1:12345678910:accesspoint:foobar",
                                                  null,
                                                  S3Configuration.builder()))
        .isInstanceOf(IllegalArgumentException.class)
        .hasMessageContaining("bar");
}
 

@Test
public void bucketArn_throwsIllegalArgumentException() {
    assertThatThrownBy(() -> verifyAccesspointArn("http",
                                                  "arn:aws:s3:us-east-1:12345678910:bucket_name:foobar",
                                                  null,
                                                  S3Configuration.builder()))
        .isInstanceOf(IllegalArgumentException.class)
        .hasMessageContaining("bucket parameter");
}
 

@Test
public void pathStyleAccessEnabled_shouldNotConvertToDnsEndpoint() {
    verifyEndpoint("http", "http://s3-test.com",
                   S3Configuration.builder().pathStyleAccessEnabled(true));
    verifyEndpoint("https", "https://s3-test.com",
                   S3Configuration.builder().pathStyleAccessEnabled(true));
}
 

@Test
public void accesspointArn_withSlashes_shouldConvertEndpoint() {
    verifyAccesspointArn("http",
                         "arn:aws:s3:us-east-1:12345678910:accesspoint/foobar",
                         "http://foobar-12345678910.s3-accesspoint.us-east-1.amazonaws.com",
                         S3Configuration.builder());
    verifyAccesspointArn("https",
                         "arn:aws:s3:us-east-1:12345678910:accesspoint/foobar",
                         "https://foobar-12345678910.s3-accesspoint.us-east-1.amazonaws.com",
                         S3Configuration.builder());
}
 

@Test
public void accesspointArn_withDualStackEnabled_shouldConvertEndpoint() {
    verifyAccesspointArn("http",
                         "arn:aws:s3:us-east-1:12345678910:accesspoint/foobar",
                         "http://foobar-12345678910.s3-accesspoint.dualstack.us-east-1.amazonaws.com",
                         S3Configuration.builder().dualstackEnabled(true));
    verifyAccesspointArn("https",
                         "arn:aws:s3:us-east-1:12345678910:accesspoint/foobar",
                         "https://foobar-12345678910.s3-accesspoint.dualstack.us-east-1.amazonaws.com",
                         S3Configuration.builder().dualstackEnabled(true));
}
 

@Test
public void accesspointArn_withCnPartition_shouldConvertEndpoint() {
    verifyAccesspointArn("http",
                         "arn:aws-cn:s3:cn-north-1:12345678910:accesspoint:foobar",
                         "http://foobar-12345678910.s3-accesspoint.cn-north-1.amazonaws.com.cn",
                         Region.of("cn-north-1"),
                         S3Configuration.builder(),
                         Region.of("cn-north-1"));
    verifyAccesspointArn("https",
                         "arn:aws-cn:s3:cn-north-1:12345678910:accesspoint:foobar",
                         "https://foobar-12345678910.s3-accesspoint.cn-north-1.amazonaws.com.cn",
                         Region.of("cn-north-1"),
                         S3Configuration.builder(),
                         Region.of("cn-north-1"));
}
 

@Test
public void accesspointArn_withDifferentPartition_useArnRegionEnabled_shouldThrowIllegalArgumentException() {
    assertThatThrownBy(() -> verifyAccesspointArn("http",
                                                  "arn:aws-cn:s3:cn-north-1:12345678910:accesspoint:foobar",
                                                  "http://foobar-12345678910.s3-accesspoint.cn-north-1.amazonaws.com.cn",
                                                  Region.of("cn-north-1"),
                                                  S3Configuration.builder().useArnRegionEnabled(true),
                                                  Region.of("us-east-1")))
        .isInstanceOf(IllegalArgumentException.class)
        .hasMessageContaining("partition");
}
 

@Test
public void modifyHttpRequest_getObjectTrailingChecksumDisabled_shouldNotModifyHttpRequest() {
    Context.ModifyHttpRequest modifyHttpRequestContext =
        modifyHttpRequestContext(GetObjectRequest.builder().build());

    SdkHttpRequest sdkHttpRequest = interceptor.modifyHttpRequest(modifyHttpRequestContext,
                                                                  new ExecutionAttributes().putAttribute(AwsSignerExecutionAttribute.SERVICE_CONFIG,
                                                                                                         S3Configuration.builder().checksumValidationEnabled(false).build()));

    assertThat(sdkHttpRequest).isEqualToComparingFieldByField(modifyHttpRequestContext.httpRequest());
}
 

@Before
public void setup() {
    s3Client = S3Client.builder()
                       .credentialsProvider(StaticCredentialsProvider.create(AwsBasicCredentials.create("akid", "skid")))
                       .region(Region.US_WEST_2).endpointOverride(URI.create(getEndpoint()))
                       .serviceConfiguration(S3Configuration.builder().checksumValidationEnabled(false).build())
                       .build();
    putObjectRequest = PutObjectRequest.builder().bucket("test").key("test").build();
}