com.amazonaws.services.s3.model.Region Java Examples

private String regionForUri(AmazonS3 client, AmazonS3URI uri) {
  String bucketRegion = client.getBucketLocation(uri.getBucket());
  Region region = Region.fromValue(bucketRegion);

  // S3 doesn't have a US East 1 region, US East 1 is really the region
  // US Standard. US Standard places the data in either an east coast
  // or west coast data center geographically closest to you.
  // SigV4 requires you to mention a region while signing a request
  // and for the S3's US standard endpoints the value to be used is "us-east-1"
  // US West 1 has an endpoint and so is treated as a stand alone region,
  // US East 1 doesn't and so is bundled into US Standard
  if (region.equals(Region.US_Standard)) {
    bucketRegion = "us-east-1";
  } else {
    bucketRegion = region.toString();
  }
  return bucketRegion;
}
 

/**
 * Create an Amazon S3 bucket if it does not exist.
 * 
 * @param client
 *        The {@link AmazonS3Client} with read and write permissions
 * @param bucketName
 *        The bucket to create
 * @throws IllegalStateException
 *         The bucket is not created before timeout occurs
 */
public static void createBucket(AmazonS3Client client, String bucketName) {
    if (!bucketExists(client, bucketName)) {
        CreateBucketRequest createBucketRequest = new CreateBucketRequest(bucketName);
        createBucketRequest.setRegion(Region.US_Standard.toString());
        client.createBucket(createBucketRequest);
    }
    long startTime = System.currentTimeMillis();
    long endTime = startTime + 60 * 1000;
    while (!bucketExists(client, bucketName) && endTime > System.currentTimeMillis()) {
        try {
            LOG.info("Waiting for Amazon S3 to create bucket " + bucketName);
            Thread.sleep(1000 * 10);
        } catch (InterruptedException e) {
        }
    }
    if (!bucketExists(client, bucketName)) {
        throw new IllegalStateException("Could not create bucket " + bucketName);
    }
    LOG.info("Created Amazon S3 bucket " + bucketName);
}
 

private AmazonS3 newGlobalInstance(
    S3S3CopierOptions s3s3CopierOptions,
    HadoopAWSCredentialProviderChain credentialsChain) {
  AmazonS3ClientBuilder builder = AmazonS3ClientBuilder
      .standard()
      .withForceGlobalBucketAccessEnabled(Boolean.TRUE)
      .withCredentials(credentialsChain);

  applyClientConfigurations(builder, s3s3CopierOptions);

  URI s3Endpoint = s3s3CopierOptions.getS3Endpoint();
  if (s3Endpoint != null) {
    EndpointConfiguration endpointConfiguration = new EndpointConfiguration(s3Endpoint.toString(),
        Region.US_Standard.getFirstRegionId());
    builder.withEndpointConfiguration(endpointConfiguration);
  }
  return builder.build();
}
 

public Bucket createBucketForInstance(String instanceId, ServiceDefinition service, String planId,
        String organizationGuid, String spaceGuid) {
    String bucketName = getBucketNameForInstance(instanceId);
    logger.info("Creating bucket '{}' for serviceInstanceId '{}'", bucketName, instanceId);
    Bucket bucket = s3.createBucket(bucketName, Region.fromValue(region));

    // TODO allow for additional, custom tagging options
    BucketTaggingConfiguration bucketTaggingConfiguration = new BucketTaggingConfiguration();
    TagSet tagSet = new TagSet();
    tagSet.setTag("serviceInstanceId", instanceId);
    tagSet.setTag("serviceDefinitionId", service.getId());
    tagSet.setTag("planId", planId);
    tagSet.setTag("organizationGuid", organizationGuid);
    tagSet.setTag("spaceGuid", spaceGuid);
    bucketTaggingConfiguration.withTagSets(tagSet);
    s3.setBucketTaggingConfiguration(bucket.getName(), bucketTaggingConfiguration);

    return bucket;
}
 

@Test
@Override
public void test() throws Exception {
	for (int i = 1; i <= 2; i++) {
		Message<?> received = this.messageCollector.forChannel(this.channels.output())
				.poll(10, TimeUnit.SECONDS);
		assertNotNull(received);
		assertThat(received, hasPayload(new File(this.config.getLocalDir(), i + ".test")));
	}

	assertEquals(2, this.config.getLocalDir().list().length);

	AWSCredentialsProvider awsCredentialsProvider =
			TestUtils.getPropertyValue(this.amazonS3, "awsCredentialsProvider", AWSCredentialsProvider.class);

	AWSCredentials credentials = awsCredentialsProvider.getCredentials();
	assertEquals(AWS_ACCESS_KEY, credentials.getAWSAccessKeyId());
	assertEquals(AWS_SECRET_KEY, credentials.getAWSSecretKey());

	assertEquals(Region.US_GovCloud, this.amazonS3.getRegion());
	assertEquals(new URI("https://s3-us-gov-west-1.amazonaws.com"),
			TestUtils.getPropertyValue(this.amazonS3, "endpoint"));
}
 

public AwsFileSystem(AwsConfiguration config, AwsFsConfiguration fsConfig) {
    AWSCredentials creds = new BasicAWSCredentials(config.getAccessKey(), config.getSecretKey());
    this.s3 = AmazonS3ClientBuilder
            .standard()
            .withRegion(config.getS3BucketRegionId())
            .withCredentials(new AWSStaticCredentialsProvider(creds))
            .build();

    this.cloudFrontBaseUrl = config.getCloudFrontBaseUrl();

    this.bucketName = fsConfig.getS3BucketName();
    if (!s3.doesBucketExistV2(bucketName)) {
        s3.createBucket(new CreateBucketRequest(bucketName, Region.fromValue(config.getS3BucketRegionId())));
    }
}
 

@Test
public void testProvenance() throws InitializationException {
    final String PROV1_FILE = "provfile1";

    final PutS3Object processor = new PutS3Object();
    final TestRunner runner = TestRunners.newTestRunner(processor);

    runner.setProperty(PutS3Object.CREDENTIALS_FILE, CREDENTIALS_FILE);
    runner.setProperty(PutS3Object.REGION, REGION);
    runner.setProperty(PutS3Object.BUCKET, BUCKET_NAME);
    runner.setProperty(PutS3Object.KEY, "${filename}");

    Map<String, String> attribs = new HashMap<>();
    attribs.put(CoreAttributes.FILENAME.key(), PROV1_FILE);
    runner.enqueue("prov1 contents".getBytes(), attribs);

    runner.assertValid();
    runner.run();
    runner.assertAllFlowFilesTransferred(PutS3Object.REL_SUCCESS);
    final List<MockFlowFile> successFiles = runner.getFlowFilesForRelationship(PutS3Object.REL_SUCCESS);
    Assert.assertEquals(1, successFiles.size());

    final List<ProvenanceEventRecord> provenanceEvents = runner.getProvenanceEvents();
    Assert.assertEquals(1, provenanceEvents.size());
    ProvenanceEventRecord provRec1 = provenanceEvents.get(0);
    Assert.assertEquals(ProvenanceEventType.SEND, provRec1.getEventType());
    Assert.assertEquals(processor.getIdentifier(), provRec1.getComponentId());
    client.setRegion(Region.fromValue(REGION).toAWSRegion());
    String targetUri = client.getUrl(BUCKET_NAME, PROV1_FILE).toString();
    Assert.assertEquals(targetUri, provRec1.getTransitUri());
    Assert.assertEquals(8, provRec1.getUpdatedAttributes().size());
    Assert.assertEquals(BUCKET_NAME, provRec1.getUpdatedAttributes().get(PutS3Object.S3_BUCKET_KEY));
}
 

@Test
void getUri_encodes_objectName() throws Exception {
	AmazonS3 s3 = mock(AmazonS3.class);
	when(s3.getRegion()).thenReturn(Region.US_West_2);
	SimpleStorageResource resource = new SimpleStorageResource(s3, "bucketName",
			"some/[objectName]", new SyncTaskExecutor());

	assertThat(resource.getURI()).isEqualTo(new URI(
			"https://s3.us-west-2.amazonaws.com/bucketName/some%2F%5BobjectName%5D"));
}
 

static software.amazon.awssdk.regions.Region getAwsRegionFromEndpoint(String endpoint) {
  // Determine if one of the known AWS regions is contained within the given endpoint, and return that region if so.
  return Optional.ofNullable(endpoint)
    .map(e -> e.toLowerCase(Locale.ROOT)) // lower-case the endpoint for easy detection
    .filter(e -> e.endsWith(S3_ENDPOINT_END) || e.endsWith(S3_CN_ENDPOINT_END)) // omit any semi-malformed endpoints
    .flatMap(e -> software.amazon.awssdk.regions.Region.regions()
      .stream()
      .filter(region -> e.contains(region.id()))
      .findFirst()) // map the endpoint to the region contained within it, if any
    .orElse(software.amazon.awssdk.regions.Region.US_EAST_1); // default to US_EAST_1 if no regions are found.
}
 

static software.amazon.awssdk.regions.Region getAWSRegionFromConfigurationOrDefault(Configuration conf) {
  final String regionOverride = conf.getTrimmed(REGION_OVERRIDE);
  if (!Strings.isNullOrEmpty(regionOverride)) {
    // set the region to what the user provided unless they provided an empty string.
    return software.amazon.awssdk.regions.Region.of(regionOverride);
  }

  return getAwsRegionFromEndpoint(conf.get(Constants.ENDPOINT));
}
 

@Test
public void typical() {
  Configuration conf = new Configuration();
  conf.set(ConfigurationVariable.REGION.getName(), "eu-west-1");
  conf.setInt(ConfigurationVariable.UPLOAD_RETRY_COUNT.getName(), 7);
  conf.setLong(ConfigurationVariable.UPLOAD_RETRY_DELAY_MS.getName(), 333L);
  AmazonS3 client = factory.newInstance(conf);
  assertThat(client, is(instanceOf(AmazonS3Client.class)));
  assertThat(client.getRegion(), is(Region.EU_Ireland));
}
 

@Override
public void validate(String name, String value) throws ParameterException {
  try {
    Region.fromValue(value);
  } catch (IllegalArgumentException e) {
    throw new ParameterException("Parameter " + name + " is not a valid AWS region (found " + value + ")", e);
  }
}
 

@Test
public void testProvenance() throws InitializationException {
    final String PROV1_FILE = "provfile1";

    final PutS3Object processor = new PutS3Object();
    final TestRunner runner = TestRunners.newTestRunner(processor);

    runner.setProperty(PutS3Object.CREDENTIALS_FILE, CREDENTIALS_FILE);
    runner.setProperty(PutS3Object.REGION, REGION);
    runner.setProperty(PutS3Object.BUCKET, BUCKET_NAME);
    runner.setProperty(PutS3Object.KEY, "${filename}");

    Map<String, String> attribs = new HashMap<>();
    attribs.put(CoreAttributes.FILENAME.key(), PROV1_FILE);
    runner.enqueue("prov1 contents".getBytes(), attribs);

    runner.assertValid();
    runner.run();
    runner.assertAllFlowFilesTransferred(PutS3Object.REL_SUCCESS);
    final List<MockFlowFile> successFiles = runner.getFlowFilesForRelationship(PutS3Object.REL_SUCCESS);
    Assert.assertEquals(1, successFiles.size());

    final List<ProvenanceEventRecord> provenanceEvents = runner.getProvenanceEvents();
    Assert.assertEquals(1, provenanceEvents.size());
    ProvenanceEventRecord provRec1 = provenanceEvents.get(0);
    Assert.assertEquals(ProvenanceEventType.SEND, provRec1.getEventType());
    Assert.assertEquals(processor.getIdentifier(), provRec1.getComponentId());
    client.setRegion(Region.fromValue(REGION).toAWSRegion());
    String targetUri = client.getUrl(BUCKET_NAME, PROV1_FILE).toString();
    Assert.assertEquals(targetUri, provRec1.getTransitUri());
    Assert.assertEquals(7, provRec1.getUpdatedAttributes().size());
    Assert.assertEquals(BUCKET_NAME, provRec1.getUpdatedAttributes().get(PutS3Object.S3_BUCKET_KEY));
}
 

@Override
protected ContainerHolder toContainerHolder() throws IOException {

  return new ContainerHolder(bucketName, new FileSystemSupplier() {
    @Override
    public FileSystem create() throws IOException {
      final String targetEndpoint;
      Optional<String> endpoint = getEndpoint(getConf());

      if (isCompatMode() && endpoint.isPresent()) {
        // if this is compatibility mode and we have an endpoint, just use that.
        targetEndpoint = endpoint.get();
      } else {
        try {
          final String bucketRegion = s3.getBucketLocation(bucketName);
          final String fallbackEndpoint = endpoint.orElseGet(() -> String.format("%ss3.%s.amazonaws.com", getHttpScheme(getConf()), bucketRegion));

          String regionEndpoint = fallbackEndpoint;
          try {
            Region region = Region.fromValue(bucketRegion);
            com.amazonaws.regions.Region awsRegion = region.toAWSRegion();
            if (awsRegion != null) {
              regionEndpoint = awsRegion.getServiceEndpoint("s3");
            }
          } catch (IllegalArgumentException iae) {
            // try heuristic mapping if not found
            regionEndpoint = fallbackEndpoint;
            logger.warn("Unknown or unmapped region {} for bucket {}. Will use following endpoint: {}",
              bucketRegion, bucketName, regionEndpoint);
          }
          // it could be null because no mapping from Region to aws region or there is no such region is the map of endpoints
          // not sure if latter is possible
          if (regionEndpoint == null) {
            logger.error("Could not get AWSRegion for bucket {}. Will use following fs.s3a.endpoint: {} ",
              bucketName, fallbackEndpoint);
          }
          targetEndpoint = (regionEndpoint != null) ? regionEndpoint : fallbackEndpoint;

        } catch (AmazonS3Exception aex) {
          if (aex.getStatusCode() == 403) {
            throw UserException.permissionError(aex)
              .message(S3_PERMISSION_ERROR_MSG)
              .build(logger);
          }
          throw aex;
        }
      }

      String location = S3_URI_SCHEMA + bucketName + "/";
      final Configuration bucketConf = new Configuration(parentConf);
      bucketConf.set(ENDPOINT, targetEndpoint);
      return fsCache.get(new Path(location).toUri(), bucketConf, S3ClientKey.UNIQUE_PROPS);
    }
  });
}
 

private software.amazon.awssdk.regions.Region getAWSBucketRegion(String bucketName) throws SdkClientException {
  final String awsRegionName = Region.fromValue(s3.getBucketLocation(bucketName)).toAWSRegion().getName();
  return software.amazon.awssdk.regions.Region.of(awsRegionName);
}
 

protected final void createBucket(final String name, final String policy) {
    this.s3.createBucket(new CreateBucketRequest(name, Region.fromValue(this.getRegion())));
    this.s3.setBucketPolicy(name, policy);
}
 

@Test
void getUrl_existingObject_returnsUrlWithS3Prefix() throws Exception {

	AmazonS3Client amazonS3 = mock(AmazonS3Client.class);

	when(amazonS3.getRegion()).thenReturn(Region.EU_Ireland);

	// Act
	SimpleStorageResource simpleStorageResource = new SimpleStorageResource(amazonS3,
			"bucket", "object", new SyncTaskExecutor());

	// Assert
	assertThat(simpleStorageResource.getURL())
			.isEqualTo(new URL("https://s3.eu-west-1.amazonaws.com/bucket/object"));

}
 

void setRegion(String region) {
  Region.fromValue(region);
  this.region = region;
}
 

boolean isLocal() {
  return Arrays.stream(Region.values()).noneMatch(r -> r.toAWSRegion().getName().equals(arn.getRegion()));
}