com.amazonaws.services.s3.model.Region Java Examples
The following examples show how to use
com.amazonaws.services.s3.model.Region.
You can vote up the ones you like or vote down the ones you don't like,
and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example #1
Source File: JceksAmazonS3ClientFactory.java From circus-train with Apache License 2.0 | 7 votes |
private String regionForUri(AmazonS3 client, AmazonS3URI uri) { String bucketRegion = client.getBucketLocation(uri.getBucket()); Region region = Region.fromValue(bucketRegion); // S3 doesn't have a US East 1 region, US East 1 is really the region // US Standard. US Standard places the data in either an east coast // or west coast data center geographically closest to you. // SigV4 requires you to mention a region while signing a request // and for the S3's US standard endpoints the value to be used is "us-east-1" // US West 1 has an endpoint and so is treated as a stand alone region, // US East 1 doesn't and so is bundled into US Standard if (region.equals(Region.US_Standard)) { bucketRegion = "us-east-1"; } else { bucketRegion = region.toString(); } return bucketRegion; }
Example #2
Source File: S3Utils.java From amazon-kinesis-connectors with Apache License 2.0 | 6 votes |
/** * Create an Amazon S3 bucket if it does not exist. * * @param client * The {@link AmazonS3Client} with read and write permissions * @param bucketName * The bucket to create * @throws IllegalStateException * The bucket is not created before timeout occurs */ public static void createBucket(AmazonS3Client client, String bucketName) { if (!bucketExists(client, bucketName)) { CreateBucketRequest createBucketRequest = new CreateBucketRequest(bucketName); createBucketRequest.setRegion(Region.US_Standard.toString()); client.createBucket(createBucketRequest); } long startTime = System.currentTimeMillis(); long endTime = startTime + 60 * 1000; while (!bucketExists(client, bucketName) && endTime > System.currentTimeMillis()) { try { LOG.info("Waiting for Amazon S3 to create bucket " + bucketName); Thread.sleep(1000 * 10); } catch (InterruptedException e) { } } if (!bucketExists(client, bucketName)) { throw new IllegalStateException("Could not create bucket " + bucketName); } LOG.info("Created Amazon S3 bucket " + bucketName); }
Example #3
Source File: JceksAmazonS3ClientFactory.java From circus-train with Apache License 2.0 | 6 votes |
private AmazonS3 newGlobalInstance( S3S3CopierOptions s3s3CopierOptions, HadoopAWSCredentialProviderChain credentialsChain) { AmazonS3ClientBuilder builder = AmazonS3ClientBuilder .standard() .withForceGlobalBucketAccessEnabled(Boolean.TRUE) .withCredentials(credentialsChain); applyClientConfigurations(builder, s3s3CopierOptions); URI s3Endpoint = s3s3CopierOptions.getS3Endpoint(); if (s3Endpoint != null) { EndpointConfiguration endpointConfiguration = new EndpointConfiguration(s3Endpoint.toString(), Region.US_Standard.getFirstRegionId()); builder.withEndpointConfiguration(endpointConfiguration); } return builder.build(); }
Example #4
Source File: S3.java From s3-cf-service-broker with Apache License 2.0 | 6 votes |
public Bucket createBucketForInstance(String instanceId, ServiceDefinition service, String planId, String organizationGuid, String spaceGuid) { String bucketName = getBucketNameForInstance(instanceId); logger.info("Creating bucket '{}' for serviceInstanceId '{}'", bucketName, instanceId); Bucket bucket = s3.createBucket(bucketName, Region.fromValue(region)); // TODO allow for additional, custom tagging options BucketTaggingConfiguration bucketTaggingConfiguration = new BucketTaggingConfiguration(); TagSet tagSet = new TagSet(); tagSet.setTag("serviceInstanceId", instanceId); tagSet.setTag("serviceDefinitionId", service.getId()); tagSet.setTag("planId", planId); tagSet.setTag("organizationGuid", organizationGuid); tagSet.setTag("spaceGuid", spaceGuid); bucketTaggingConfiguration.withTagSets(tagSet); s3.setBucketTaggingConfiguration(bucket.getName(), bucketTaggingConfiguration); return bucket; }
Example #5
Source File: AmazonS3SourceMockTests.java From spring-cloud-stream-app-starters with Apache License 2.0 | 6 votes |
@Test @Override public void test() throws Exception { for (int i = 1; i <= 2; i++) { Message<?> received = this.messageCollector.forChannel(this.channels.output()) .poll(10, TimeUnit.SECONDS); assertNotNull(received); assertThat(received, hasPayload(new File(this.config.getLocalDir(), i + ".test"))); } assertEquals(2, this.config.getLocalDir().list().length); AWSCredentialsProvider awsCredentialsProvider = TestUtils.getPropertyValue(this.amazonS3, "awsCredentialsProvider", AWSCredentialsProvider.class); AWSCredentials credentials = awsCredentialsProvider.getCredentials(); assertEquals(AWS_ACCESS_KEY, credentials.getAWSAccessKeyId()); assertEquals(AWS_SECRET_KEY, credentials.getAWSSecretKey()); assertEquals(Region.US_GovCloud, this.amazonS3.getRegion()); assertEquals(new URI("https://s3-us-gov-west-1.amazonaws.com"), TestUtils.getPropertyValue(this.amazonS3, "endpoint")); }
Example #6
Source File: AwsFileSystem.java From judgels with GNU General Public License v2.0 | 5 votes |
public AwsFileSystem(AwsConfiguration config, AwsFsConfiguration fsConfig) { AWSCredentials creds = new BasicAWSCredentials(config.getAccessKey(), config.getSecretKey()); this.s3 = AmazonS3ClientBuilder .standard() .withRegion(config.getS3BucketRegionId()) .withCredentials(new AWSStaticCredentialsProvider(creds)) .build(); this.cloudFrontBaseUrl = config.getCloudFrontBaseUrl(); this.bucketName = fsConfig.getS3BucketName(); if (!s3.doesBucketExistV2(bucketName)) { s3.createBucket(new CreateBucketRequest(bucketName, Region.fromValue(config.getS3BucketRegionId()))); } }
Example #7
Source File: ITPutS3Object.java From nifi with Apache License 2.0 | 5 votes |
@Test public void testProvenance() throws InitializationException { final String PROV1_FILE = "provfile1"; final PutS3Object processor = new PutS3Object(); final TestRunner runner = TestRunners.newTestRunner(processor); runner.setProperty(PutS3Object.CREDENTIALS_FILE, CREDENTIALS_FILE); runner.setProperty(PutS3Object.REGION, REGION); runner.setProperty(PutS3Object.BUCKET, BUCKET_NAME); runner.setProperty(PutS3Object.KEY, "${filename}"); Map<String, String> attribs = new HashMap<>(); attribs.put(CoreAttributes.FILENAME.key(), PROV1_FILE); runner.enqueue("prov1 contents".getBytes(), attribs); runner.assertValid(); runner.run(); runner.assertAllFlowFilesTransferred(PutS3Object.REL_SUCCESS); final List<MockFlowFile> successFiles = runner.getFlowFilesForRelationship(PutS3Object.REL_SUCCESS); Assert.assertEquals(1, successFiles.size()); final List<ProvenanceEventRecord> provenanceEvents = runner.getProvenanceEvents(); Assert.assertEquals(1, provenanceEvents.size()); ProvenanceEventRecord provRec1 = provenanceEvents.get(0); Assert.assertEquals(ProvenanceEventType.SEND, provRec1.getEventType()); Assert.assertEquals(processor.getIdentifier(), provRec1.getComponentId()); client.setRegion(Region.fromValue(REGION).toAWSRegion()); String targetUri = client.getUrl(BUCKET_NAME, PROV1_FILE).toString(); Assert.assertEquals(targetUri, provRec1.getTransitUri()); Assert.assertEquals(8, provRec1.getUpdatedAttributes().size()); Assert.assertEquals(BUCKET_NAME, provRec1.getUpdatedAttributes().get(PutS3Object.S3_BUCKET_KEY)); }
Example #8
Source File: SimpleStorageResourceTest.java From spring-cloud-aws with Apache License 2.0 | 5 votes |
@Test void getUri_encodes_objectName() throws Exception { AmazonS3 s3 = mock(AmazonS3.class); when(s3.getRegion()).thenReturn(Region.US_West_2); SimpleStorageResource resource = new SimpleStorageResource(s3, "bucketName", "some/[objectName]", new SyncTaskExecutor()); assertThat(resource.getURI()).isEqualTo(new URI( "https://s3.us-west-2.amazonaws.com/bucketName/some%2F%5BobjectName%5D")); }
Example #9
Source File: S3FileSystem.java From dremio-oss with Apache License 2.0 | 5 votes |
static software.amazon.awssdk.regions.Region getAwsRegionFromEndpoint(String endpoint) { // Determine if one of the known AWS regions is contained within the given endpoint, and return that region if so. return Optional.ofNullable(endpoint) .map(e -> e.toLowerCase(Locale.ROOT)) // lower-case the endpoint for easy detection .filter(e -> e.endsWith(S3_ENDPOINT_END) || e.endsWith(S3_CN_ENDPOINT_END)) // omit any semi-malformed endpoints .flatMap(e -> software.amazon.awssdk.regions.Region.regions() .stream() .filter(region -> e.contains(region.id())) .findFirst()) // map the endpoint to the region contained within it, if any .orElse(software.amazon.awssdk.regions.Region.US_EAST_1); // default to US_EAST_1 if no regions are found. }
Example #10
Source File: S3FileSystem.java From dremio-oss with Apache License 2.0 | 5 votes |
static software.amazon.awssdk.regions.Region getAWSRegionFromConfigurationOrDefault(Configuration conf) { final String regionOverride = conf.getTrimmed(REGION_OVERRIDE); if (!Strings.isNullOrEmpty(regionOverride)) { // set the region to what the user provided unless they provided an empty string. return software.amazon.awssdk.regions.Region.of(regionOverride); } return getAwsRegionFromEndpoint(conf.get(Constants.ENDPOINT)); }
Example #11
Source File: AwsS3ClientFactoryTest.java From circus-train with Apache License 2.0 | 5 votes |
@Test public void typical() { Configuration conf = new Configuration(); conf.set(ConfigurationVariable.REGION.getName(), "eu-west-1"); conf.setInt(ConfigurationVariable.UPLOAD_RETRY_COUNT.getName(), 7); conf.setLong(ConfigurationVariable.UPLOAD_RETRY_DELAY_MS.getName(), 333L); AmazonS3 client = factory.newInstance(conf); assertThat(client, is(instanceOf(AmazonS3Client.class))); assertThat(client.getRegion(), is(Region.EU_Ireland)); }
Example #12
Source File: RegionValidator.java From circus-train with Apache License 2.0 | 5 votes |
@Override public void validate(String name, String value) throws ParameterException { try { Region.fromValue(value); } catch (IllegalArgumentException e) { throw new ParameterException("Parameter " + name + " is not a valid AWS region (found " + value + ")", e); } }
Example #13
Source File: ITPutS3Object.java From localization_nifi with Apache License 2.0 | 5 votes |
@Test public void testProvenance() throws InitializationException { final String PROV1_FILE = "provfile1"; final PutS3Object processor = new PutS3Object(); final TestRunner runner = TestRunners.newTestRunner(processor); runner.setProperty(PutS3Object.CREDENTIALS_FILE, CREDENTIALS_FILE); runner.setProperty(PutS3Object.REGION, REGION); runner.setProperty(PutS3Object.BUCKET, BUCKET_NAME); runner.setProperty(PutS3Object.KEY, "${filename}"); Map<String, String> attribs = new HashMap<>(); attribs.put(CoreAttributes.FILENAME.key(), PROV1_FILE); runner.enqueue("prov1 contents".getBytes(), attribs); runner.assertValid(); runner.run(); runner.assertAllFlowFilesTransferred(PutS3Object.REL_SUCCESS); final List<MockFlowFile> successFiles = runner.getFlowFilesForRelationship(PutS3Object.REL_SUCCESS); Assert.assertEquals(1, successFiles.size()); final List<ProvenanceEventRecord> provenanceEvents = runner.getProvenanceEvents(); Assert.assertEquals(1, provenanceEvents.size()); ProvenanceEventRecord provRec1 = provenanceEvents.get(0); Assert.assertEquals(ProvenanceEventType.SEND, provRec1.getEventType()); Assert.assertEquals(processor.getIdentifier(), provRec1.getComponentId()); client.setRegion(Region.fromValue(REGION).toAWSRegion()); String targetUri = client.getUrl(BUCKET_NAME, PROV1_FILE).toString(); Assert.assertEquals(targetUri, provRec1.getTransitUri()); Assert.assertEquals(7, provRec1.getUpdatedAttributes().size()); Assert.assertEquals(BUCKET_NAME, provRec1.getUpdatedAttributes().get(PutS3Object.S3_BUCKET_KEY)); }
Example #14
Source File: S3FileSystem.java From dremio-oss with Apache License 2.0 | 4 votes |
@Override protected ContainerHolder toContainerHolder() throws IOException { return new ContainerHolder(bucketName, new FileSystemSupplier() { @Override public FileSystem create() throws IOException { final String targetEndpoint; Optional<String> endpoint = getEndpoint(getConf()); if (isCompatMode() && endpoint.isPresent()) { // if this is compatibility mode and we have an endpoint, just use that. targetEndpoint = endpoint.get(); } else { try { final String bucketRegion = s3.getBucketLocation(bucketName); final String fallbackEndpoint = endpoint.orElseGet(() -> String.format("%ss3.%s.amazonaws.com", getHttpScheme(getConf()), bucketRegion)); String regionEndpoint = fallbackEndpoint; try { Region region = Region.fromValue(bucketRegion); com.amazonaws.regions.Region awsRegion = region.toAWSRegion(); if (awsRegion != null) { regionEndpoint = awsRegion.getServiceEndpoint("s3"); } } catch (IllegalArgumentException iae) { // try heuristic mapping if not found regionEndpoint = fallbackEndpoint; logger.warn("Unknown or unmapped region {} for bucket {}. Will use following endpoint: {}", bucketRegion, bucketName, regionEndpoint); } // it could be null because no mapping from Region to aws region or there is no such region is the map of endpoints // not sure if latter is possible if (regionEndpoint == null) { logger.error("Could not get AWSRegion for bucket {}. Will use following fs.s3a.endpoint: {} ", bucketName, fallbackEndpoint); } targetEndpoint = (regionEndpoint != null) ? regionEndpoint : fallbackEndpoint; } catch (AmazonS3Exception aex) { if (aex.getStatusCode() == 403) { throw UserException.permissionError(aex) .message(S3_PERMISSION_ERROR_MSG) .build(logger); } throw aex; } } String location = S3_URI_SCHEMA + bucketName + "/"; final Configuration bucketConf = new Configuration(parentConf); bucketConf.set(ENDPOINT, targetEndpoint); return fsCache.get(new Path(location).toUri(), bucketConf, S3ClientKey.UNIQUE_PROPS); } }); }
Example #15
Source File: S3FileSystem.java From dremio-oss with Apache License 2.0 | 4 votes |
private software.amazon.awssdk.regions.Region getAWSBucketRegion(String bucketName) throws SdkClientException { final String awsRegionName = Region.fromValue(s3.getBucketLocation(bucketName)).toAWSRegion().getName(); return software.amazon.awssdk.regions.Region.of(awsRegionName); }
Example #16
Source File: AAWSTest.java From aws-cf-templates with Apache License 2.0 | 4 votes |
protected final void createBucket(final String name, final String policy) { this.s3.createBucket(new CreateBucketRequest(name, Region.fromValue(this.getRegion()))); this.s3.setBucketPolicy(name, policy); }
Example #17
Source File: SimpleStorageResourceTest.java From spring-cloud-aws with Apache License 2.0 | 4 votes |
@Test void getUrl_existingObject_returnsUrlWithS3Prefix() throws Exception { AmazonS3Client amazonS3 = mock(AmazonS3Client.class); when(amazonS3.getRegion()).thenReturn(Region.EU_Ireland); // Act SimpleStorageResource simpleStorageResource = new SimpleStorageResource(amazonS3, "bucket", "object", new SyncTaskExecutor()); // Assert assertThat(simpleStorageResource.getURL()) .isEqualTo(new URL("https://s3.eu-west-1.amazonaws.com/bucket/object")); }
Example #18
Source File: S3MapReduceCpOptions.java From circus-train with Apache License 2.0 | 4 votes |
void setRegion(String region) { Region.fromValue(region); this.region = region; }
Example #19
Source File: DynamoClient.java From xyz-hub with Apache License 2.0 | 4 votes |
boolean isLocal() { return Arrays.stream(Region.values()).noneMatch(r -> r.toAWSRegion().getName().equals(arn.getRegion())); }