Java Code Examples for com.amazonaws.services.s3.AmazonS3ClientBuilder

The following examples show how to use com.amazonaws.services.s3.AmazonS3ClientBuilder. These examples are extracted from open source projects. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source Project: pacbot   Source File: AWSErrorManager.java    License: Apache License 2.0 8 votes vote down vote up
/**
 * Fetch error info.
 *
 * @param datasource the datasource
 * @param errorList the error list
 */
private void fetchErrorInfo(String datasource, List<Map<String,String>> errorList){
	if(errorInfo==null){
		ObjectMapper objectMapper = new ObjectMapper();
    	List<Map<String, String>> inventoryErrors = new ArrayList<>();
    	AmazonS3 s3Client = AmazonS3ClientBuilder.standard()
                .withCredentials(new AWSStaticCredentialsProvider(new CredentialProvider().getCredentials(s3Account,s3Role))).withRegion(s3Region).build();
    	try {
	    	S3Object inventoryErrorData = s3Client.getObject(new GetObjectRequest(bucketName,dataPath+"/"+datasource+"-loaderror.data"));
	    	try (BufferedReader reader = new BufferedReader(new InputStreamReader(inventoryErrorData.getObjectContent()))) {
				inventoryErrors = objectMapper.readValue(reader.lines().collect(Collectors.joining("\n")),new TypeReference<List<Map<String, String>>>() {});
	        }
    	} catch (IOException e) {
    		LOGGER.error("Exception in collecting inventory error data",e);
            Map<String,String> errorMap = new HashMap<>();
            errorMap.put(ERROR, "Exception in collecting inventory error data");
            errorMap.put(ERROR_TYPE, WARN);
            errorMap.put(EXCEPTION, e.getMessage());
            errorList.add(errorMap);
		}
    	errorInfo = inventoryErrors.parallelStream().collect(Collectors.groupingBy(obj -> obj.get("type")));
	}
}
 
Example 2
Source Project: s3proxy   Source File: AwsSdkTest.java    License: Apache License 2.0 7 votes vote down vote up
@Test
public void testAwsV4SignaturePayloadUnsigned() throws Exception {
    client = AmazonS3ClientBuilder.standard()
            .withChunkedEncodingDisabled(true)
            .withPayloadSigningEnabled(false)
            .withCredentials(new AWSStaticCredentialsProvider(awsCreds))
            .withEndpointConfiguration(s3EndpointConfig)
            .build();

    ObjectMetadata metadata = new ObjectMetadata();
    metadata.setContentLength(BYTE_SOURCE.size());
    client.putObject(containerName, "foo",
            BYTE_SOURCE.openStream(), metadata);

    S3Object object = client.getObject(containerName, "foo");
    assertThat(object.getObjectMetadata().getContentLength()).isEqualTo(
            BYTE_SOURCE.size());
    try (InputStream actual = object.getObjectContent();
            InputStream expected = BYTE_SOURCE.openStream()) {
        assertThat(actual).hasContentEqualTo(expected);
    }
}
 
Example 3
Source Project: ecs-sync   Source File: AwsS3Test.java    License: Apache License 2.0 6 votes vote down vote up
@Before
public void setup() throws Exception {
    Properties syncProperties = TestConfig.getProperties();
    endpoint = syncProperties.getProperty(TestConfig.PROP_S3_ENDPOINT);
    accessKey = syncProperties.getProperty(TestConfig.PROP_S3_ACCESS_KEY_ID);
    secretKey = syncProperties.getProperty(TestConfig.PROP_S3_SECRET_KEY);
    region = syncProperties.getProperty(TestConfig.PROP_S3_REGION);
    String proxyUri = syncProperties.getProperty(TestConfig.PROP_HTTP_PROXY_URI);
    Assume.assumeNotNull(endpoint, accessKey, secretKey);
    endpointUri = new URI(endpoint);

    ClientConfiguration config = new ClientConfiguration().withSignerOverride("S3SignerType");
    if (proxyUri != null) {
        URI uri = new URI(proxyUri);
        config.setProxyHost(uri.getHost());
        config.setProxyPort(uri.getPort());
    }

    AmazonS3ClientBuilder builder = AmazonS3ClientBuilder.standard()
            .withCredentials(new AWSStaticCredentialsProvider(new BasicAWSCredentials(accessKey, secretKey)))
            .withClientConfiguration(config)
            .withEndpointConfiguration(new AwsClientBuilder.EndpointConfiguration(endpoint, region));

    s3 = builder.build();
}
 
Example 4
Source Project: aws-doc-sdk-examples   Source File: S3Encrypt.java    License: Apache License 2.0 6 votes vote down vote up
/**
 * Uses an asymmetric key pair instead of a symmetric key. Note this does not change the algorithm used to encrypt
 * the content, that will still be a symmetric key algorithm (AES/CBC in this case) using the derived CEK. It does impact
 * the algorithm used to encrypt the CEK, in this case we use RSA/ECB/OAEPWithSHA-256AndMGF1Padding.
 */
// snippet-start:[s3.java1.s3_encrypt.encryption_only_asymetric_key]
public void encryptionOnly_CustomerManagedAsymetricKey() throws NoSuchAlgorithmException {
    // snippet-start:[s3.java1.s3_encrypt.encryption_only_asymetric_key_build]
    KeyPair keyPair = KeyPairGenerator.getInstance("RSA").generateKeyPair();
    AmazonS3Encryption s3Encryption = AmazonS3EncryptionClientBuilder
            .standard()
            .withRegion(Regions.US_WEST_2)
            .withCryptoConfiguration(new CryptoConfiguration(CryptoMode.EncryptionOnly))
            .withEncryptionMaterials(new StaticEncryptionMaterialsProvider(new EncryptionMaterials(keyPair)))
            .build();

    AmazonS3 s3NonEncrypt = AmazonS3ClientBuilder.standard().withRegion(Regions.DEFAULT_REGION).build();
    // snippet-end:[s3.java1.s3_encrypt.encryption_only_asymetric_key_build]

    // snippet-start:[s3.java1.s3_encrypt.encryption_only_asymetric_key_put_object]
    s3Encryption.putObject(BUCKET_NAME, ENCRYPTED_KEY, "some contents");
    s3NonEncrypt.putObject(BUCKET_NAME, NON_ENCRYPTED_KEY, "some other contents");
    // snippet-end:[s3.java1.s3_encrypt.encryption_only_asymetric_key_put_object]
    // snippet-start:[s3.java1.s3_encrypt.encryption_only_asymetric_key_retrieve]
    System.out.println(s3Encryption.getObjectAsString(BUCKET_NAME, ENCRYPTED_KEY));
    System.out.println(s3Encryption.getObjectAsString(BUCKET_NAME, NON_ENCRYPTED_KEY));
    // snippet-end:[s3.java1.s3_encrypt.encryption_only_asymetric_key_retrieve]
}
 
Example 5
Source Project: aws-doc-sdk-examples   Source File: SetAcl.java    License: Apache License 2.0 6 votes vote down vote up
public static void setObjectAcl(String bucket_name, String object_key, String email, String access) {
    System.out.format("Setting %s access for %s\n", access, email);
    System.out.println("for object: " + object_key);
    System.out.println(" in bucket: " + bucket_name);

    final AmazonS3 s3 = AmazonS3ClientBuilder.standard().withRegion(Regions.DEFAULT_REGION).build();
    try {
        // get the current ACL
        AccessControlList acl = s3.getObjectAcl(bucket_name, object_key);
        // set access for the grantee
        EmailAddressGrantee grantee = new EmailAddressGrantee(email);
        Permission permission = Permission.valueOf(access);
        acl.grantPermission(grantee, permission);
        s3.setObjectAcl(bucket_name, object_key, acl);
    } catch (AmazonServiceException e) {
        System.err.println(e.getErrorMessage());
        System.exit(1);
    }
}
 
Example 6
Source Project: testcontainers-java   Source File: LocalstackContainerTest.java    License: MIT License 6 votes vote down vote up
@Test
public void s3TestOverBridgeNetwork() throws IOException {
    AmazonS3 s3 = AmazonS3ClientBuilder
        .standard()
        .withEndpointConfiguration(localstack.getEndpointConfiguration(S3))
        .withCredentials(localstack.getDefaultCredentialsProvider())
        .build();

    final String bucketName = "foo";
    s3.createBucket(bucketName);
    s3.putObject(bucketName, "bar", "baz");

    final List<Bucket> buckets = s3.listBuckets();
    final Optional<Bucket> maybeBucket = buckets.stream().filter(b -> b.getName().equals(bucketName)).findFirst();
    assertTrue("The created bucket is present", maybeBucket.isPresent());
    final Bucket bucket = maybeBucket.get();

    assertEquals("The created bucket has the right name", bucketName, bucket.getName());

    final ObjectListing objectListing = s3.listObjects(bucketName);
    assertEquals("The created bucket has 1 item in it", 1, objectListing.getObjectSummaries().size());

    final S3Object object = s3.getObject(bucketName, "bar");
    final String content = IOUtils.toString(object.getObjectContent(), Charset.forName("UTF-8"));
    assertEquals("The object can be retrieved", "baz", content);
}
 
Example 7
Source Project: aws-doc-sdk-examples   Source File: ListObjects.java    License: Apache License 2.0 6 votes vote down vote up
public static void main(String[] args) {
    final String USAGE = "\n" +
            "To run this example, supply the name of a bucket to list!\n" +
            "\n" +
            "Ex: ListObjects <bucket-name>\n";

    if (args.length < 1) {
        System.out.println(USAGE);
        System.exit(1);
    }

    String bucket_name = args[0];

    System.out.format("Objects in S3 bucket %s:\n", bucket_name);
    final AmazonS3 s3 = AmazonS3ClientBuilder.standard().withRegion(Regions.DEFAULT_REGION).build();
    ListObjectsV2Result result = s3.listObjectsV2(bucket_name);
    List<S3ObjectSummary> objects = result.getObjectSummaries();
    for (S3ObjectSummary os : objects) {
        System.out.println("* " + os.getKey());
    }
}
 
Example 8
Source Project: spring-cloud-aws   Source File: AmazonS3ClientFactory.java    License: Apache License 2.0 6 votes vote down vote up
AmazonS3 createClientForEndpointUrl(AmazonS3 prototype, String endpointUrl,
		Regions bucketRegion) {
	Assert.notNull(prototype, "AmazonS3 must not be null");
	Assert.notNull(endpointUrl, "Endpoint Url must not be null");

	String region = bucketRegion != null ? bucketRegion.getName()
			: getRegion(endpointUrl);
	Assert.notNull(region,
			"Error detecting region from endpoint url:'" + endpointUrl + "'");

	if (!this.clientCache.containsKey(region)) {
		AmazonS3ClientBuilder amazonS3ClientBuilder = buildAmazonS3ForRegion(
				prototype, region);
		this.clientCache.putIfAbsent(region, amazonS3ClientBuilder.build());
	}

	return this.clientCache.get(region);
}
 
Example 9
Source Project: Flink-CEPplus   Source File: S3UtilProgram.java    License: Apache License 2.0 6 votes vote down vote up
private static void numberOfLinesInFilesWithFullAndNamePrefix(ParameterTool params) {
	final String bucket = params.getRequired("bucket");
	final String s3prefix = params.getRequired("s3prefix");
	final String s3filePrefix = params.get("s3filePrefix", "");
	int parallelism = params.getInt("parallelism", 10);

	List<String> files = listByFullPathPrefix(bucket, s3prefix);

	ExecutorService executor = Executors.newFixedThreadPool(parallelism);
	AmazonS3 s3client = AmazonS3ClientBuilder.defaultClient();
	List<CompletableFuture<Integer>> requests =
		submitLineCountingRequestsForFilesAsync(executor, s3client, bucket, files, s3filePrefix);
	int count = waitAndComputeTotalLineCountResult(requests);

	executor.shutdownNow();
	s3client.shutdown();
	System.out.print(count);
}
 
Example 10
Source Project: flink   Source File: S3UtilProgram.java    License: Apache License 2.0 6 votes vote down vote up
private static void numberOfLinesInFilesWithFullAndNamePrefix(ParameterTool params) {
	final String bucket = params.getRequired("bucket");
	final String s3prefix = params.getRequired("s3prefix");
	final String s3filePrefix = params.get("s3filePrefix", "");
	int parallelism = params.getInt("parallelism", 10);

	List<String> files = listByFullPathPrefix(bucket, s3prefix);

	ExecutorService executor = Executors.newFixedThreadPool(parallelism);
	AmazonS3 s3client = AmazonS3ClientBuilder.defaultClient();
	List<CompletableFuture<Integer>> requests =
		submitLineCountingRequestsForFilesAsync(executor, s3client, bucket, files, s3filePrefix);
	int count = waitAndComputeTotalLineCountResult(requests);

	executor.shutdownNow();
	s3client.shutdown();
	System.out.print(count);
}
 
Example 11
Source Project: aws-doc-sdk-examples   Source File: S3Encrypt.java    License: Apache License 2.0 6 votes vote down vote up
/**
 * This uses the V2 metadata schema with a key wrap algorithm of 'kms' and a CEK algorithm of AES/CBC/PKCS5Padding.
 */
// snippet-start:[s3.java1.s3_encrypt.kms_encryption_only]
public void encryptionOnly_KmsManagedKey() throws NoSuchAlgorithmException {
    // snippet-start:[s3.java1.s3_encrypt.kms_encryption_only_build]
    AmazonS3Encryption s3Encryption = AmazonS3EncryptionClientBuilder
            .standard()
            .withRegion(Regions.US_WEST_2)
            .withCryptoConfiguration(new CryptoConfiguration(CryptoMode.EncryptionOnly).withAwsKmsRegion(Region.getRegion(Regions.US_WEST_2)))
            // Can either be Key ID or alias (prefixed with 'alias/')
            .withEncryptionMaterials(new KMSEncryptionMaterialsProvider("alias/s3-kms-key"))
            .build();

    AmazonS3 s3NonEncrypt = AmazonS3ClientBuilder.standard().withRegion(Regions.DEFAULT_REGION).build();
    // snippet-end:[s3.java1.s3_encrypt.kms_encryption_only_build]

    // snippet-start:[s3.java1.s3_encrypt.kms_encryption_only_put_object]
    s3Encryption.putObject(BUCKET_NAME, ENCRYPTED_KEY, "some contents");
    s3NonEncrypt.putObject(BUCKET_NAME, NON_ENCRYPTED_KEY, "some other contents");
    // snippet-end:[s3.java1.s3_encrypt.kms_encryption_only_put_object]
    // snippet-start:[s3.java1.s3_encrypt.kms_encryption_only_retrieve]
    System.out.println(s3Encryption.getObjectAsString(BUCKET_NAME, ENCRYPTED_KEY));
    System.out.println(s3Encryption.getObjectAsString(BUCKET_NAME, NON_ENCRYPTED_KEY));
    // snippet-end:[s3.java1.s3_encrypt.kms_encryption_only_retrieve]
}
 
Example 12
Source Project: aws-cf-templates   Source File: ACloudFormationTest.java    License: Apache License 2.0 6 votes vote down vote up
protected final void updateStack(final String stackName, final String template, final Parameter... parameters) {
    UpdateStackRequest req = new UpdateStackRequest()
            .withStackName(stackName)
            .withParameters(parameters)
            .withCapabilities(Capability.CAPABILITY_IAM);
    if (Config.has(Config.Key.TEMPLATE_DIR)) {
        final String dir = Config.get(Config.Key.TEMPLATE_DIR);
        if (Config.has(Config.Key.BUCKET_NAME)) {
            final String bucketName = Config.get(Config.Key.BUCKET_NAME);
            final String bucketRegion = Config.get(Config.Key.BUCKET_REGION);
            final AmazonS3 s3local = AmazonS3ClientBuilder.standard().withCredentials(this.credentialsProvider).withRegion(bucketRegion).build();
            s3local.putObject(bucketName, stackName, new File(dir + template));
            req = req.withTemplateURL("https://s3-" + bucketRegion + ".amazonaws.com/" + bucketName + "/" + stackName);
        } else {
            final String body = readFile(dir + template, Charset.forName("UTF-8"));
            req = req.withTemplateBody(body);
        }
    } else {
        req = req.withTemplateURL("https://s3-eu-west-1.amazonaws.com/widdix-aws-cf-templates/" + template);
    }
    this.cf.updateStack(req);
    this.waitForStack(stackName, FinalStatus.UPDATE_COMPLETE);
}
 
Example 13
Source Project: athenz   Source File: CloudStore.java    License: Apache License 2.0 6 votes vote down vote up
public AmazonS3 getS3Client() {

        if (!awsEnabled) {
            throw new ResourceException(ResourceException.INTERNAL_SERVER_ERROR,
                    "AWS Support not enabled");
        }

        if (credentials == null) {
            throw new ResourceException(ResourceException.INTERNAL_SERVER_ERROR,
                    "AWS Role credentials are not available");
        }

        return AmazonS3ClientBuilder.standard()
                .withCredentials(new AWSStaticCredentialsProvider(credentials))
                .withRegion(Regions.fromName(awsRegion))
                .build();
    }
 
Example 14
Source Project: pacbot   Source File: ErrorManager.java    License: Apache License 2.0 6 votes vote down vote up
/**
 * Fetch error info.
 *
 * @param datasource the datasource
 * @param errorList the error list
 */
private void fetchErrorInfo(List<Map<String,String>> errorList){
	if(errorInfo==null){
		ObjectMapper objectMapper = new ObjectMapper();
    	List<Map<String, String>> inventoryErrors = new ArrayList<>();
    	AmazonS3 s3Client = AmazonS3ClientBuilder.standard()
                .withCredentials(new AWSStaticCredentialsProvider(new CredentialProvider().getCredentials(s3Account,s3Role))).withRegion(s3Region).build();
    	try {
	    	S3Object inventoryErrorData = s3Client.getObject(new GetObjectRequest(bucketName,dataPath+"/"+dataSource+"-loaderror.data"));
	    	try (BufferedReader reader = new BufferedReader(new InputStreamReader(inventoryErrorData.getObjectContent()))) {
				inventoryErrors = objectMapper.readValue(reader.lines().collect(Collectors.joining("\n")),new TypeReference<List<Map<String, String>>>() {});
	        }
    	} catch (IOException e) {
    		LOGGER.error("Exception in collecting inventory error data",e);
            Map<String,String> errorMap = new HashMap<>();
            errorMap.put(ERROR, "Exception in collecting inventory error data");
            errorMap.put(ERROR_TYPE, WARN);
            errorMap.put(EXCEPTION, e.getMessage());
            errorList.add(errorMap);
		}
    	errorInfo = inventoryErrors.parallelStream().collect(Collectors.groupingBy(obj -> obj.get("type")));
	}
}
 
Example 15
Source Project: genie   Source File: S3ClientFactory.java    License: Apache License 2.0 6 votes vote down vote up
private AmazonS3 buildS3Client(final S3ClientKey s3ClientKey) {
    // TODO: Do something about allowing ClientConfiguration to be passed in
    return AmazonS3ClientBuilder
        .standard()
        .withRegion(s3ClientKey.getRegion())
        .withForceGlobalBucketAccessEnabled(true)
        .withCredentials(
            s3ClientKey
                .getRoleARN()
                .map(
                    roleARN -> {
                        // TODO: Perhaps rename with more detailed info?
                        final String roleSession = "Genie-Agent-" + UUID.randomUUID().toString();

                        return (AWSCredentialsProvider) new STSAssumeRoleSessionCredentialsProvider
                            .Builder(roleARN, roleSession)
                            .withStsClient(this.stsClient)
                            .build();
                    }
                )
                .orElse(this.awsCredentialsProvider)
        )
        .build();
}
 
Example 16
Source Project: rdf-delta   Source File: S3.java    License: Apache License 2.0 6 votes vote down vote up
public static AmazonS3 buildS3(LocalServerConfig configuration) {
    String region = configuration.getProperty(pRegion);
    String endpoint = configuration.getProperty(pEndpoint);
    String credentialsFile =  configuration.getProperty(pCredentialFile);
    String credentialsProfile =  configuration.getProperty(pCredentialProfile);

    AmazonS3ClientBuilder builder = AmazonS3ClientBuilder.standard();
    if ( endpoint == null )
        builder.withRegion(region);
    else  {
        // Needed for S3mock
        builder.withPathStyleAccessEnabled(true);
        builder.withEndpointConfiguration(new EndpointConfiguration(endpoint, region));
        builder.withCredentials(new AWSStaticCredentialsProvider(new AnonymousAWSCredentials()));
    }
    if ( credentialsFile != null )
        builder.withCredentials(new ProfileCredentialsProvider(credentialsFile, credentialsProfile));
    return builder.build();
}
 
Example 17
Source Project: oodt   Source File: S3DataTransfererFactory.java    License: Apache License 2.0 6 votes vote down vote up
@Override
public S3DataTransferer createDataTransfer() {
String bucketName = System.getProperty(BUCKET_NAME_PROPERTY);
String region = System.getProperty(REGION_PROPERTY);		
String accessKey = System.getProperty(ACCESS_KEY_PROPERTY);
  String secretKey = System.getProperty(SECRET_KEY_PROPERTY);
  boolean encrypt = Boolean.getBoolean(ENCRYPT_PROPERTY);

AmazonS3Client s3 = (AmazonS3Client) AmazonS3ClientBuilder.standard()
        .withRegion(region)
        .withCredentials(
                new AWSStaticCredentialsProvider(
                        new BasicAWSCredentials(accessKey, secretKey)))
        .build();

  return new S3DataTransferer(s3, bucketName, encrypt);
}
 
Example 18
Source Project: aws-xray-sdk-java   Source File: TracingHandlerTest.java    License: Apache License 2.0 6 votes vote down vote up
@Test
public void testS3PutObjectSubsegmentContainsBucketName() {
    // Setup test
    AmazonS3 s3 = AmazonS3ClientBuilder
        .standard()
        .withRequestHandlers(new TracingHandler())
        .withRegion(Regions.US_EAST_1)
        .withCredentials(new AWSStaticCredentialsProvider(new BasicAWSCredentials("fake", "fake")))
        .build();
    mockHttpClient(s3, null);

    String bucket = "test-bucket";
    String key = "test-key";
    // Test logic 
    Segment segment = AWSXRay.beginSegment("test");
    s3.putObject(bucket, key, "This is a test from java");
    Assert.assertEquals(1, segment.getSubsegments().size());
    Assert.assertEquals("PutObject", segment.getSubsegments().get(0).getAws().get("operation"));
    Assert.assertEquals(bucket, segment.getSubsegments().get(0).getAws().get("bucket_name"));
    Assert.assertEquals(key, segment.getSubsegments().get(0).getAws().get("key"));
}
 
Example 19
Source Project: camel-kafka-connector   Source File: AWSClientUtils.java    License: Apache License 2.0 5 votes vote down vote up
public static AmazonS3 newS3Client() {
    LOG.debug("Creating a new S3 client");
    AmazonS3ClientBuilder clientBuilder = AmazonS3ClientBuilder.standard();

    String awsInstanceType = System.getProperty("aws-service.instance.type");
    String region = getRegion();

    if (awsInstanceType == null || awsInstanceType.equals("local-aws-container")) {
        String amazonHost = System.getProperty(AWSConfigs.AMAZON_AWS_HOST);
        ClientConfiguration clientConfiguration = new ClientConfiguration();
        clientConfiguration.setProtocol(Protocol.HTTP);

        clientBuilder
                .withEndpointConfiguration(new AwsClientBuilder.EndpointConfiguration(amazonHost, region))
                .withClientConfiguration(clientConfiguration)
                .withCredentials(new TestAWSCredentialsProvider("accesskey", "secretkey"));
    } else {
        clientBuilder
                .withRegion(region)
                .withCredentials(new TestAWSCredentialsProvider());
    }

    clientBuilder
            .withPathStyleAccessEnabled(true);

    return clientBuilder.build();
}
 
Example 20
Source Project: aws-doc-sdk-examples   Source File: DeleteObjects.java    License: Apache License 2.0 5 votes vote down vote up
public static void main(String[] args) {
    final String USAGE = "\n" +
            "To run this example, supply the name of an S3 bucket and at least\n" +
            "one object name (key) to delete.\n" +
            "\n" +
            "Ex: DeleteObjects <bucketname> <objectname1> [objectname2, ...]\n";

    if (args.length < 2) {
        System.out.println(USAGE);
        System.exit(1);
    }

    String bucket_name = args[0];
    String[] object_keys = Arrays.copyOfRange(args, 1, args.length);

    System.out.println("Deleting objects from S3 bucket: " + bucket_name);
    for (String k : object_keys) {
        System.out.println(" * " + k);
    }

    final AmazonS3 s3 = AmazonS3ClientBuilder.standard().withRegion(Regions.DEFAULT_REGION).build();
    try {
        DeleteObjectsRequest dor = new DeleteObjectsRequest(bucket_name)
                .withKeys(object_keys);
        s3.deleteObjects(dor);
    } catch (AmazonServiceException e) {
        System.err.println(e.getErrorMessage());
        System.exit(1);
    }
    System.out.println("Done!");
}
 
Example 21
Source Project: aws-doc-sdk-examples   Source File: CreateBucket.java    License: Apache License 2.0 5 votes vote down vote up
public static Bucket createBucket(String bucket_name) {
    final AmazonS3 s3 = AmazonS3ClientBuilder.standard().withRegion(Regions.DEFAULT_REGION).build();
    Bucket b = null;
    if (s3.doesBucketExistV2(bucket_name)) {
        System.out.format("Bucket %s already exists.\n", bucket_name);
        b = getBucket(bucket_name);
    } else {
        try {
            b = s3.createBucket(bucket_name);
        } catch (AmazonS3Exception e) {
            System.err.println(e.getErrorMessage());
        }
    }
    return b;
}
 
Example 22
Source Project: aws-athena-query-federation   Source File: RecordHandler.java    License: Apache License 2.0 5 votes vote down vote up
/**
 * @param sourceType Used to aid in logging diagnostic info when raising a support case.
 */
public RecordHandler(String sourceType)
{
    this.sourceType = sourceType;
    this.amazonS3 = AmazonS3ClientBuilder.defaultClient();
    this.secretsManager = new CachableSecretsManager(AWSSecretsManagerClientBuilder.defaultClient());
    this.athena = AmazonAthenaClientBuilder.defaultClient();
}
 
Example 23
Source Project: s3proxy   Source File: S3ProxyRuleTest.java    License: Apache License 2.0 5 votes vote down vote up
@Before
public final void setUp() throws Exception {
    s3Client = AmazonS3ClientBuilder
        .standard()
        .withCredentials(
            new AWSStaticCredentialsProvider(
                new BasicAWSCredentials(
                    s3Proxy.getAccessKey(), s3Proxy.getSecretKey())))
        .withEndpointConfiguration(
            new EndpointConfiguration(
                s3Proxy.getUri().toString(), Regions.US_EAST_1.getName()))
        .build();

    s3Client.createBucket(MY_TEST_BUCKET);
}
 
Example 24
public CloudwatchRecordHandler()
{
    this(AmazonS3ClientBuilder.defaultClient(),
            AWSSecretsManagerClientBuilder.defaultClient(),
            AmazonAthenaClientBuilder.defaultClient(),
            AWSLogsClientBuilder.defaultClient());
}
 
Example 25
public DocDBRecordHandler()
{
    this(AmazonS3ClientBuilder.defaultClient(),
            AWSSecretsManagerClientBuilder.defaultClient(),
            AmazonAthenaClientBuilder.defaultClient(),
            new DocDBConnectionFactory());
}
 
Example 26
Source Project: aws-doc-sdk-examples   Source File: ListBuckets.java    License: Apache License 2.0 5 votes vote down vote up
public static void main(String[] args) {
    final AmazonS3 s3 = AmazonS3ClientBuilder.standard().withRegion(Regions.DEFAULT_REGION).build();
    List<Bucket> buckets = s3.listBuckets();
    System.out.println("Your Amazon S3 buckets are:");
    for (Bucket b : buckets) {
        System.out.println("* " + b.getName());
    }
}
 
Example 27
Source Project: iaf   Source File: AmazonS3FileSystem.java    License: Apache License 2.0 5 votes vote down vote up
@Override
public void open() {
	CredentialFactory cf = new CredentialFactory(getAuthAlias(), getAccessKey(), getSecretKey());
	BasicAWSCredentials awsCreds = new BasicAWSCredentials(cf.getUsername(), cf.getPassword());
	AmazonS3ClientBuilder s3ClientBuilder = AmazonS3ClientBuilder.standard()
			.withChunkedEncodingDisabled(isChunkedEncodingDisabled())
			.withForceGlobalBucketAccessEnabled(isForceGlobalBucketAccessEnabled()).withRegion(getClientRegion())
			.withCredentials(new AWSStaticCredentialsProvider(awsCreds))
			.withClientConfiguration(this.getProxyConfig());
	s3Client = s3ClientBuilder.build();
}
 
Example 28
Source Project: Flink-CEPplus   Source File: S3UtilProgram.java    License: Apache License 2.0 5 votes vote down vote up
private static void deleteByFullPathPrefix(ParameterTool params) {
	final String bucket = params.getRequired("bucket");
	final String s3prefix = params.getRequired("s3prefix");
	String[] keys = listByFullPathPrefix(bucket, s3prefix).toArray(new String[] {});
	if (keys.length > 0) {
		DeleteObjectsRequest request = new DeleteObjectsRequest(bucket).withKeys(keys);
		AmazonS3ClientBuilder.defaultClient().deleteObjects(request);
	}
}
 
Example 29
Source Project: Flink-CEPplus   Source File: S3UtilProgram.java    License: Apache License 2.0 5 votes vote down vote up
private static void numberOfLinesInFile(ParameterTool params) {
	final String bucket = params.getRequired("bucket");
	final String s3file = params.getRequired("s3file");
	AmazonS3 s3client = AmazonS3ClientBuilder.defaultClient();
	System.out.print(S3QueryUtil.queryFile(s3client, bucket, s3file, countQuery));
	s3client.shutdown();
}
 
Example 30
Source Project: graylog-plugin-aws   Source File: S3Reader.java    License: Apache License 2.0 5 votes vote down vote up
public S3Reader(Region region, HttpUrl proxyUrl, AWSAuthProvider authProvider) {
    AmazonS3ClientBuilder clientBuilder = AmazonS3ClientBuilder.standard().withRegion(region.getName()).withCredentials(authProvider);

    if(proxyUrl != null) {
        clientBuilder.withClientConfiguration(Proxy.forAWS(proxyUrl));
    }

    this.client = clientBuilder.build();
}